feat(apple): gamepad UI v2 — controller settings + add host, aurora, macOS

Sources reorganized (client: Home/Session/Settings/Stores/Support/Trust; kit:
Audio/Connection/Gamepad/Input/Support/Video/Views) with the big files split
along the same seams.

The gamepad mode is couch-complete, and now on macOS too (the living-room
Mac case), not just iOS/iPadOS:

- GamepadSettingsView: a console-style, fully controller-navigable settings
  screen (X from the launcher) — up/down moves focus, left/right steps values
  (clamped, boundary thud), A cycles/toggles, B closes; the focused row shows a
  one-line description. Backed by GamepadMenuList, the vertical sibling of
  GamepadCarousel, and SettingsOptions — the option lists hoisted out of
  SettingsView statics and shared by the touch, tvOS and gamepad settings.
- GamepadAddHostView + GamepadKeyboard: register a host end to end with a pad
  — field rows open an on-screen controller keyboard (dpad grid, A types,
  X backspaces, B done); the launcher carousel ends in an Add Host tile, so
  the dead-end "add one with touch first" empty state is gone.
- Launcher polish: contextual hint bar with the pad's real button glyphs,
  controller name + battery chip, one shared console chrome.
- GamepadScreenBackground: an animated aurora (TimelineView-driven drifting
  blobs in the brand's violet family, breathing radii, slow hue shift,
  legibility scrim; freezes under Reduce Motion). Pure SwiftUI on purpose — a
  .metal library only bundles reliably in one of the two build systems (SPM vs
  the xcodeproj's synced folders) these sources compile under.
- macOS port: settings/add-host/library present as sized sheets (a macOS sheet
  takes its content's IDEAL size, and the GeometryReader-driven screens
  collapsed to nothing), NSScreen-based mode lists, scroll indicators .never
  (the "always show scroll bars" setting overrides .hidden), tray scrims so
  scrolled rows dim under the pinned title/hints, extra title clearance, and a
  PUNKTFUNK_FORCE_GAMEPAD_UI=1 dev hook — launcher/settings/add-host/keyboard/
  library render-verified live on a real Mac + LAN hosts.
- GamepadMenuInput: X button support, and (re)start now snapshots held buttons
  so a controller handoff press never fires twice (the B that closed the
  keyboard no longer also cancels the screen underneath).
- Cleanups: one "Connection failed" alert in ContentView instead of one per
  home screen; HostDiscovery.advertises/unsaved shared by both home screens.
- host: can_encode_444 stub for the non-Linux/Windows host build (the macOS
  synthetic-source loopback used by the Swift tests).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-02 11:05:10 +02:00
parent e925d00194
commit 133e25849d
84 changed files with 4231 additions and 2698 deletions
@@ -1,202 +0,0 @@
// Annex-B HEVC CoreMedia plumbing.
//
// The punktfunk host emits Annex-B access units with in-band VPS/SPS/PPS on every IDR
// (deliberately the client needs no out-of-band extradata). VideoToolbox wants the AVCC
// flavor instead: a CMVideoFormatDescription built from the parameter sets, and sample
// buffers whose NALs are 4-byte-length-prefixed. This file converts between the two.
//
// SCAFFOLD: written on the Linux host, not yet compiled against Xcode.
import CoreMedia
import Foundation
/// The video codec of the host's elementary stream negotiated in the Welcome and read via
/// `punktfunk_connection_codec`. Both are Annex-B with in-band parameter sets on every IDR; they
/// differ only in NAL-header layout and which parameter sets exist (HEVC adds a VPS). AV1 is not an
/// Annex-B/NAL codec and isn't handled here (hosts don't emit it on the native path yet).
public enum VideoCodec: Equatable {
case h264
case hevc
/// Resolve from the wire `Welcome.codec` byte (`PUNKTFUNK_CODEC_*`; unknown HEVC).
public init(wire: UInt8) {
self = wire == 0x01 ? .h264 : .hevc // 0x01 = PUNKTFUNK_CODEC_H264
}
}
public enum AnnexB {
/// Split an Annex-B stream into NAL units (start codes 00 00 01 / 00 00 00 01 stripped).
/// All zeros immediately preceding a start code are dropped: they're either the
/// 4-byte-code prefix or `trailing_zero_8bits` padding, never NAL payload (emulation
/// prevention keeps 00 00 0x out of conforming NAL bytes) same policy as ffmpeg.
public static func nalUnits(in data: Data) -> [Data] {
var nals: [Data] = []
let bytes = [UInt8](data)
var i = 0
var start = -1
while i + 2 < bytes.count {
if bytes[i] == 0, bytes[i + 1] == 0, bytes[i + 2] == 1 {
var codeStart = i
while codeStart > 0, bytes[codeStart - 1] == 0 {
codeStart -= 1
}
if start >= 0, start < codeStart {
nals.append(Data(bytes[start..<codeStart]))
}
start = i + 3
i += 3
} else {
i += 1
}
}
if start >= 0, start < bytes.count {
nals.append(Data(bytes[start...]))
}
return nals
}
/// HEVC NAL unit type (bits 1..6 of the first byte).
public static func hevcNalType(_ nal: Data) -> UInt8 {
guard let first = nal.first else { return 0xFF }
return (first >> 1) & 0x3F
}
/// H.264 NAL unit type (bits 0..4 of the first byte).
public static func h264NalType(_ nal: Data) -> UInt8 {
guard let first = nal.first else { return 0xFF }
return first & 0x1F
}
/// True if this NAL is a parameter set for `codec` (dropped from AVCC; kept for the format desc).
/// HEVC: VPS 32 / SPS 33 / PPS 34. H.264: SPS 7 / PPS 8 (no VPS).
private static func isParameterSet(_ nal: Data, _ codec: VideoCodec) -> Bool {
switch codec {
case .hevc: let t = hevcNalType(nal); return t == 32 || t == 33 || t == 34
case .h264: let t = h264NalType(nal); return t == 7 || t == 8
}
}
/// Build a format description from an IDR AU's in-band parameter sets (HEVC: VPS/SPS/PPS;
/// H.264: SPS/PPS). Returns nil when the AU carries no parameter sets (non-IDR).
public static func formatDescription(fromIDR au: Data, codec: VideoCodec)
-> CMVideoFormatDescription?
{
// Collect the parameter-set NALs in the order VideoToolbox wants them (HEVC: VPS,SPS,PPS;
// H.264: SPS,PPS).
var vps: Data?, sps: Data?, pps: Data?
for nal in nalUnits(in: au) {
switch codec {
case .hevc:
switch hevcNalType(nal) {
case 32: vps = nal
case 33: sps = nal
case 34: pps = nal
default: break
}
case .h264:
switch h264NalType(nal) {
case 7: sps = nal
case 8: pps = nal
default: break
}
}
}
guard let sps, let pps else { return nil }
let sets: [Data] = codec == .hevc ? [vps, sps, pps].compactMap { $0 } : [sps, pps]
guard codec == .h264 || sets.count == 3 else { return nil } // HEVC needs the VPS too
var format: CMVideoFormatDescription?
// Pin every parameter set's bytes for the duration of the create call, then hand
// VideoToolbox parallel pointer/size arrays.
var pointers: [UnsafePointer<UInt8>] = []
var sizes: [Int] = []
func withAll(_ i: Int, _ body: () -> Void) {
if i == sets.count { body(); return }
sets[i].withUnsafeBytes { raw in
pointers.append(raw.bindMemory(to: UInt8.self).baseAddress!)
sizes.append(sets[i].count)
withAll(i + 1, body)
}
}
var status: OSStatus = -1
withAll(0) {
switch codec {
case .hevc:
status = CMVideoFormatDescriptionCreateFromHEVCParameterSets(
allocator: kCFAllocatorDefault,
parameterSetCount: pointers.count,
parameterSetPointers: pointers,
parameterSetSizes: sizes,
nalUnitHeaderLength: 4,
extensions: nil,
formatDescriptionOut: &format)
case .h264:
status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
allocator: kCFAllocatorDefault,
parameterSetCount: pointers.count,
parameterSetPointers: pointers,
parameterSetSizes: sizes,
nalUnitHeaderLength: 4,
formatDescriptionOut: &format)
}
}
return status == noErr ? format : nil
}
/// Re-pack an Annex-B AU as AVCC (4-byte big-endian length before each NAL), dropping
/// the parameter-set NALs (they live in the format description).
public static func avcc(from au: Data, codec: VideoCodec) -> Data {
var out = Data(capacity: au.count + 16)
for nal in nalUnits(in: au) {
if isParameterSet(nal, codec) { continue }
var len = UInt32(nal.count).bigEndian
withUnsafeBytes(of: &len) { out.append(contentsOf: $0) }
out.append(nal)
}
return out
}
/// Wrap one AU as a decode-ready CMSampleBuffer.
public static func sampleBuffer(
au: AccessUnit, format: CMVideoFormatDescription, codec: VideoCodec
) -> CMSampleBuffer? {
let avccData = avcc(from: au.data, codec: codec)
var blockBuffer: CMBlockBuffer?
guard CMBlockBufferCreateWithMemoryBlock(
allocator: kCFAllocatorDefault, memoryBlock: nil,
blockLength: avccData.count, blockAllocator: kCFAllocatorDefault,
customBlockSource: nil, offsetToData: 0, dataLength: avccData.count,
flags: 0, blockBufferOut: &blockBuffer) == noErr,
let block = blockBuffer
else { return nil }
let copied = avccData.withUnsafeBytes { raw in
CMBlockBufferReplaceDataBytes(
with: raw.baseAddress!, blockBuffer: block,
offsetIntoDestination: 0, dataLength: avccData.count)
}
guard copied == noErr else { return nil }
var timing = CMSampleTimingInfo(
duration: .invalid,
presentationTimeStamp: CMTime(value: Int64(au.ptsNs), timescale: 1_000_000_000),
decodeTimeStamp: .invalid)
var sampleSize = avccData.count
var sample: CMSampleBuffer?
guard CMSampleBufferCreate(
allocator: kCFAllocatorDefault, dataBuffer: block, dataReady: true,
makeDataReadyCallback: nil, refcon: nil, formatDescription: format,
sampleCount: 1, sampleTimingEntryCount: 1, sampleTimingArray: &timing,
sampleSizeEntryCount: 1, sampleSizeArray: &sampleSize,
sampleBufferOut: &sample) == noErr
else { return nil }
// Low-latency display: render on arrival, don't wait for a clock.
if let attachments = CMSampleBufferGetSampleAttachmentsArray(sample!, createIfNecessary: true) {
let dict = unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self)
CFDictionarySetValue(
dict,
Unmanaged.passUnretained(kCMSampleAttachmentKey_DisplayImmediately).toOpaque(),
Unmanaged.passUnretained(kCFBooleanTrue).toOpaque())
}
return sample
}
}
@@ -0,0 +1,129 @@
import AVFoundation
import os
/// SPSC-ish jitter ring (interleaved float, `channels` per frame), drain thread render
/// callback. The unfair lock is held for microseconds; fine at render-callback rates. Priming:
/// reads return silence until enough is buffered (at least `prefill`, and at least one
/// packet more than the device's render quantum large-buffer devices would otherwise
/// chronically out-demand the prefill and oscillate prime dropout re-prime), and an
/// underrun re-primes, concealing jitter as one short dip instead of sustained crackle.
/// All counts stay whole frames (multiples of `channels`), so the interleave can never slip.
final class AudioRing: @unchecked Sendable {
private var buf: [Float]
private var readIdx = 0
private var writeIdx = 0
private var primed = false
private var renderQuantum = 0
private let prefill: Int
private let highWater: Int
private let channels: Int
private let lock = OSAllocatedUnfairLock()
/// `capacity`/`prefill` in samples (interleaved `channels` per frame, both whole frames).
init(capacity: Int, prefill: Int, channels: Int) {
buf = [Float](repeating: 0, count: capacity)
self.prefill = prefill
self.channels = channels
highWater = prefill * 4
}
func write(_ samples: UnsafePointer<Float>, count: Int) {
lock.lock()
defer { lock.unlock() }
let capacity = buf.count
// A single write larger than the whole ring would push readIdx PAST writeIdx below
// (inverting the valid range corruption). It never happens (one decoded packet is far
// under capacity), but guard rather than corrupt.
guard count <= capacity else { return }
if writeIdx + count - readIdx > capacity {
readIdx = writeIdx + count - capacity // overflow: drop oldest
}
for i in 0..<count {
buf[(writeIdx + i) % capacity] = samples[i]
}
writeIdx += count
// Latency clamp: both ends run at 48 kHz, so backlog from a network stall (or
// creeping host-vs-DAC clock skew) never drains on its own without this, one
// 300 ms hiccup leaves audio 300 ms behind video for the rest of the session.
// Shedding down to 2× prefill costs one audible blip instead.
if writeIdx - readIdx > highWater {
readIdx = writeIdx - prefill * 2
}
}
/// Fills `out` completely (silence beyond what's buffered).
func read(into out: UnsafeMutablePointer<Float>, count: Int) {
lock.lock()
defer { lock.unlock() }
renderQuantum = max(renderQuantum, count)
let available = writeIdx - readIdx
if !primed {
// One 5 ms host packet (240 frames × channels) of slack beyond the device's demand.
if available >= max(prefill, renderQuantum + 240 * channels) {
primed = true
} else {
for i in 0..<count { out[i] = 0 }
return
}
}
let n = min(available, count)
let capacity = buf.count
for i in 0..<n {
out[i] = buf[(readIdx + i) % capacity]
}
readIdx += n
if n < count {
for i in n..<count { out[i] = 0 }
primed = false // underrun re-prime before resuming
}
}
}
/// CoreAudio channel layout for the canonical wire order FL FR FC LFE RL RR [SL SR]. nil for
/// stereo (the standard layout is correct). For 5.1/7.1 we list explicit channel labels via
/// `kAudioChannelLayoutTag_UseChannelDescriptions` preset tags (DTS_5_1 etc.) don't reliably
/// match Moonlight's order. NB the 7.1 mapping (verified against the WASAPI 0x63F + SPA orderings):
/// wire idx 4-5 = RL/RR = the WAVE *back* pair LeftSurround/RightSurround; idx 6-7 = SL/SR = the
/// WAVE *side* pair LeftSurroundDirect/RightSurroundDirect. (Using RearSurround* for 6-7 would
/// swap side/back vs the Windows/Linux clients.)
func wireChannelLayout(channels: Int) -> AVAudioChannelLayout? {
let labels: [AudioChannelLabel]
switch channels {
case 6:
labels = [
kAudioChannelLabel_Left, kAudioChannelLabel_Right, kAudioChannelLabel_Center,
kAudioChannelLabel_LFEScreen, kAudioChannelLabel_LeftSurround,
kAudioChannelLabel_RightSurround,
]
case 8:
labels = [
kAudioChannelLabel_Left, kAudioChannelLabel_Right, kAudioChannelLabel_Center,
kAudioChannelLabel_LFEScreen,
kAudioChannelLabel_LeftSurround, kAudioChannelLabel_RightSurround, // wire RL/RR (back)
kAudioChannelLabel_LeftSurroundDirect, kAudioChannelLabel_RightSurroundDirect, // wire SL/SR (side)
]
default:
return nil
}
let size = MemoryLayout<AudioChannelLayout>.size
+ (labels.count - 1) * MemoryLayout<AudioChannelDescription>.stride
let raw = UnsafeMutableRawPointer.allocate(byteCount: size, alignment: 16)
defer { raw.deallocate() }
let layout = raw.bindMemory(to: AudioChannelLayout.self, capacity: 1)
layout.pointee.mChannelLayoutTag = kAudioChannelLayoutTag_UseChannelDescriptions
layout.pointee.mChannelBitmap = AudioChannelBitmap(rawValue: 0)
layout.pointee.mNumberChannelDescriptions = UInt32(labels.count)
// `mChannelDescriptions` is the C variable-length tail array (declared `[1]`, over-allocated
// above). Scope the pointer with `withUnsafeMutablePointer` taking `&mChannelDescriptions`
// inline yields a pointer valid only for that expression, so building a buffer from it that
// outlives the call is a dangling-pointer bug. Inside the closure it stays valid while we fill it.
withUnsafeMutablePointer(to: &layout.pointee.mChannelDescriptions) { tail in
let descs = UnsafeMutableBufferPointer(start: tail, count: labels.count)
for (i, lbl) in labels.enumerated() {
descs[i] = AudioChannelDescription(
mChannelLabel: lbl, mChannelFlags: AudioChannelFlags(rawValue: 0),
mCoordinates: (0, 0, 0))
}
}
return AVAudioChannelLayout(layout: layout)
}
@@ -19,99 +19,6 @@ import os
private let log = Logger(subsystem: "io.unom.punktfunk", category: "audio")
/// SPSC-ish jitter ring (interleaved float, `channels` per frame), drain thread render
/// callback. The unfair lock is held for microseconds; fine at render-callback rates. Priming:
/// reads return silence until enough is buffered (at least `prefill`, and at least one
/// packet more than the device's render quantum large-buffer devices would otherwise
/// chronically out-demand the prefill and oscillate prime dropout re-prime), and an
/// underrun re-primes, concealing jitter as one short dip instead of sustained crackle.
/// All counts stay whole frames (multiples of `channels`), so the interleave can never slip.
final class AudioRing: @unchecked Sendable {
private var buf: [Float]
private var readIdx = 0
private var writeIdx = 0
private var primed = false
private var renderQuantum = 0
private let prefill: Int
private let highWater: Int
private let channels: Int
private let lock = OSAllocatedUnfairLock()
/// `capacity`/`prefill` in samples (interleaved `channels` per frame, both whole frames).
init(capacity: Int, prefill: Int, channels: Int) {
buf = [Float](repeating: 0, count: capacity)
self.prefill = prefill
self.channels = channels
highWater = prefill * 4
}
func write(_ samples: UnsafePointer<Float>, count: Int) {
lock.lock()
defer { lock.unlock() }
let capacity = buf.count
// A single write larger than the whole ring would push readIdx PAST writeIdx below
// (inverting the valid range corruption). It never happens (one decoded packet is far
// under capacity), but guard rather than corrupt.
guard count <= capacity else { return }
if writeIdx + count - readIdx > capacity {
readIdx = writeIdx + count - capacity // overflow: drop oldest
}
for i in 0..<count {
buf[(writeIdx + i) % capacity] = samples[i]
}
writeIdx += count
// Latency clamp: both ends run at 48 kHz, so backlog from a network stall (or
// creeping host-vs-DAC clock skew) never drains on its own without this, one
// 300 ms hiccup leaves audio 300 ms behind video for the rest of the session.
// Shedding down to 2× prefill costs one audible blip instead.
if writeIdx - readIdx > highWater {
readIdx = writeIdx - prefill * 2
}
}
/// Fills `out` completely (silence beyond what's buffered).
func read(into out: UnsafeMutablePointer<Float>, count: Int) {
lock.lock()
defer { lock.unlock() }
renderQuantum = max(renderQuantum, count)
let available = writeIdx - readIdx
if !primed {
// One 5 ms host packet (240 frames × channels) of slack beyond the device's demand.
if available >= max(prefill, renderQuantum + 240 * channels) {
primed = true
} else {
for i in 0..<count { out[i] = 0 }
return
}
}
let n = min(available, count)
let capacity = buf.count
for i in 0..<n {
out[i] = buf[(readIdx + i) % capacity]
}
readIdx += n
if n < count {
for i in n..<count { out[i] = 0 }
primed = false // underrun re-prime before resuming
}
}
}
private final class StopFlag: @unchecked Sendable {
private let lock = NSLock()
private var stopped = false
var isStopped: Bool {
lock.lock()
defer { lock.unlock() }
return stopped
}
func stop() {
lock.lock()
stopped = true
lock.unlock()
}
}
/// Render-block-owned scratch storage: freed exactly when the closure (and thus the
/// last possible render call) is released never racing CoreAudio.
private final class ScratchBuffer {
@@ -120,55 +27,6 @@ private final class ScratchBuffer {
deinit { ptr.deallocate() }
}
/// CoreAudio channel layout for the canonical wire order FL FR FC LFE RL RR [SL SR]. nil for
/// stereo (the standard layout is correct). For 5.1/7.1 we list explicit channel labels via
/// `kAudioChannelLayoutTag_UseChannelDescriptions` preset tags (DTS_5_1 etc.) don't reliably
/// match Moonlight's order. NB the 7.1 mapping (verified against the WASAPI 0x63F + SPA orderings):
/// wire idx 4-5 = RL/RR = the WAVE *back* pair LeftSurround/RightSurround; idx 6-7 = SL/SR = the
/// WAVE *side* pair LeftSurroundDirect/RightSurroundDirect. (Using RearSurround* for 6-7 would
/// swap side/back vs the Windows/Linux clients.)
private func wireChannelLayout(channels: Int) -> AVAudioChannelLayout? {
let labels: [AudioChannelLabel]
switch channels {
case 6:
labels = [
kAudioChannelLabel_Left, kAudioChannelLabel_Right, kAudioChannelLabel_Center,
kAudioChannelLabel_LFEScreen, kAudioChannelLabel_LeftSurround,
kAudioChannelLabel_RightSurround,
]
case 8:
labels = [
kAudioChannelLabel_Left, kAudioChannelLabel_Right, kAudioChannelLabel_Center,
kAudioChannelLabel_LFEScreen,
kAudioChannelLabel_LeftSurround, kAudioChannelLabel_RightSurround, // wire RL/RR (back)
kAudioChannelLabel_LeftSurroundDirect, kAudioChannelLabel_RightSurroundDirect, // wire SL/SR (side)
]
default:
return nil
}
let size = MemoryLayout<AudioChannelLayout>.size
+ (labels.count - 1) * MemoryLayout<AudioChannelDescription>.stride
let raw = UnsafeMutableRawPointer.allocate(byteCount: size, alignment: 16)
defer { raw.deallocate() }
let layout = raw.bindMemory(to: AudioChannelLayout.self, capacity: 1)
layout.pointee.mChannelLayoutTag = kAudioChannelLayoutTag_UseChannelDescriptions
layout.pointee.mChannelBitmap = AudioChannelBitmap(rawValue: 0)
layout.pointee.mNumberChannelDescriptions = UInt32(labels.count)
// `mChannelDescriptions` is the C variable-length tail array (declared `[1]`, over-allocated
// above). Scope the pointer with `withUnsafeMutablePointer` taking `&mChannelDescriptions`
// inline yields a pointer valid only for that expression, so building a buffer from it that
// outlives the call is a dangling-pointer bug. Inside the closure it stays valid while we fill it.
withUnsafeMutablePointer(to: &layout.pointee.mChannelDescriptions) { tail in
let descs = UnsafeMutableBufferPointer(start: tail, count: labels.count)
for (i, lbl) in labels.enumerated() {
descs[i] = AudioChannelDescription(
mChannelLabel: lbl, mChannelFlags: AudioChannelFlags(rawValue: 0),
mCoordinates: (0, 0, 0))
}
}
return AVAudioChannelLayout(layout: layout)
}
public final class SessionAudio {
private let connection: PunktfunkConnection
private let flag = StopFlag()
@@ -0,0 +1,59 @@
// The client's persistent identity + the SPAKE2 PIN pairing ceremony the trust
// bootstrap that precedes any pinned PunktfunkConnection.
import Foundation
import PunktfunkCore
/// This client's persistent self-signed identity. Generate ONCE with `generateIdentity()`,
/// store both PEMs (Keychain), present on every connect the certificate's fingerprint is
/// how hosts recognize this client after pairing.
public struct ClientIdentity: Sendable {
public let certPEM: String
public let keyPEM: String
public init(certPEM: String, keyPEM: String) {
self.certPEM = certPEM
self.keyPEM = keyPEM
}
}
/// Generate a fresh client identity (self-signed cert + key, PEM).
public func generateIdentity() throws -> ClientIdentity {
var cert = [CChar](repeating: 0, count: 4096)
var key = [CChar](repeating: 0, count: 4096)
let rc = punktfunk_generate_identity(&cert, UInt(cert.count), &key, UInt(key.count))
guard rc == PUNKTFUNK_STATUS_OK.rawValue else {
throw PunktfunkClientError.status(rc)
}
return ClientIdentity(certPEM: String(cString: cert), keyPEM: String(cString: key))
}
/// Run the PIN pairing ceremony: the host displays a 4-digit PIN (its log/UI), the user
/// types it here. On success the host stores this client's identity and the returned
/// fingerprint is the host's now-VERIFIED identity persist it and pass it as `pinSHA256`
/// to every later connect. Throws `.wrongPIN` when the proof is rejected.
public func pair(
host: String, port: UInt16 = 9777,
identity: ClientIdentity, pin: String, name: String,
timeoutMs: UInt32 = 90_000
) throws -> Data {
var observed = [UInt8](repeating: 0, count: 32)
// The C header types PunktfunkStatus as a bare int32 (C17, no enum import), so the ABI
// functions return Int32 directly compare against the enum constants' rawValue, the
// same bridging the connection methods use (statusOK etc.).
let rc = host.withCString { cs in
identity.certPEM.withCString { cert in
identity.keyPEM.withCString { key in
pin.withCString { p in
name.withCString { n in
punktfunk_pair(cs, port, cert, key, p, n, &observed, timeoutMs)
}
}
}
}
}
switch rc {
case PUNKTFUNK_STATUS_OK.rawValue: return Data(observed)
case PUNKTFUNK_STATUS_CRYPTO.rawValue: throw PunktfunkClientError.wrongPIN
default: throw PunktfunkClientError.status(rc)
}
}
@@ -0,0 +1,87 @@
// Convenience constructors for the wire input events (field semantics match
// punktfunk_core::input::InputEvent; see punktfunk_core.h).
import Foundation
import PunktfunkCore
public extension PunktfunkInputEvent {
private static func make(
_ kind: UInt32, code: UInt32, x: Int32, y: Int32, flags: UInt32 = 0
) -> PunktfunkInputEvent {
PunktfunkInputEvent(kind: UInt8(kind), _pad: (0, 0, 0), code: code, x: x, y: y, flags: flags)
}
static func mouseMove(dx: Int32, dy: Int32) -> PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_MOUSE_MOVE.rawValue, code: 0, x: dx, y: dy)
}
/// Absolute cursor position in client-surface pixels the host places its cursor
/// there (same letterbox mapping and `flags` surface-dims packing as the touch events).
/// Used by the iPad pointer fallback when the scene can't pointer-lock and GCMouse's
/// relative deltas aren't available; the surface dimensions must each fit in 16 bits.
static func mouseMoveAbs(
x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32
) -> PunktfunkInputEvent {
make(
PUNKTFUNK_INPUT_KIND_MOUSE_MOVE_ABS.rawValue, code: 0, x: x, y: y,
flags: ((surfaceWidth & 0xFFFF) << 16) | (surfaceHeight & 0xFFFF))
}
/// GameStream button ids: 1=left 2=middle 3=right 4=X1 5=X2 (host maps to evdev BTN_*).
static func mouseButton(_ button: UInt32, down: Bool) -> PunktfunkInputEvent {
make(
(down ? PUNKTFUNK_INPUT_KIND_MOUSE_BUTTON_DOWN : PUNKTFUNK_INPUT_KIND_MOUSE_BUTTON_UP).rawValue,
code: button, x: 0, y: 0)
}
/// `vk` is a Windows virtual-key code (the host's vk_to_evdev table consumes these).
static func key(_ vk: UInt32, down: Bool) -> PunktfunkInputEvent {
make((down ? PUNKTFUNK_INPUT_KIND_KEY_DOWN : PUNKTFUNK_INPUT_KIND_KEY_UP).rawValue, code: vk, x: 0, y: 0)
}
/// WHEEL_DELTA(120)-scaled; positive = up (vertical) / right (horizontal) the
/// convention Moonlight/SDL use; the host maps onto the ei/wl axes.
static func scroll(_ delta: Int32, horizontal: Bool = false) -> PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_MOUSE_SCROLL.rawValue, code: horizontal ? 1 : 0, x: delta, y: 0)
}
// Gamepad (wire contract in punktfunk_core::input::gamepad): one transition per event,
// `pad` = controller index, accumulated host-side into a virtual Xbox 360 or DualSense
// pad (the session's negotiated `GamepadType`).
/// `button` is a GameStream buttonFlags bit (A=0x1000 B=0x2000 X=0x4000 Y=0x8000,
/// dpad=0x1/2/4/8, start=0x10 back=0x20 LS=0x40 RS=0x80 LB=0x100 RB=0x200 guide=0x400,
/// touchpad click=0x100000 DualSense sessions only, the xpad has no such button).
static func gamepadButton(_ button: UInt32, down: Bool, pad: UInt32 = 0) -> PunktfunkInputEvent {
make(
PUNKTFUNK_INPUT_KIND_GAMEPAD_BUTTON.rawValue,
code: button, x: down ? 1 : 0, y: 0, flags: pad)
}
/// Axis ids: 0=LSX 1=LSY 2=RSX 3=RSY (32768...32767, XInput convention: +y = UP
/// `GCControllerDirectionPad.yAxis` already matches, no flip), 4=LT 5=RT (0...255).
static func gamepadAxis(_ axis: UInt32, value: Int32, pad: UInt32 = 0) -> PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_GAMEPAD_AXIS.rawValue, code: axis, x: value, y: 0, flags: pad)
}
// Touch (host-side: libei ei_touchscreen on the virtual output). `id` distinguishes
// fingers and is reusable after touchUp; coordinates are absolute pixels on the
// client's touch surface, whose size rides in `flags` so the host can rescale
// the surface dimensions must each fit in 16 bits. Built for the iOS variant
// (UITouch these); nothing on macOS emits them yet.
static func touchDown(
id: UInt32, x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32
) -> PunktfunkInputEvent {
make(
PUNKTFUNK_INPUT_KIND_TOUCH_DOWN.rawValue, code: id, x: x, y: y,
flags: ((surfaceWidth & 0xFFFF) << 16) | (surfaceHeight & 0xFFFF))
}
static func touchMove(
id: UInt32, x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32
) -> PunktfunkInputEvent {
make(
PUNKTFUNK_INPUT_KIND_TOUCH_MOVE.rawValue, code: id, x: x, y: y,
flags: ((surfaceWidth & 0xFFFF) << 16) | (surfaceHeight & 0xFFFF))
}
static func touchUp(id: UInt32) -> PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_TOUCH_UP.rawValue, code: id, x: 0, y: 0)
}
}
@@ -57,60 +57,6 @@ public enum PunktfunkClientError: Error {
case status(Int32)
}
/// This client's persistent self-signed identity. Generate ONCE with `generateIdentity()`,
/// store both PEMs (Keychain), present on every connect the certificate's fingerprint is
/// how hosts recognize this client after pairing.
public struct ClientIdentity: Sendable {
public let certPEM: String
public let keyPEM: String
public init(certPEM: String, keyPEM: String) {
self.certPEM = certPEM
self.keyPEM = keyPEM
}
}
/// Generate a fresh client identity (self-signed cert + key, PEM).
public func generateIdentity() throws -> ClientIdentity {
var cert = [CChar](repeating: 0, count: 4096)
var key = [CChar](repeating: 0, count: 4096)
let rc = punktfunk_generate_identity(&cert, UInt(cert.count), &key, UInt(key.count))
guard rc == PUNKTFUNK_STATUS_OK.rawValue else {
throw PunktfunkClientError.status(rc)
}
return ClientIdentity(certPEM: String(cString: cert), keyPEM: String(cString: key))
}
/// Run the PIN pairing ceremony: the host displays a 4-digit PIN (its log/UI), the user
/// types it here. On success the host stores this client's identity and the returned
/// fingerprint is the host's now-VERIFIED identity persist it and pass it as `pinSHA256`
/// to every later connect. Throws `.wrongPIN` when the proof is rejected.
public func pair(
host: String, port: UInt16 = 9777,
identity: ClientIdentity, pin: String, name: String,
timeoutMs: UInt32 = 90_000
) throws -> Data {
var observed = [UInt8](repeating: 0, count: 32)
// The C header types PunktfunkStatus as a bare int32 (C17, no enum import), so the ABI
// functions return Int32 directly compare against the enum constants' rawValue, the
// same bridging the connection methods use (statusOK etc.).
let rc = host.withCString { cs in
identity.certPEM.withCString { cert in
identity.keyPEM.withCString { key in
pin.withCString { p in
name.withCString { n in
punktfunk_pair(cs, port, cert, key, p, n, &observed, timeoutMs)
}
}
}
}
}
switch rc {
case PUNKTFUNK_STATUS_OK.rawValue: return Data(observed)
case PUNKTFUNK_STATUS_CRYPTO.rawValue: throw PunktfunkClientError.wrongPIN
default: throw PunktfunkClientError.status(rc)
}
}
/// `withCString` over an optional nil maps to a NULL C pointer.
func withOptionalCString<R>(_ s: String?, _ body: (UnsafePointer<CChar>?) -> R) -> R {
guard let s else { return body(nil) }
@@ -803,87 +749,3 @@ public final class PunktfunkConnection {
return closeRequested ? nil : handle
}
}
// Convenience constructors for the wire input events (field semantics match
// punktfunk_core::input::InputEvent; see punktfunk_core.h).
public extension PunktfunkInputEvent {
private static func make(
_ kind: UInt32, code: UInt32, x: Int32, y: Int32, flags: UInt32 = 0
) -> PunktfunkInputEvent {
PunktfunkInputEvent(kind: UInt8(kind), _pad: (0, 0, 0), code: code, x: x, y: y, flags: flags)
}
static func mouseMove(dx: Int32, dy: Int32) -> PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_MOUSE_MOVE.rawValue, code: 0, x: dx, y: dy)
}
/// Absolute cursor position in client-surface pixels the host places its cursor
/// there (same letterbox mapping and `flags` surface-dims packing as the touch events).
/// Used by the iPad pointer fallback when the scene can't pointer-lock and GCMouse's
/// relative deltas aren't available; the surface dimensions must each fit in 16 bits.
static func mouseMoveAbs(
x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32
) -> PunktfunkInputEvent {
make(
PUNKTFUNK_INPUT_KIND_MOUSE_MOVE_ABS.rawValue, code: 0, x: x, y: y,
flags: ((surfaceWidth & 0xFFFF) << 16) | (surfaceHeight & 0xFFFF))
}
/// GameStream button ids: 1=left 2=middle 3=right 4=X1 5=X2 (host maps to evdev BTN_*).
static func mouseButton(_ button: UInt32, down: Bool) -> PunktfunkInputEvent {
make(
(down ? PUNKTFUNK_INPUT_KIND_MOUSE_BUTTON_DOWN : PUNKTFUNK_INPUT_KIND_MOUSE_BUTTON_UP).rawValue,
code: button, x: 0, y: 0)
}
/// `vk` is a Windows virtual-key code (the host's vk_to_evdev table consumes these).
static func key(_ vk: UInt32, down: Bool) -> PunktfunkInputEvent {
make((down ? PUNKTFUNK_INPUT_KIND_KEY_DOWN : PUNKTFUNK_INPUT_KIND_KEY_UP).rawValue, code: vk, x: 0, y: 0)
}
/// WHEEL_DELTA(120)-scaled; positive = up (vertical) / right (horizontal) the
/// convention Moonlight/SDL use; the host maps onto the ei/wl axes.
static func scroll(_ delta: Int32, horizontal: Bool = false) -> PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_MOUSE_SCROLL.rawValue, code: horizontal ? 1 : 0, x: delta, y: 0)
}
// Gamepad (wire contract in punktfunk_core::input::gamepad): one transition per event,
// `pad` = controller index, accumulated host-side into a virtual Xbox 360 or DualSense
// pad (the session's negotiated `GamepadType`).
/// `button` is a GameStream buttonFlags bit (A=0x1000 B=0x2000 X=0x4000 Y=0x8000,
/// dpad=0x1/2/4/8, start=0x10 back=0x20 LS=0x40 RS=0x80 LB=0x100 RB=0x200 guide=0x400,
/// touchpad click=0x100000 DualSense sessions only, the xpad has no such button).
static func gamepadButton(_ button: UInt32, down: Bool, pad: UInt32 = 0) -> PunktfunkInputEvent {
make(
PUNKTFUNK_INPUT_KIND_GAMEPAD_BUTTON.rawValue,
code: button, x: down ? 1 : 0, y: 0, flags: pad)
}
/// Axis ids: 0=LSX 1=LSY 2=RSX 3=RSY (32768...32767, XInput convention: +y = UP
/// `GCControllerDirectionPad.yAxis` already matches, no flip), 4=LT 5=RT (0...255).
static func gamepadAxis(_ axis: UInt32, value: Int32, pad: UInt32 = 0) -> PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_GAMEPAD_AXIS.rawValue, code: axis, x: value, y: 0, flags: pad)
}
// Touch (host-side: libei ei_touchscreen on the virtual output). `id` distinguishes
// fingers and is reusable after touchUp; coordinates are absolute pixels on the
// client's touch surface, whose size rides in `flags` so the host can rescale
// the surface dimensions must each fit in 16 bits. Built for the iOS variant
// (UITouch these); nothing on macOS emits them yet.
static func touchDown(
id: UInt32, x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32
) -> PunktfunkInputEvent {
make(
PUNKTFUNK_INPUT_KIND_TOUCH_DOWN.rawValue, code: id, x: x, y: y,
flags: ((surfaceWidth & 0xFFFF) << 16) | (surfaceHeight & 0xFFFF))
}
static func touchMove(
id: UInt32, x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32
) -> PunktfunkInputEvent {
make(
PUNKTFUNK_INPUT_KIND_TOUCH_MOVE.rawValue, code: id, x: x, y: y,
flags: ((surfaceWidth & 0xFFFF) << 16) | (surfaceHeight & 0xFFFF))
}
static func touchUp(id: UInt32) -> PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_TOUCH_UP.rawValue, code: id, x: 0, y: 0)
}
}
@@ -0,0 +1,73 @@
#if DEBUG
import Combine
import GameController
/// Local feedback driver for the Settings Controllers "Test Controller" panel (DEBUG builds
/// only). It drives the SAME CoreHaptics rumble renderer and `DualSenseTriggerEffect` path a
/// live session uses just aimed at the physically-connected controller instead of the
/// hostclient feedback planes so rumble, the adaptive triggers, the lightbar and the player
/// LEDs can be confirmed on-device without a host. Reusing the real renderers is the point:
/// a passing test exercises the exact code a session runs.
@MainActor
public final class ControllerTester: ObservableObject {
private let renderer = RumbleRenderer()
private weak var controller: GCController?
/// The rumble backend now in use "DualSense HID · USB/Bluetooth", "CoreHaptics", or ""
/// for the test panel to display so it's obvious which path a given pad takes.
@Published public private(set) var rumbleBackend = ""
public init() {}
/// Aim the feedback at a controller (nil releases it). Idempotent safe to call on every
/// active-controller change.
public func target(_ c: GCController?) {
guard c !== controller else { return }
controller = c
renderer.retarget(c) { [weak self] note in
Task { @MainActor in self?.rumbleBackend = note }
}
}
/// Drive both motors at 0...1 amplitudes low = left/heavy, high = right/light mapped to
/// the 0...0xFFFF wire range the session carries, through the real `RumbleRenderer`.
public func rumble(low: Float, high: Float) {
func u16(_ v: Float) -> UInt16 { UInt16((min(max(v, 0), 1) * 65535).rounded()) }
renderer.apply(low: u16(low), high: u16(high))
}
public func stopRumble() { renderer.apply(low: 0, high: 0) }
/// Replay an adaptive-trigger effect on a DualSense via the real `DualSenseTriggerEffect`
/// renderer. `right == false` L2, `true` R2. No-op on a non-DualSense pad.
public func applyTrigger(_ effect: DualSenseTriggerEffect, right: Bool) {
guard let ds = controller?.extendedGamepad as? GCDualSenseGamepad else { return }
effect.apply(to: right ? ds.rightTrigger : ds.leftTrigger)
}
public func resetTriggers() {
guard let ds = controller?.extendedGamepad as? GCDualSenseGamepad else { return }
ds.leftTrigger.setModeOff()
ds.rightTrigger.setModeOff()
}
/// Lightbar colour (DualSense / DualShock 4); nil turns it off. No-op without a light.
public func setLight(_ color: GCColor?) {
controller?.light?.color = color ?? GCColor(red: 0, green: 0, blue: 0)
}
/// Player-indicator LEDs (`.index1`...`.index4`, or `.indexUnset` to clear).
public func setPlayerIndex(_ index: GCControllerPlayerIndex) {
controller?.playerIndex = index
}
/// Silence every channel and release the controller call on the panel's disappear.
public func stop() {
resetTriggers()
setPlayerIndex(.indexUnset)
setLight(nil)
renderer.retarget(nil) // async teardown: stops the motors + drops the controller ref
controller = nil
}
}
#endif
@@ -29,64 +29,6 @@ import Combine
import Foundation
import GameController
/// The gamepad wire contract (mirrors `punktfunk_core::input::gamepad`).
public enum GamepadWire {
public static let dpadUp: UInt32 = 0x0001
public static let dpadDown: UInt32 = 0x0002
public static let dpadLeft: UInt32 = 0x0004
public static let dpadRight: UInt32 = 0x0008
public static let start: UInt32 = 0x0010
public static let back: UInt32 = 0x0020
public static let leftStickClick: UInt32 = 0x0040
public static let rightStickClick: UInt32 = 0x0080
public static let leftShoulder: UInt32 = 0x0100
public static let rightShoulder: UInt32 = 0x0200
public static let guide: UInt32 = 0x0400
public static let a: UInt32 = 0x1000
public static let b: UInt32 = 0x2000
public static let x: UInt32 = 0x4000
public static let y: UInt32 = 0x8000
/// DualSense touchpad click (Moonlight's extended-button bit position).
public static let touchpadClick: UInt32 = 0x10_0000
public static let allButtons: [UInt32] = [
dpadUp, dpadDown, dpadLeft, dpadRight, start, back,
leftStickClick, rightStickClick, leftShoulder, rightShoulder, guide,
a, b, x, y, touchpadClick,
]
public static let axisLSX: UInt32 = 0
public static let axisLSY: UInt32 = 1
public static let axisRSX: UInt32 = 2
public static let axisRSY: UInt32 = 3
public static let axisLT: UInt32 = 4
public static let axisRT: UInt32 = 5
/// Raw DualSense gyro units per rad/s: hid-playstation's calibration over the host's
/// fixed blob resolves to 20 LSB per deg/s.
public static let gyroLSBPerRadS: Float = 20 * 180 / .pi
/// Raw DualSense accelerometer units per g (same derivation).
public static let accelLSBPerG: Float = 10_000
/// GC touchpad coordinates (±1, +y up) wire (0...65535, origin top-left, +y down).
public static func touchpad(x: Float, y: Float) -> (x: UInt16, y: UInt16) {
let wx = ((x.clamped(to: -1...1) + 1) / 2 * 65535).rounded()
let wy = ((1 - y.clamped(to: -1...1)) / 2 * 65535).rounded()
return (UInt16(wx), UInt16(wy))
}
/// Scale + clamp one motion component into the raw signed-16 sensor domain.
public static func motionRaw(_ value: Float, scale: Float) -> Int16 {
Int16((value * scale).rounded().clamped(to: Float(Int16.min)...Float(Int16.max)))
}
}
extension Float {
fileprivate func clamped(to range: ClosedRange<Float>) -> Float {
Swift.min(Swift.max(self, range.lowerBound), range.upperBound)
}
}
@MainActor
public final class GamepadCapture {
private let connection: PunktfunkConnection
@@ -0,0 +1,195 @@
// Hostclient gamepad feedback rendering: one drain thread polls the rumble (0xCA) and
// HID-output (0xCD) planes and replays them on the active physical controller
//
// rumble CHHapticEngine players (per-handle localities when the pad has them,
// one combined engine otherwise),
// lightbar GCDeviceLight,
// player LEDs GCController.playerIndex (the DS bit patterns map to player 14),
// trigger FX DualSenseTriggerEffect.parse GCDualSenseAdaptiveTrigger.
//
// Only pad 0 is rendered (exactly one controller is forwarded). HID-output traffic exists
// only on PlayStation-pad sessions (a DualSense, or a DualShock 4 = lightbar only) the
// drain always polls both planes with short timeouts and never spins, so an Xbox session
// just renders rumble. GameController profile mutation
// happens on main; CHHapticEngine work on its own serial queue; the drain thread itself
// touches neither. When GamepadManager switches the active controller mid-session, the
// old pad is reset (triggers off, player index unset) and the last known feedback state
// is replayed onto the new one.
import Combine
import Foundation
import GameController
public final class GamepadFeedback {
private let connection: PunktfunkConnection
private let flag = StopFlag()
private let drainDone = DispatchSemaphore(value: 0)
private var drainStarted = false
private let rumble = RumbleRenderer()
private var activeSub: AnyCancellable?
// Last applied feedback (main-actor) replayed when the active controller changes.
@MainActor private var target: GCController?
@MainActor private var lastLight: (r: UInt8, g: UInt8, b: UInt8)?
@MainActor private var lastPlayerBits: UInt8?
@MainActor private var lastTrigger: [DualSenseTriggerEffect?] = [nil, nil]
public init(connection: PunktfunkConnection, manager: GamepadManager) {
self.connection = connection
// Capture self weakly in the hop too, so the inner sink's weak capture isn't shadowing
// an implicit strong one and the subscription (stored on self) never retain-cycles.
Task { @MainActor [weak self] in
guard let self else { return }
self.activeSub = manager.$active.sink { [weak self] dc in
MainActor.assumeIsolated { self?.retarget(dc?.controller) }
}
}
}
/// Safety net: the drain thread captures `connection` strongly and only `self` weakly, so if
/// this is dropped without `stop()` (an abrupt teardown) the thread would poll forever and
/// leak the connection signal it to exit. (`stop()` is the normal path and also joins it.)
deinit { flag.stop() }
/// Map the DualSense player-LED bit patterns (5 LEDs, hid-playstation's player
/// conventions) onto GCControllerPlayerIndex. Unknown patterns fall back to the lit
/// count, clamped to the four indices GC offers.
public static func playerIndex(forBits bits: UInt8) -> GCControllerPlayerIndex {
switch bits & 0x1F {
case 0: return .indexUnset
case 0b00100: return .index1
case 0b01010: return .index2
case 0b10101: return .index3
case 0b11011: return .index4
default:
let lit = (bits & 0x1F).nonzeroBitCount
return GCControllerPlayerIndex(rawValue: min(lit, 4) - 1) ?? .index1
}
}
public func start() {
guard !drainStarted else { return }
drainStarted = true
// Hidout traffic (lightbar / player LEDs / triggers) only exists on a PlayStation-pad
// session a DualSense or a DualShock 4 (lightbar only). Block briefly on it there and
// let rumble own the wait elsewhere; on an Xbox session it stays nonblocking.
let thread = Thread { [connection, flag, drainDone, weak self] in
while !flag.isStopped {
do {
// Poll the feedback planes NON-BLOCKING. A blocking poll (timeoutMs > 0) holds
// the connection's shared feedback lock for its whole wait; the video pump drains
// HDR mastering metadata (nextHdrMeta) on the SAME lock every frame, so a blocking
// poll here starved it and throttled HDR to ~1 fps (SDR, which never drains HDR
// meta, was unaffected). Pacing with a short sleep OUTSIDE the lock (below) keeps
// rumble/HID latency low while leaving the lock free between polls.
if let r = try connection.nextRumble(timeoutMs: 0), r.pad == 0 {
self?.rumble.apply(low: r.low, high: r.high)
}
// Drain a BOUNDED burst of hidout events so sustained 0xCD traffic (a game writing
// per-frame LED/trigger reports) can't spin here or block stop() past one cycle.
var burst = 0
while burst < 64, !flag.isStopped,
let ev = try connection.nextHidOutput(timeoutMs: 0) {
self?.render(ev)
burst += 1
}
} catch {
break // .closed (or fatal) the session is over
}
// ~8 ms poll cadence (125 Hz), slept OUTSIDE the feedback lock low rumble/HID
// latency without holding the lock the HDR-meta drain needs.
if !flag.isStopped { Thread.sleep(forTimeInterval: 0.008) }
}
drainDone.signal()
}
thread.name = "punktfunk-feedback"
thread.qualityOfService = .userInteractive
thread.start()
}
/// Stop the drain and silence the motors. Blocks until the drain thread exits ( one
/// poll cycle) call off the main actor, before `connection.close()`.
public func stop() {
flag.stop()
if drainStarted {
drainDone.wait()
drainStarted = false
}
rumble.stop()
// Drop the retarget subscription and the dead session's cached feedback a
// controller change after teardown must not replay this session's triggers/LEDs.
Task { @MainActor in
self.activeSub = nil
self.lastLight = nil
self.lastPlayerBits = nil
self.lastTrigger = [nil, nil]
self.reset(self.target)
self.target = nil
}
}
private func render(_ ev: PunktfunkConnection.HidOutputEvent) {
DispatchQueue.main.async {
MainActor.assumeIsolated { self.apply(ev) }
}
}
@MainActor
private func apply(_ ev: PunktfunkConnection.HidOutputEvent) {
switch ev {
case let .led(pad, r, g, b):
guard pad == 0 else { return }
lastLight = (r, g, b)
target?.light?.color = GCColor(
red: Float(r) / 255, green: Float(g) / 255, blue: Float(b) / 255)
case let .playerLEDs(pad, bits):
guard pad == 0 else { return }
lastPlayerBits = bits
target?.playerIndex = Self.playerIndex(forBits: bits)
case let .triggerEffect(pad, which, effect):
guard pad == 0, which < 2 else { return }
let parsed = DualSenseTriggerEffect.parse(effect)
lastTrigger[Int(which)] = parsed
if let trigger = adaptiveTrigger(which) {
parsed.apply(to: trigger)
}
}
}
@MainActor
private func retarget(_ controller: GCController?) {
guard controller !== target else { return }
reset(target)
target = controller
rumble.retarget(controller)
// Replay the session's feedback state so a swapped-in controller looks the same.
if let (r, g, b) = lastLight {
controller?.light?.color = GCColor(
red: Float(r) / 255, green: Float(g) / 255, blue: Float(b) / 255)
}
if let bits = lastPlayerBits {
controller?.playerIndex = Self.playerIndex(forBits: bits)
}
for which in 0..<2 {
if let effect = lastTrigger[which], let trigger = adaptiveTrigger(UInt8(which)) {
effect.apply(to: trigger)
}
}
}
@MainActor
private func reset(_ controller: GCController?) {
guard let c = controller else { return }
c.playerIndex = .indexUnset
if let ds = c.extendedGamepad as? GCDualSenseGamepad {
ds.leftTrigger.setModeOff()
ds.rightTrigger.setModeOff()
}
}
@MainActor
private func adaptiveTrigger(_ which: UInt8) -> GCDualSenseAdaptiveTrigger? {
guard let ds = target?.extendedGamepad as? GCDualSenseGamepad else { return nil }
return which == 0 ? ds.leftTrigger : ds.rightTrigger
}
}
@@ -14,8 +14,9 @@
// off or race over.
//
// The button set mirrors a console launcher: A confirms, B backs out, Y is a screen's secondary
// action, and the shoulders (L1/R1) are optional fast "jump" steps. Directional moves auto-repeat
// on a held stick/dpad after an initial delay; every button is edge-triggered (fires once per press).
// action, X a tertiary one, and the shoulders (L1/R1) are optional fast "jump" steps. Directional
// moves auto-repeat on a held stick/dpad after an initial delay; every button is edge-triggered
// (fires once per press).
import Foundation
import GameController
@@ -29,10 +30,18 @@ public final class GamepadMenuInput {
private let manager: GamepadManager
private var pollTimer: Timer?
private var isActive = false
/// Seed the pressed-state trackers from the LIVE controller on the first poll after a
/// (re)start, firing nothing. Screens hand the controller off (a keyboard closes, a cover
/// dismisses) while the user is still holding the very button that triggered the handoff
/// without this, the next screen's first poll would read that held button as a fresh edge
/// and act on the same press twice (e.g. the B that closed the keyboard also backing out
/// of the screen underneath).
private var needsSnapshot = false
private var currentDirection: Direction?
private var repeatTimer: Timer?
private var wasConfirmPressed = false
private var wasSecondaryPressed = false
private var wasTertiaryPressed = false
private var wasBackPressed = false
private var wasLeftShoulderPressed = false
private var wasRightShoulderPressed = false
@@ -44,6 +53,8 @@ public final class GamepadMenuInput {
public var onConfirm: (() -> Void)?
/// Button Y (or equivalent secondary action, e.g. "open library") edge-triggered.
public var onSecondary: (() -> Void)?
/// Button X (or equivalent tertiary action, e.g. "settings" / "delete") edge-triggered.
public var onTertiary: (() -> Void)?
/// Button B (or equivalent back/dismiss) edge-triggered.
public var onBack: (() -> Void)?
/// Shoulder buttons (L1 `false` / R1 `true`) edge-triggered fast-jump steps, optional per
@@ -63,6 +74,7 @@ public final class GamepadMenuInput {
public func start() {
guard !isActive else { return }
isActive = true
needsSnapshot = true
let timer = Timer(timeInterval: pollInterval, repeats: true) { [weak self] _ in
Task { @MainActor in self?.poll() }
}
@@ -79,6 +91,7 @@ public final class GamepadMenuInput {
currentDirection = nil
wasConfirmPressed = false
wasSecondaryPressed = false
wasTertiaryPressed = false
wasBackPressed = false
wasLeftShoulderPressed = false
wasRightShoulderPressed = false
@@ -89,8 +102,24 @@ public final class GamepadMenuInput {
private func poll() {
guard isActive, let gamepad = manager.active?.controller.extendedGamepad else { return }
if needsSnapshot {
// Adopt whatever is held right now without firing (see `needsSnapshot`): a button
// must be RELEASED after a handoff before it can act here, and a held direction only
// keeps moving once it changes or re-engages.
needsSnapshot = false
wasConfirmPressed = gamepad.buttonA.isPressed
wasSecondaryPressed = gamepad.buttonY.isPressed
wasTertiaryPressed = gamepad.buttonX.isPressed
wasBackPressed = gamepad.buttonB.isPressed
wasLeftShoulderPressed = gamepad.leftShoulder.isPressed
wasRightShoulderPressed = gamepad.rightShoulder.isPressed
currentDirection = directionFrom(gamepad)
return
}
edge(gamepad.buttonA.isPressed, &wasConfirmPressed) { onConfirm?() }
edge(gamepad.buttonY.isPressed, &wasSecondaryPressed) { onSecondary?() }
edge(gamepad.buttonX.isPressed, &wasTertiaryPressed) { onTertiary?() }
edge(gamepad.buttonB.isPressed, &wasBackPressed) { onBack?() }
edge(gamepad.leftShoulder.isPressed, &wasLeftShoulderPressed) { onShoulder?(false) }
edge(gamepad.rightShoulder.isPressed, &wasRightShoulderPressed) { onShoulder?(true) }
@@ -0,0 +1,26 @@
// Whether the iOS/iPadOS/macOS UI should be in its controller-friendly mode (the console-style
// host launcher, gamepad settings, and the coverflow library browser instead of the touch/desktop
// layouts). A pure function, not a singleton: the reactivity comes from callers already observing
// `GamepadManager.shared` and the `DefaultsKey.gamepadUIEnabled` @AppStorage themselves (the same
// local-read pattern SettingsView already uses for GamepadManager), so this stays the single place
// the two combine without adding a second ObservableObject or an environment key nobody else needs.
import Foundation
public enum GamepadUIEnvironment {
/// `enabledSetting` is the user's Settings toggle (`DefaultsKey.gamepadUIEnabled`);
/// `gamepadConnected` is `GamepadManager.shared.active != nil` active only once a usable
/// controller is actually attached (a non-extended-profile device leaves `active` nil, which
/// keeps the touch UI). A `Bool` rather than the `DiscoveredController` itself: this function's
/// whole job is the AND, so there's nothing else to inspect, and it keeps the helper testable
/// without a real `GCController` (which XCTest can't construct).
public static func isActive(gamepadConnected: Bool, enabledSetting: Bool) -> Bool {
enabledSetting && (gamepadConnected || forced)
}
/// Dev-only escape hatch (like ContentView's `PUNKTFUNK_AUTOCONNECT`): pretend a controller is
/// attached so the gamepad UI can be exercised/screenshotted without physical hardware
/// essential on a headless CI Mac and for `swift run` UI work. Never set in production.
private static let forced =
ProcessInfo.processInfo.environment["PUNKTFUNK_FORCE_GAMEPAD_UI"] == "1"
}
@@ -0,0 +1,62 @@
// The gamepad wire contract shared by capture (GamepadCapture), feedback (GamepadFeedback),
// and the tests button bits, axis ids, and the touchpad/motion unit conversions.
import Foundation
/// The gamepad wire contract (mirrors `punktfunk_core::input::gamepad`).
public enum GamepadWire {
public static let dpadUp: UInt32 = 0x0001
public static let dpadDown: UInt32 = 0x0002
public static let dpadLeft: UInt32 = 0x0004
public static let dpadRight: UInt32 = 0x0008
public static let start: UInt32 = 0x0010
public static let back: UInt32 = 0x0020
public static let leftStickClick: UInt32 = 0x0040
public static let rightStickClick: UInt32 = 0x0080
public static let leftShoulder: UInt32 = 0x0100
public static let rightShoulder: UInt32 = 0x0200
public static let guide: UInt32 = 0x0400
public static let a: UInt32 = 0x1000
public static let b: UInt32 = 0x2000
public static let x: UInt32 = 0x4000
public static let y: UInt32 = 0x8000
/// DualSense touchpad click (Moonlight's extended-button bit position).
public static let touchpadClick: UInt32 = 0x10_0000
public static let allButtons: [UInt32] = [
dpadUp, dpadDown, dpadLeft, dpadRight, start, back,
leftStickClick, rightStickClick, leftShoulder, rightShoulder, guide,
a, b, x, y, touchpadClick,
]
public static let axisLSX: UInt32 = 0
public static let axisLSY: UInt32 = 1
public static let axisRSX: UInt32 = 2
public static let axisRSY: UInt32 = 3
public static let axisLT: UInt32 = 4
public static let axisRT: UInt32 = 5
/// Raw DualSense gyro units per rad/s: hid-playstation's calibration over the host's
/// fixed blob resolves to 20 LSB per deg/s.
public static let gyroLSBPerRadS: Float = 20 * 180 / .pi
/// Raw DualSense accelerometer units per g (same derivation).
public static let accelLSBPerG: Float = 10_000
/// GC touchpad coordinates (±1, +y up) wire (0...65535, origin top-left, +y down).
public static func touchpad(x: Float, y: Float) -> (x: UInt16, y: UInt16) {
let wx = ((x.clamped(to: -1...1) + 1) / 2 * 65535).rounded()
let wy = ((1 - y.clamped(to: -1...1)) / 2 * 65535).rounded()
return (UInt16(wx), UInt16(wy))
}
/// Scale + clamp one motion component into the raw signed-16 sensor domain.
public static func motionRaw(_ value: Float, scale: Float) -> Int16 {
Int16((value * scale).rounded().clamped(to: Float(Int16.min)...Float(Int16.max)))
}
}
extension Float {
fileprivate func clamped(to range: ClosedRange<Float>) -> Float {
Swift.min(Swift.max(self, range.lowerBound), range.upperBound)
}
}
@@ -1,22 +1,3 @@
// Hostclient gamepad feedback rendering: one drain thread polls the rumble (0xCA) and
// HID-output (0xCD) planes and replays them on the active physical controller
//
// rumble CHHapticEngine players (per-handle localities when the pad has them,
// one combined engine otherwise),
// lightbar GCDeviceLight,
// player LEDs GCController.playerIndex (the DS bit patterns map to player 14),
// trigger FX DualSenseTriggerEffect.parse GCDualSenseAdaptiveTrigger.
//
// Only pad 0 is rendered (exactly one controller is forwarded). HID-output traffic exists
// only on PlayStation-pad sessions (a DualSense, or a DualShock 4 = lightbar only) the
// drain always polls both planes with short timeouts and never spins, so an Xbox session
// just renders rumble. GameController profile mutation
// happens on main; CHHapticEngine work on its own serial queue; the drain thread itself
// touches neither. When GamepadManager switches the active controller mid-session, the
// old pad is reset (triggers off, player index unset) and the last known feedback state
// is replayed onto the new one.
import Combine
import CoreHaptics
import Foundation
import GameController
@@ -24,21 +5,6 @@ import os
private let log = Logger(subsystem: "io.unom.punktfunk", category: "gamepad")
private final class FeedbackStopFlag: @unchecked Sendable {
private let lock = NSLock()
private var stopped = false
var isStopped: Bool {
lock.lock()
defer { lock.unlock() }
return stopped
}
func stop() {
lock.lock()
stopped = true
lock.unlock()
}
}
/// Rumble CoreHaptics, isolated on one serial queue (CHHapticEngine is not main-bound,
/// but it isn't a free-for-all either). Engines are created lazily on the first nonzero
/// amplitude and torn down on retarget; players run only while their motor is on, so an
@@ -47,7 +13,7 @@ private final class FeedbackStopFlag: @unchecked Sendable {
///
/// `@unchecked Sendable` is sound because every property (`controller`/`low`/`high`/`broken`) is
/// read and written only inside `queue` closures the serial queue is the synchronization.
private final class RumbleRenderer: @unchecked Sendable {
final class RumbleRenderer: @unchecked Sendable {
private let queue = DispatchQueue(label: "io.unom.punktfunk.haptics", qos: .userInteractive)
/// One actuator's started engine plus the player currently driving it (nil = idle). The
@@ -316,248 +282,3 @@ private final class RumbleRenderer: @unchecked Sendable {
return c == nil ? "" : "CoreHaptics"
}
}
public final class GamepadFeedback {
private let connection: PunktfunkConnection
private let flag = FeedbackStopFlag()
private let drainDone = DispatchSemaphore(value: 0)
private var drainStarted = false
private let rumble = RumbleRenderer()
private var activeSub: AnyCancellable?
// Last applied feedback (main-actor) replayed when the active controller changes.
@MainActor private var target: GCController?
@MainActor private var lastLight: (r: UInt8, g: UInt8, b: UInt8)?
@MainActor private var lastPlayerBits: UInt8?
@MainActor private var lastTrigger: [DualSenseTriggerEffect?] = [nil, nil]
public init(connection: PunktfunkConnection, manager: GamepadManager) {
self.connection = connection
// Capture self weakly in the hop too, so the inner sink's weak capture isn't shadowing
// an implicit strong one and the subscription (stored on self) never retain-cycles.
Task { @MainActor [weak self] in
guard let self else { return }
self.activeSub = manager.$active.sink { [weak self] dc in
MainActor.assumeIsolated { self?.retarget(dc?.controller) }
}
}
}
/// Safety net: the drain thread captures `connection` strongly and only `self` weakly, so if
/// this is dropped without `stop()` (an abrupt teardown) the thread would poll forever and
/// leak the connection signal it to exit. (`stop()` is the normal path and also joins it.)
deinit { flag.stop() }
/// Map the DualSense player-LED bit patterns (5 LEDs, hid-playstation's player
/// conventions) onto GCControllerPlayerIndex. Unknown patterns fall back to the lit
/// count, clamped to the four indices GC offers.
public static func playerIndex(forBits bits: UInt8) -> GCControllerPlayerIndex {
switch bits & 0x1F {
case 0: return .indexUnset
case 0b00100: return .index1
case 0b01010: return .index2
case 0b10101: return .index3
case 0b11011: return .index4
default:
let lit = (bits & 0x1F).nonzeroBitCount
return GCControllerPlayerIndex(rawValue: min(lit, 4) - 1) ?? .index1
}
}
public func start() {
guard !drainStarted else { return }
drainStarted = true
// Hidout traffic (lightbar / player LEDs / triggers) only exists on a PlayStation-pad
// session a DualSense or a DualShock 4 (lightbar only). Block briefly on it there and
// let rumble own the wait elsewhere; on an Xbox session it stays nonblocking.
let thread = Thread { [connection, flag, drainDone, weak self] in
while !flag.isStopped {
do {
// Poll the feedback planes NON-BLOCKING. A blocking poll (timeoutMs > 0) holds
// the connection's shared feedback lock for its whole wait; the video pump drains
// HDR mastering metadata (nextHdrMeta) on the SAME lock every frame, so a blocking
// poll here starved it and throttled HDR to ~1 fps (SDR, which never drains HDR
// meta, was unaffected). Pacing with a short sleep OUTSIDE the lock (below) keeps
// rumble/HID latency low while leaving the lock free between polls.
if let r = try connection.nextRumble(timeoutMs: 0), r.pad == 0 {
self?.rumble.apply(low: r.low, high: r.high)
}
// Drain a BOUNDED burst of hidout events so sustained 0xCD traffic (a game writing
// per-frame LED/trigger reports) can't spin here or block stop() past one cycle.
var burst = 0
while burst < 64, !flag.isStopped,
let ev = try connection.nextHidOutput(timeoutMs: 0) {
self?.render(ev)
burst += 1
}
} catch {
break // .closed (or fatal) the session is over
}
// ~8 ms poll cadence (125 Hz), slept OUTSIDE the feedback lock low rumble/HID
// latency without holding the lock the HDR-meta drain needs.
if !flag.isStopped { Thread.sleep(forTimeInterval: 0.008) }
}
drainDone.signal()
}
thread.name = "punktfunk-feedback"
thread.qualityOfService = .userInteractive
thread.start()
}
/// Stop the drain and silence the motors. Blocks until the drain thread exits ( one
/// poll cycle) call off the main actor, before `connection.close()`.
public func stop() {
flag.stop()
if drainStarted {
drainDone.wait()
drainStarted = false
}
rumble.stop()
// Drop the retarget subscription and the dead session's cached feedback a
// controller change after teardown must not replay this session's triggers/LEDs.
Task { @MainActor in
self.activeSub = nil
self.lastLight = nil
self.lastPlayerBits = nil
self.lastTrigger = [nil, nil]
self.reset(self.target)
self.target = nil
}
}
private func render(_ ev: PunktfunkConnection.HidOutputEvent) {
DispatchQueue.main.async {
MainActor.assumeIsolated { self.apply(ev) }
}
}
@MainActor
private func apply(_ ev: PunktfunkConnection.HidOutputEvent) {
switch ev {
case let .led(pad, r, g, b):
guard pad == 0 else { return }
lastLight = (r, g, b)
target?.light?.color = GCColor(
red: Float(r) / 255, green: Float(g) / 255, blue: Float(b) / 255)
case let .playerLEDs(pad, bits):
guard pad == 0 else { return }
lastPlayerBits = bits
target?.playerIndex = Self.playerIndex(forBits: bits)
case let .triggerEffect(pad, which, effect):
guard pad == 0, which < 2 else { return }
let parsed = DualSenseTriggerEffect.parse(effect)
lastTrigger[Int(which)] = parsed
if let trigger = adaptiveTrigger(which) {
parsed.apply(to: trigger)
}
}
}
@MainActor
private func retarget(_ controller: GCController?) {
guard controller !== target else { return }
reset(target)
target = controller
rumble.retarget(controller)
// Replay the session's feedback state so a swapped-in controller looks the same.
if let (r, g, b) = lastLight {
controller?.light?.color = GCColor(
red: Float(r) / 255, green: Float(g) / 255, blue: Float(b) / 255)
}
if let bits = lastPlayerBits {
controller?.playerIndex = Self.playerIndex(forBits: bits)
}
for which in 0..<2 {
if let effect = lastTrigger[which], let trigger = adaptiveTrigger(UInt8(which)) {
effect.apply(to: trigger)
}
}
}
@MainActor
private func reset(_ controller: GCController?) {
guard let c = controller else { return }
c.playerIndex = .indexUnset
if let ds = c.extendedGamepad as? GCDualSenseGamepad {
ds.leftTrigger.setModeOff()
ds.rightTrigger.setModeOff()
}
}
@MainActor
private func adaptiveTrigger(_ which: UInt8) -> GCDualSenseAdaptiveTrigger? {
guard let ds = target?.extendedGamepad as? GCDualSenseGamepad else { return nil }
return which == 0 ? ds.leftTrigger : ds.rightTrigger
}
}
#if DEBUG
/// Local feedback driver for the Settings Controllers "Test Controller" panel (DEBUG builds
/// only). It drives the SAME CoreHaptics rumble renderer and `DualSenseTriggerEffect` path a
/// live session uses just aimed at the physically-connected controller instead of the
/// hostclient feedback planes so rumble, the adaptive triggers, the lightbar and the player
/// LEDs can be confirmed on-device without a host. Reusing the real renderers is the point:
/// a passing test exercises the exact code a session runs.
@MainActor
public final class ControllerTester: ObservableObject {
private let renderer = RumbleRenderer()
private weak var controller: GCController?
/// The rumble backend now in use "DualSense HID · USB/Bluetooth", "CoreHaptics", or ""
/// for the test panel to display so it's obvious which path a given pad takes.
@Published public private(set) var rumbleBackend = ""
public init() {}
/// Aim the feedback at a controller (nil releases it). Idempotent safe to call on every
/// active-controller change.
public func target(_ c: GCController?) {
guard c !== controller else { return }
controller = c
renderer.retarget(c) { [weak self] note in
Task { @MainActor in self?.rumbleBackend = note }
}
}
/// Drive both motors at 0...1 amplitudes low = left/heavy, high = right/light mapped to
/// the 0...0xFFFF wire range the session carries, through the real `RumbleRenderer`.
public func rumble(low: Float, high: Float) {
func u16(_ v: Float) -> UInt16 { UInt16((min(max(v, 0), 1) * 65535).rounded()) }
renderer.apply(low: u16(low), high: u16(high))
}
public func stopRumble() { renderer.apply(low: 0, high: 0) }
/// Replay an adaptive-trigger effect on a DualSense via the real `DualSenseTriggerEffect`
/// renderer. `right == false` L2, `true` R2. No-op on a non-DualSense pad.
public func applyTrigger(_ effect: DualSenseTriggerEffect, right: Bool) {
guard let ds = controller?.extendedGamepad as? GCDualSenseGamepad else { return }
effect.apply(to: right ? ds.rightTrigger : ds.leftTrigger)
}
public func resetTriggers() {
guard let ds = controller?.extendedGamepad as? GCDualSenseGamepad else { return }
ds.leftTrigger.setModeOff()
ds.rightTrigger.setModeOff()
}
/// Lightbar colour (DualSense / DualShock 4); nil turns it off. No-op without a light.
public func setLight(_ color: GCColor?) {
controller?.light?.color = color ?? GCColor(red: 0, green: 0, blue: 0)
}
/// Player-indicator LEDs (`.index1`...`.index4`, or `.indexUnset` to clear).
public func setPlayerIndex(_ index: GCControllerPlayerIndex) {
controller?.playerIndex = index
}
/// Silence every channel and release the controller call on the panel's disappear.
public func stop() {
resetTriggers()
setPlayerIndex(.indexUnset)
setLight(nil)
renderer.retarget(nil) // async teardown: stops the motors + drops the controller ref
controller = nil
}
}
#endif
@@ -1,20 +0,0 @@
// Whether the iOS/iPadOS UI should be in its controller-friendly mode (larger focus targets on
// the host grid, the coverflow library browser instead of the plain grid). A pure function, not a
// singleton: the reactivity comes from callers already observing `GamepadManager.shared` and the
// `DefaultsKey.gamepadUIEnabled` @AppStorage themselves (the same local-read pattern SettingsView
// already uses for GamepadManager), so this stays the single place the two combine without adding
// a second ObservableObject or an environment key nobody else needs.
import Foundation
public enum GamepadUIEnvironment {
/// `enabledSetting` is the user's Settings toggle (`DefaultsKey.gamepadUIEnabled`);
/// `gamepadConnected` is `GamepadManager.shared.active != nil` active only once a usable
/// controller is actually attached (a non-extended-profile device leaves `active` nil, which
/// keeps the touch UI). A `Bool` rather than the `DiscoveredController` itself: this function's
/// whole job is the AND, so there's nothing else to inspect, and it keeps the helper testable
/// without a real `GCController` (which XCTest can't construct).
public static func isActive(gamepadConnected: Bool, enabledSetting: Bool) -> Bool {
enabledSetting && gamepadConnected
}
}
@@ -571,102 +571,4 @@ public final class InputCapture {
}
#endif
}
/// HID usage (GCKeyCode raw) Windows VK (the host maps VK evdev; every VK emitted
/// here exists in punktfunk-host/src/inject.rs::vk_to_evdev extend the two together).
static let hidToVK: [Int: UInt32] = {
var m: [Int: UInt32] = [:]
// az: HID 0x04..0x1D VK 'A'..'Z'.
for i in 0..<26 { m[0x04 + i] = UInt32(0x41 + i) }
// 19: HID 0x1E..0x26 VK '1'..'9'; then 0: HID 0x27 VK '0' (set separately
// the '0' key sits AFTER '9' in HID but its VK 0x30 sits BEFORE '1' (0x31)).
for i in 0..<9 { m[0x1E + i] = UInt32(0x31 + i) }
m[0x27] = 0x30
m[0x28] = 0x0D // return
m[0x29] = 0x1B // escape
m[0x2A] = 0x08 // backspace
m[0x2B] = 0x09 // tab
m[0x2C] = 0x20 // space
m[0x2D] = 0xBD; m[0x2E] = 0xBB // - =
m[0x2F] = 0xDB; m[0x30] = 0xDD; m[0x31] = 0xDC // [ ] backslash
m[0x33] = 0xBA; m[0x34] = 0xDE; m[0x35] = 0xC0 // ; ' `
m[0x36] = 0xBC; m[0x37] = 0xBE; m[0x38] = 0xBF // , . /
m[0x39] = 0x14 // caps lock
// F1..F12: HID 0x3A..0x45 VK 0x70..0x7B.
for i in 0..<12 { m[0x3A + i] = UInt32(0x70 + i) }
m[0x46] = 0x2C; m[0x47] = 0x91; m[0x48] = 0x13 // printscreen scrolllock pause
m[0x4F] = 0x27; m[0x50] = 0x25; m[0x51] = 0x28; m[0x52] = 0x26 // arrows R L D U
m[0x49] = 0x2D; m[0x4A] = 0x24; m[0x4B] = 0x21 // insert home pageup
m[0x4C] = 0x2E; m[0x4D] = 0x23; m[0x4E] = 0x22 // delete end pagedown
// Keypad: NumLock, / * - +, Enter, 1..9, 0, decimal. KP Enter goes as
// VK_SEPARATOR (0x6C) this host maps it to KEY_KPENTER (Windows itself would
// send VK_RETURN+extended, which vk_to_evdev can't distinguish).
m[0x53] = 0x90
m[0x54] = 0x6F; m[0x55] = 0x6A; m[0x56] = 0x6D; m[0x57] = 0x6B
m[0x58] = 0x6C
for i in 0..<9 { m[0x59 + i] = UInt32(0x61 + i) }
m[0x62] = 0x60; m[0x63] = 0x6E
m[0x64] = 0xE2 // ISO 102nd key (<> next to left shift on ISO layouts)
m[0x65] = 0x5D // menu/application
m[0xE0] = 0xA2; m[0xE1] = 0xA0; m[0xE2] = 0xA4; m[0xE3] = 0x5B // Lctrl Lshift Lalt Lcmd
m[0xE4] = 0xA3; m[0xE5] = 0xA1; m[0xE6] = 0xA5; m[0xE7] = 0x5C // Rctrl Rshift Ralt Rcmd
return m
}()
#if os(macOS)
/// NSEvent.keyCode (Carbon virtual keycode, kVK_*) Windows VK. The macOS NSEvent key
/// path is keyed by keyCode (a layout-independent hardware position), NOT by HID usage,
/// so it needs its own table but it emits the EXACT SAME Windows VK integers `hidToVK`
/// already produces for each physical key (A0x41, Return0x0D, KeypadEnter0x6C, ), so
/// the host's vk_to_evdev (inject.rs) accepts both with zero change. Modifier keys come
/// via flagsChanged (handleFlagsChanged), not keyDown, so they're absent here. Keys with
/// no host evdev arm (F13F20, KeypadEquals, the Fn key) are omitted nil swallowed.
static let keyCodeToVK: [UInt16: UInt32] = {
var m: [UInt16: UInt32] = [:]
// Letters kVK_ANSI_A..Z (scattered keycodes) VK 'A'..'Z'.
m[0x00] = 0x41; m[0x01] = 0x53; m[0x02] = 0x44; m[0x03] = 0x46 // A S D F
m[0x04] = 0x48; m[0x05] = 0x47; m[0x06] = 0x5A; m[0x07] = 0x58 // H G Z X
m[0x08] = 0x43; m[0x09] = 0x56; m[0x0B] = 0x42; m[0x0C] = 0x51 // C V B Q
m[0x0D] = 0x57; m[0x0E] = 0x45; m[0x0F] = 0x52; m[0x10] = 0x59 // W E R Y
m[0x11] = 0x54; m[0x1F] = 0x4F; m[0x20] = 0x55; m[0x22] = 0x49 // T O U I
m[0x23] = 0x50; m[0x25] = 0x4C; m[0x26] = 0x4A; m[0x28] = 0x4B // P L J K
m[0x2D] = 0x4E; m[0x2E] = 0x4D // N M
// Digit row kVK_ANSI_1..0 (scattered) VK '1'..'9','0'.
m[0x12] = 0x31; m[0x13] = 0x32; m[0x14] = 0x33; m[0x15] = 0x34 // 1 2 3 4
m[0x16] = 0x36; m[0x17] = 0x35; m[0x19] = 0x39; m[0x1A] = 0x37 // 6 5 9 7
m[0x1C] = 0x38; m[0x1D] = 0x30 // 8 0
// Whitespace / control.
m[0x24] = 0x0D // return
m[0x30] = 0x09 // tab
m[0x31] = 0x20 // space
m[0x33] = 0x08 // delete (backspace)
m[0x35] = 0x1B // escape
m[0x75] = 0x2E // forward delete (VK_DELETE)
m[0x39] = 0x14 // caps lock
// Punctuation (US ANSI) + ISO 102nd key.
m[0x1B] = 0xBD; m[0x18] = 0xBB // - = (OEM_MINUS OEM_PLUS)
m[0x21] = 0xDB; m[0x1E] = 0xDD; m[0x2A] = 0xDC // [ ] backslash (OEM_4 6 5)
m[0x29] = 0xBA; m[0x27] = 0xDE; m[0x32] = 0xC0 // ; ' ` (OEM_1 7 3)
m[0x2B] = 0xBC; m[0x2F] = 0xBE; m[0x2C] = 0xBF // , . / (OEM_COMMA PERIOD 2)
m[0x0A] = 0xE2 // ISO 102nd key (<> next to left shift; OEM_102)
// Function keys F1..F12 (scattered) VK 0x70..0x7B. F13+ omitted (no host arm).
m[0x7A] = 0x70; m[0x78] = 0x71; m[0x63] = 0x72; m[0x76] = 0x73 // F1 F2 F3 F4
m[0x60] = 0x74; m[0x61] = 0x75; m[0x62] = 0x76; m[0x64] = 0x77 // F5 F6 F7 F8
m[0x65] = 0x78; m[0x6D] = 0x79; m[0x67] = 0x7A; m[0x6F] = 0x7B // F9 F10 F11 F12
// Arrows.
m[0x7B] = 0x25; m[0x7C] = 0x27; m[0x7D] = 0x28; m[0x7E] = 0x26 // left right down up
// Nav cluster (Apple keycodes; Help sits where Insert is).
m[0x72] = 0x2D; m[0x73] = 0x24; m[0x74] = 0x21 // insert home pageup
m[0x77] = 0x23; m[0x79] = 0x22 // end pagedown (forward-delete handled above)
// Keypad kVK_ANSI_Keypad0..9 (scattered) VK_NUMPAD0..9, plus the operators.
m[0x52] = 0x60; m[0x53] = 0x61; m[0x54] = 0x62; m[0x55] = 0x63 // KP0 KP1 KP2 KP3
m[0x56] = 0x64; m[0x57] = 0x65; m[0x58] = 0x66; m[0x59] = 0x67 // KP4 KP5 KP6 KP7
m[0x5B] = 0x68; m[0x5C] = 0x69 // KP8 KP9
m[0x41] = 0x6E; m[0x43] = 0x6A; m[0x45] = 0x6B // KP decimal multiply plus
m[0x4E] = 0x6D; m[0x4B] = 0x6F // KP minus divide
m[0x4C] = 0x6C // KP enter VK_SEPARATOR (host maps to KEY_KPENTER, matching hidToVK)
m[0x47] = 0x90 // KP clear sits where NumLock is VK_NUMLOCK. (KP equals 0x51 dropped.)
return m
}()
#endif
}
@@ -0,0 +1,102 @@
// InputCapture's static keymap tables: HID usage Windows VK (the GCKeyboard path on all
// platforms) and, on macOS, NSEvent.keyCode Windows VK (the NSEvent key path).
extension InputCapture {
/// HID usage (GCKeyCode raw) Windows VK (the host maps VK evdev; every VK emitted
/// here exists in punktfunk-host/src/inject.rs::vk_to_evdev extend the two together).
static let hidToVK: [Int: UInt32] = {
var m: [Int: UInt32] = [:]
// az: HID 0x04..0x1D VK 'A'..'Z'.
for i in 0..<26 { m[0x04 + i] = UInt32(0x41 + i) }
// 19: HID 0x1E..0x26 VK '1'..'9'; then 0: HID 0x27 VK '0' (set separately
// the '0' key sits AFTER '9' in HID but its VK 0x30 sits BEFORE '1' (0x31)).
for i in 0..<9 { m[0x1E + i] = UInt32(0x31 + i) }
m[0x27] = 0x30
m[0x28] = 0x0D // return
m[0x29] = 0x1B // escape
m[0x2A] = 0x08 // backspace
m[0x2B] = 0x09 // tab
m[0x2C] = 0x20 // space
m[0x2D] = 0xBD; m[0x2E] = 0xBB // - =
m[0x2F] = 0xDB; m[0x30] = 0xDD; m[0x31] = 0xDC // [ ] backslash
m[0x33] = 0xBA; m[0x34] = 0xDE; m[0x35] = 0xC0 // ; ' `
m[0x36] = 0xBC; m[0x37] = 0xBE; m[0x38] = 0xBF // , . /
m[0x39] = 0x14 // caps lock
// F1..F12: HID 0x3A..0x45 VK 0x70..0x7B.
for i in 0..<12 { m[0x3A + i] = UInt32(0x70 + i) }
m[0x46] = 0x2C; m[0x47] = 0x91; m[0x48] = 0x13 // printscreen scrolllock pause
m[0x4F] = 0x27; m[0x50] = 0x25; m[0x51] = 0x28; m[0x52] = 0x26 // arrows R L D U
m[0x49] = 0x2D; m[0x4A] = 0x24; m[0x4B] = 0x21 // insert home pageup
m[0x4C] = 0x2E; m[0x4D] = 0x23; m[0x4E] = 0x22 // delete end pagedown
// Keypad: NumLock, / * - +, Enter, 1..9, 0, decimal. KP Enter goes as
// VK_SEPARATOR (0x6C) this host maps it to KEY_KPENTER (Windows itself would
// send VK_RETURN+extended, which vk_to_evdev can't distinguish).
m[0x53] = 0x90
m[0x54] = 0x6F; m[0x55] = 0x6A; m[0x56] = 0x6D; m[0x57] = 0x6B
m[0x58] = 0x6C
for i in 0..<9 { m[0x59 + i] = UInt32(0x61 + i) }
m[0x62] = 0x60; m[0x63] = 0x6E
m[0x64] = 0xE2 // ISO 102nd key (<> next to left shift on ISO layouts)
m[0x65] = 0x5D // menu/application
m[0xE0] = 0xA2; m[0xE1] = 0xA0; m[0xE2] = 0xA4; m[0xE3] = 0x5B // Lctrl Lshift Lalt Lcmd
m[0xE4] = 0xA3; m[0xE5] = 0xA1; m[0xE6] = 0xA5; m[0xE7] = 0x5C // Rctrl Rshift Ralt Rcmd
return m
}()
#if os(macOS)
/// NSEvent.keyCode (Carbon virtual keycode, kVK_*) Windows VK. The macOS NSEvent key
/// path is keyed by keyCode (a layout-independent hardware position), NOT by HID usage,
/// so it needs its own table but it emits the EXACT SAME Windows VK integers `hidToVK`
/// already produces for each physical key (A0x41, Return0x0D, KeypadEnter0x6C, ), so
/// the host's vk_to_evdev (inject.rs) accepts both with zero change. Modifier keys come
/// via flagsChanged (handleFlagsChanged), not keyDown, so they're absent here. Keys with
/// no host evdev arm (F13F20, KeypadEquals, the Fn key) are omitted nil swallowed.
static let keyCodeToVK: [UInt16: UInt32] = {
var m: [UInt16: UInt32] = [:]
// Letters kVK_ANSI_A..Z (scattered keycodes) VK 'A'..'Z'.
m[0x00] = 0x41; m[0x01] = 0x53; m[0x02] = 0x44; m[0x03] = 0x46 // A S D F
m[0x04] = 0x48; m[0x05] = 0x47; m[0x06] = 0x5A; m[0x07] = 0x58 // H G Z X
m[0x08] = 0x43; m[0x09] = 0x56; m[0x0B] = 0x42; m[0x0C] = 0x51 // C V B Q
m[0x0D] = 0x57; m[0x0E] = 0x45; m[0x0F] = 0x52; m[0x10] = 0x59 // W E R Y
m[0x11] = 0x54; m[0x1F] = 0x4F; m[0x20] = 0x55; m[0x22] = 0x49 // T O U I
m[0x23] = 0x50; m[0x25] = 0x4C; m[0x26] = 0x4A; m[0x28] = 0x4B // P L J K
m[0x2D] = 0x4E; m[0x2E] = 0x4D // N M
// Digit row kVK_ANSI_1..0 (scattered) VK '1'..'9','0'.
m[0x12] = 0x31; m[0x13] = 0x32; m[0x14] = 0x33; m[0x15] = 0x34 // 1 2 3 4
m[0x16] = 0x36; m[0x17] = 0x35; m[0x19] = 0x39; m[0x1A] = 0x37 // 6 5 9 7
m[0x1C] = 0x38; m[0x1D] = 0x30 // 8 0
// Whitespace / control.
m[0x24] = 0x0D // return
m[0x30] = 0x09 // tab
m[0x31] = 0x20 // space
m[0x33] = 0x08 // delete (backspace)
m[0x35] = 0x1B // escape
m[0x75] = 0x2E // forward delete (VK_DELETE)
m[0x39] = 0x14 // caps lock
// Punctuation (US ANSI) + ISO 102nd key.
m[0x1B] = 0xBD; m[0x18] = 0xBB // - = (OEM_MINUS OEM_PLUS)
m[0x21] = 0xDB; m[0x1E] = 0xDD; m[0x2A] = 0xDC // [ ] backslash (OEM_4 6 5)
m[0x29] = 0xBA; m[0x27] = 0xDE; m[0x32] = 0xC0 // ; ' ` (OEM_1 7 3)
m[0x2B] = 0xBC; m[0x2F] = 0xBE; m[0x2C] = 0xBF // , . / (OEM_COMMA PERIOD 2)
m[0x0A] = 0xE2 // ISO 102nd key (<> next to left shift; OEM_102)
// Function keys F1..F12 (scattered) VK 0x70..0x7B. F13+ omitted (no host arm).
m[0x7A] = 0x70; m[0x78] = 0x71; m[0x63] = 0x72; m[0x76] = 0x73 // F1 F2 F3 F4
m[0x60] = 0x74; m[0x61] = 0x75; m[0x62] = 0x76; m[0x64] = 0x77 // F5 F6 F7 F8
m[0x65] = 0x78; m[0x6D] = 0x79; m[0x67] = 0x7A; m[0x6F] = 0x7B // F9 F10 F11 F12
// Arrows.
m[0x7B] = 0x25; m[0x7C] = 0x27; m[0x7D] = 0x28; m[0x7E] = 0x26 // left right down up
// Nav cluster (Apple keycodes; Help sits where Insert is).
m[0x72] = 0x2D; m[0x73] = 0x24; m[0x74] = 0x21 // insert home pageup
m[0x77] = 0x23; m[0x79] = 0x22 // end pagedown (forward-delete handled above)
// Keypad kVK_ANSI_Keypad0..9 (scattered) VK_NUMPAD0..9, plus the operators.
m[0x52] = 0x60; m[0x53] = 0x61; m[0x54] = 0x62; m[0x55] = 0x63 // KP0 KP1 KP2 KP3
m[0x56] = 0x64; m[0x57] = 0x65; m[0x58] = 0x66; m[0x59] = 0x67 // KP4 KP5 KP6 KP7
m[0x5B] = 0x68; m[0x5C] = 0x69 // KP8 KP9
m[0x41] = 0x6E; m[0x43] = 0x6A; m[0x45] = 0x6B // KP decimal multiply plus
m[0x4E] = 0x6D; m[0x4B] = 0x6F // KP minus divide
m[0x4C] = 0x6C // KP enter VK_SEPARATOR (host maps to KEY_KPENTER, matching hidToVK)
m[0x47] = 0x90 // KP clear sits where NumLock is VK_NUMLOCK. (KP equals 0x51 dropped.)
return m
}()
#endif
}
@@ -0,0 +1,9 @@
import CoreGraphics
extension CGFloat {
/// Clamp into `range` keeps the absolute-cursor mapping inside the host's pixel
/// bounds even if a stray event reports a point a hair past the video rect.
func clamped(to range: ClosedRange<CGFloat>) -> CGFloat {
Swift.min(Swift.max(self, range.lowerBound), range.upperBound)
}
}
@@ -51,8 +51,8 @@ public enum DefaultsKey {
/// Which corner the statistics overlay sits in a `HUDPlacement` raw value
/// ("topLeading"/"topTrailing"/"bottomLeading"/"bottomTrailing"). Default top-trailing.
public static let hudPlacement = "punktfunk.hudPlacement"
/// iOS/iPadOS: switch the host list and game library to a controller-friendly layout
/// (larger focus targets, a coverflow-style library) whenever a gamepad is connected. On by
/// default; see `GamepadUIEnvironment.isActive`.
/// iOS/iPadOS/macOS: switch the host list, settings and game library to a controller-friendly
/// layout (the console launcher, gamepad-navigable settings, a coverflow-style library)
/// whenever a gamepad is connected. On by default; see `GamepadUIEnvironment.isActive`.
public static let gamepadUIEnabled = "punktfunk.gamepadUIEnabled"
}
@@ -0,0 +1,21 @@
// One NSLock-guarded boolean, set once: the cancellation handle shared by the session services
// (the two video pumps, audio playback/mic, gamepad feedback). Each start() creates a fresh flag
// and hands it to its worker thread(s); stop() sets it permanently, so a stale worker can never
// be revived by a newer start.
import Foundation
final class StopFlag: @unchecked Sendable {
private let lock = NSLock()
private var stopped = false
var isStopped: Bool {
lock.lock()
defer { lock.unlock() }
return stopped
}
func stop() {
lock.lock()
stopped = true
lock.unlock()
}
}
@@ -0,0 +1,265 @@
// Annex-B (HEVC / H.264) CoreMedia plumbing.
//
// The punktfunk host emits Annex-B access units with in-band parameter sets on every IDR
// (deliberately the client needs no out-of-band extradata). VideoToolbox wants the AVCC
// flavor instead: a CMVideoFormatDescription built from the parameter sets, and sample
// buffers whose NALs are 4-byte-length-prefixed. This file converts between the two, for
// the codec the host resolved in the Welcome (`connection.videoCodec`) HEVC and H.264
// differ only in NAL-header layout and which parameter sets exist (HEVC adds a VPS). AV1
// is not an Annex-B/NAL codec and isn't handled here (hosts don't emit it on the native
// path yet).
//
// HOT PATH: both pumps run `formatDescription(fromIDR:codec:)` + `sampleBuffer(au:format:codec:)`
// once per AU, so the conversion is built on `forEachNAL` a zero-copy scan over the AU's bytes
// (ranges, not materialized Datas) and `sampleBuffer` packs the AVCC form straight into
// the CMBlockBuffer's own allocation. Per AU that leaves exactly one copy here (source
// block buffer) instead of the naive scan-copy-slice-repack chain.
import CoreMedia
import Foundation
/// The video codec of the host's elementary stream negotiated in the Welcome and read via
/// `punktfunk_connection_codec`.
public enum VideoCodec: Equatable {
case h264
case hevc
/// Resolve from the wire `Welcome.codec` byte (`PUNKTFUNK_CODEC_*`; unknown HEVC).
public init(wire: UInt8) {
self = wire == 0x01 ? .h264 : .hevc // 0x01 = PUNKTFUNK_CODEC_H264
}
/// NAL unit type from a NAL's first byte. HEVC: bits 1..6; H.264: bits 0..4.
fileprivate func nalType(_ first: UInt8) -> UInt8 {
self == .hevc ? (first >> 1) & 0x3F : first & 0x1F
}
/// True for a parameter-set NAL (dropped from AVCC; kept for the format description).
/// HEVC: VPS 32 / SPS 33 / PPS 34. H.264: SPS 7 / PPS 8 (no VPS).
fileprivate func isParameterSet(_ first: UInt8) -> Bool {
let t = nalType(first)
return self == .hevc ? (32...34).contains(t) : t == 7 || t == 8
}
/// True for a VCL (slice) NAL in a conforming AU no parameter set follows the first one,
/// so the format-description scan can stop there.
fileprivate func isVCL(_ first: UInt8) -> Bool {
let t = nalType(first)
return self == .hevc ? t <= 31 : (1...5).contains(t)
}
}
public enum AnnexB {
/// Walk the NAL units of `data` without copying: `body` receives the buffer base and each
/// NAL's byte range (start codes 00 00 01 / 00 00 00 01 excluded), and returns false to
/// stop the walk early (e.g. at the first VCL NAL). All zeros immediately preceding a
/// start code are dropped: they're either the 4-byte-code prefix or `trailing_zero_8bits`
/// padding, never NAL payload (emulation prevention keeps 00 00 0x out of conforming NAL
/// bytes) same policy as ffmpeg. The base pointer is only valid inside `body`.
static func forEachNAL(
in data: Data, _ body: (_ base: UnsafePointer<UInt8>, _ range: Range<Int>) -> Bool
) {
data.withUnsafeBytes { (raw: UnsafeRawBufferPointer) in
guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return }
let count = raw.count
var i = 0
var start = -1
while i + 2 < count {
if base[i] == 0, base[i + 1] == 0, base[i + 2] == 1 {
var codeStart = i
while codeStart > 0, base[codeStart - 1] == 0 {
codeStart -= 1
}
if start >= 0, start < codeStart, !body(base, start..<codeStart) { return }
start = i + 3
i += 3
} else {
i += 1
}
}
if start >= 0, start < count {
_ = body(base, start..<count)
}
}
}
/// Split an Annex-B stream into NAL units (start codes stripped see `forEachNAL` for
/// the boundary policy). Materializes a Data per NAL; the streaming paths use
/// `forEachNAL` directly instead.
public static func nalUnits(in data: Data) -> [Data] {
var nals: [Data] = []
forEachNAL(in: data) { base, range in
nals.append(Data(bytes: base + range.lowerBound, count: range.count))
return true
}
return nals
}
/// HEVC NAL unit type (bits 1..6 of the first byte).
public static func hevcNalType(_ nal: Data) -> UInt8 {
guard let first = nal.first else { return 0xFF }
return (first >> 1) & 0x3F
}
/// H.264 NAL unit type (bits 0..4 of the first byte).
public static func h264NalType(_ nal: Data) -> UInt8 {
guard let first = nal.first else { return 0xFF }
return first & 0x1F
}
/// Build a format description from an IDR AU's in-band parameter sets (HEVC: VPS/SPS/PPS;
/// H.264: SPS/PPS). Returns nil when the AU carries no parameter sets (non-IDR). Runs per
/// AU on the pump thread: parameter sets precede the first VCL NAL in a conforming AU, so
/// the scan stops there a delta frame (no leading parameter sets) costs a few byte
/// compares, no copies.
public static func formatDescription(
fromIDR au: Data, codec: VideoCodec
) -> CMVideoFormatDescription? {
var vps: Data?, sps: Data?, pps: Data?
forEachNAL(in: au) { base, range in
let first = base[range.lowerBound]
switch codec.nalType(first) {
case 32 where codec == .hevc:
vps = Data(bytes: base + range.lowerBound, count: range.count)
case 33 where codec == .hevc, 7 where codec == .h264:
sps = Data(bytes: base + range.lowerBound, count: range.count)
case 34 where codec == .hevc, 8 where codec == .h264:
pps = Data(bytes: base + range.lowerBound, count: range.count)
default:
if codec.isVCL(first) { return false } // no parameter sets can follow
// AUD/SEI/ may precede the slices; keep scanning.
}
return true
}
guard let sps, let pps else { return nil }
// In the order VideoToolbox wants them: HEVC VPS,SPS,PPS (VPS required); H.264 SPS,PPS.
let sets: [Data]
switch codec {
case .hevc:
guard let vps else { return nil }
sets = [vps, sps, pps]
case .h264:
sets = [sps, pps]
}
var format: CMVideoFormatDescription?
// Pin every parameter set's bytes for the duration of the create call, then hand
// VideoToolbox parallel pointer/size arrays.
var pointers: [UnsafePointer<UInt8>] = []
var sizes: [Int] = []
func withAll(_ i: Int, _ body: () -> Void) {
if i == sets.count { body(); return }
sets[i].withUnsafeBytes { raw in
pointers.append(raw.bindMemory(to: UInt8.self).baseAddress!)
sizes.append(sets[i].count)
withAll(i + 1, body)
}
}
var status: OSStatus = -1
withAll(0) {
switch codec {
case .hevc:
status = CMVideoFormatDescriptionCreateFromHEVCParameterSets(
allocator: kCFAllocatorDefault,
parameterSetCount: pointers.count,
parameterSetPointers: pointers,
parameterSetSizes: sizes,
nalUnitHeaderLength: 4,
extensions: nil,
formatDescriptionOut: &format)
case .h264:
status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
allocator: kCFAllocatorDefault,
parameterSetCount: pointers.count,
parameterSetPointers: pointers,
parameterSetSizes: sizes,
nalUnitHeaderLength: 4,
formatDescriptionOut: &format)
}
}
return status == noErr ? format : nil
}
/// Re-pack an Annex-B AU as AVCC (4-byte big-endian length before each NAL), dropping
/// the parameter-set NALs (they live in the format description).
public static func avcc(from au: Data, codec: VideoCodec) -> Data {
var out = Data(capacity: au.count + 16)
forEachNAL(in: au) { base, range in
if codec.isParameterSet(base[range.lowerBound]) { return true }
var len = UInt32(range.count).bigEndian
withUnsafeBytes(of: &len) { out.append(contentsOf: $0) }
out.append(UnsafeBufferPointer(start: base + range.lowerBound, count: range.count))
return true
}
return out
}
/// Wrap one AU as a decode-ready CMSampleBuffer. The AVCC form is packed directly into
/// the CMBlockBuffer's allocation (sized by a first cheap scan) no intermediate Data.
public static func sampleBuffer(
au: AccessUnit, format: CMVideoFormatDescription, codec: VideoCodec
) -> CMSampleBuffer? {
// Pass 1: byte scan only total AVCC size of the payload (non-parameter-set) NALs.
var total = 0
forEachNAL(in: au.data) { base, range in
if !codec.isParameterSet(base[range.lowerBound]) { total += 4 + range.count }
return true
}
// Nothing decodable (a parameter-set-only AU our host never sends one): drop it
// rather than hand the decoder an empty sample.
guard total > 0 else { return nil }
var blockBuffer: CMBlockBuffer?
guard CMBlockBufferCreateWithMemoryBlock(
allocator: kCFAllocatorDefault, memoryBlock: nil,
blockLength: total, blockAllocator: kCFAllocatorDefault,
customBlockSource: nil, offsetToData: 0, dataLength: total,
flags: kCMBlockBufferAssureMemoryNowFlag, blockBufferOut: &blockBuffer) == noErr,
let block = blockBuffer
else { return nil }
var dstLen = 0
var dstPtr: UnsafeMutablePointer<CChar>?
guard CMBlockBufferGetDataPointer(
block, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: &dstLen,
dataPointerOut: &dstPtr) == noErr,
dstLen == total, let dstPtr
else { return nil }
// Pass 2: the single copy length prefix + payload per NAL, straight into the block.
let dst = UnsafeMutableRawPointer(dstPtr)
var off = 0
forEachNAL(in: au.data) { base, range in
if codec.isParameterSet(base[range.lowerBound]) { return true }
var len = UInt32(range.count).bigEndian
withUnsafeBytes(of: &len) {
dst.advanced(by: off).copyMemory(from: $0.baseAddress!, byteCount: 4)
}
dst.advanced(by: off + 4)
.copyMemory(from: base + range.lowerBound, byteCount: range.count)
off += 4 + range.count
return true
}
var timing = CMSampleTimingInfo(
duration: .invalid,
presentationTimeStamp: CMTime(value: Int64(au.ptsNs), timescale: 1_000_000_000),
decodeTimeStamp: .invalid)
var sampleSize = total
var sample: CMSampleBuffer?
guard CMSampleBufferCreate(
allocator: kCFAllocatorDefault, dataBuffer: block, dataReady: true,
makeDataReadyCallback: nil, refcon: nil, formatDescription: format,
sampleCount: 1, sampleTimingEntryCount: 1, sampleTimingArray: &timing,
sampleSizeEntryCount: 1, sampleSizeArray: &sampleSize,
sampleBufferOut: &sample) == noErr
else { return nil }
// Low-latency display: render on arrival, don't wait for a clock.
if let attachments = CMSampleBufferGetSampleAttachmentsArray(sample!, createIfNecessary: true) {
let dict = unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self)
CFDictionarySetValue(
dict,
Unmanaged.passUnretained(kCMSampleAttachmentKey_DisplayImmediately).toOpaque(),
Unmanaged.passUnretained(kCFBooleanTrue).toOpaque())
}
return sample
}
}
@@ -0,0 +1,29 @@
// Throttled host keyframe requests for decode recovery, shared by both pumps (StreamPump /
// Stage2Pipeline). Wedge signals arrive from several threads the decoder's async error callback
// (a VT thread), a submit failure on the pump thread, the framesDropped poll and the decode stays
// stalled for several frames until the requested IDR lands, so requests are coalesced (100 ms, the
// throttle the working Android path uses: fast enough that a lost recovery IDR is re-requested
// promptly, bounded so a sustained freeze can't flood the control stream). Bound to the live
// connection at pump start, unbound on stop.
import Foundation
final class KeyframeRecovery: @unchecked Sendable {
private let lock = NSLock()
private var connection: PunktfunkConnection?
private var lastNs: UInt64 = 0
func bind(_ c: PunktfunkConnection?) {
lock.lock(); connection = c; lastNs = 0; lock.unlock()
}
func request() {
lock.lock()
let now = DispatchTime.now().uptimeNanoseconds
let due = lastNs == 0 || now &- lastNs > 100_000_000 // 100 ms since the last request
if due { lastNs = now }
let conn = due ? connection : nil
lock.unlock()
conn?.requestKeyframe()
}
}
@@ -44,10 +44,11 @@ vertex VOut pf_vtx(uint vid [[vertex_id]]) {
return o;
}
// Bicubic (Catmull-Rom) sampling of the single-channel luma plane. When the drawable is larger
// than the decoded frame (a window/view bigger than the host's fixed mode), a bilinear upscale
// looks soft; Catmull-Rom keeps edges crisp matching AVSampleBufferDisplayLayer's (stage-1)
// scaler and reduces to the exact texel at 1:1, so a native-resolution present stays pixel-exact.
// Bicubic (Catmull-Rom) sampling of the single-channel luma plane. The drawable is sized to the
// LAYER's pixels (see `render`), so this kernel performs the decodedon-screen scale: when the
// window/view is bigger than the host's fixed mode a bilinear upscale looks soft; Catmull-Rom
// keeps edges crisp matching AVSampleBufferDisplayLayer's (stage-1) scaler and reduces to the
// exact texel at 1:1, so a native-resolution present stays pixel-exact.
// Nine bilinear taps (TheRealMJP's optimisation of the 16-tap kernel); `s` MUST be a linear
// sampler. Luma carries the perceived detail, so only it gets bicubic; chroma stays bilinear.
float catmullRomLuma(texture2d<float> tex, sampler s, float2 uv) {
@@ -77,14 +78,27 @@ float catmullRomLuma(texture2d<float> tex, sampler s, float2 uv) {
return r;
}
// 4:2:0 chroma is left-cosited horizontally (H.273 chroma_loc type 0 the MPEG convention the
// host encodes and VideoToolbox decodes as-is), but sampling the half-res plane at the luma UV
// assumes CENTER siting a ~0.5-luma-px rightward chroma shift on hard colored edges. Offset the
// sample by +0.25 chroma texels to re-align (libplacebo/mpv's correction). Vertical siting for
// type 0 is centered, which plain sampling already matches. A full-size 4:4:4 plane has no
// subsampling to correct the offset self-disables when the plane widths match.
float2 chromaUV(texture2d<float> lumaTex, texture2d<float> chromaTex, float2 uv) {
if (chromaTex.get_width() < lumaTex.get_width()) {
uv.x += 0.25 / float(chromaTex.get_width());
}
return uv;
}
// SDR: 8-bit NV12 / 4:4:4 (BT.709, limited/video range) full-range RGB. Chroma is sampled at the
// same UV as luma, so a full-size 4:4:4 chroma plane needs no shader change vs 4:2:0.
// (siting-corrected) luma UV, so a full-size 4:4:4 chroma plane needs no shader change vs 4:2:0.
fragment float4 pf_frag(VOut in [[stage_in]],
texture2d<float> lumaTex [[texture(0)]],
texture2d<float> chromaTex [[texture(1)]]) {
constexpr sampler s(filter::linear, address::clamp_to_edge);
float y = catmullRomLuma(lumaTex, s, in.uv);
float2 c = chromaTex.sample(s, in.uv).rg;
float2 c = chromaTex.sample(s, chromaUV(lumaTex, chromaTex, in.uv)).rg;
// BT.709, 8-bit limited (video) range full-range RGB.
y = (y - 16.0/255.0) * (255.0/219.0);
float u = (c.x - 128.0/255.0) * (255.0/224.0);
@@ -105,7 +119,7 @@ fragment float4 pf_frag_hdr(VOut in [[stage_in]],
texture2d<float> chromaTex [[texture(1)]]) {
constexpr sampler s(filter::linear, address::clamp_to_edge);
float y = catmullRomLuma(lumaTex, s, in.uv);
float2 c = chromaTex.sample(s, in.uv).rg;
float2 c = chromaTex.sample(s, chromaUV(lumaTex, chromaTex, in.uv)).rg;
// BT.2020 10-bit limited (video) range full-range PQ RGB.
y = (y - 64.0/1023.0) * (1023.0/876.0);
float u = (c.x - 512.0/1023.0) * (1023.0/896.0);
@@ -185,10 +199,11 @@ public final class MetalVideoPresenter {
// (the display link is the pacing source) the fix for the fullscreen stutter. macOS-only.
layer.displaySyncEnabled = false
#endif
// Render the drawable at the DECODED frame's resolution (set per-frame in `render`) and let the
// system compositor scale it to the layer's bounds the same `.resizeAspect` path stage-1's
// AVSampleBufferDisplayLayer uses. A native-resolution present is then pixel-exact (1:1, no
// shader scaling); a resized window rescales via the system's scaler.
// The drawable is rendered at the LAYER's pixel size (set per-frame in `render`), so the
// shader not the compositor performs the decodedon-screen scale (bicubic luma; the
// compositor's contentsGravity path is plain bilinear). The gravity stays aspect-fit as a
// transient fallback: during a live resize the compositor may composite a drawable from
// the previous layout before the next render catches up.
layer.contentsGravity = .resizeAspect
// Triple-buffer: more in-flight drawables before `nextDrawable()` (called on the display-link /
// MAIN thread) has to block waiting for one to free.
@@ -277,9 +292,15 @@ public final class MetalVideoPresenter {
/// Draw one decoded frame to the next drawable and present it. MAIN THREAD (the display link).
/// `isHDR` selects the 10-bit BT.2020 PQ path vs the 8-bit BT.709 path and is reconciled with the
/// layer config via `configure`. Returns true on success; false when there's no drawable yet, a
/// texture couldn't be made, or Metal errored the caller then doesn't stamp a present.
/// texture couldn't be made, or Metal errored the caller then doesn't stamp a present (and can
/// requeue the frame). `onPresented` fires once the drawable actually reached glass, with the
/// `CLOCK_REALTIME` instant from the drawable's `presentedTime` or nil when the system reports
/// none (a dropped drawable). It runs on a Metal callback thread; keep the handler thread-safe.
@discardableResult
public func render(_ pixelBuffer: CVPixelBuffer, isHDR: Bool = false) -> Bool {
public func render(
_ pixelBuffer: CVPixelBuffer, isHDR: Bool = false,
onPresented: ((Int64?) -> Void)? = nil
) -> Bool {
// Reconcile the layer with the decoded frame's HDR-ness (handles a mid-session SDRHDR flip).
configure(hdr: isHDR)
@@ -298,15 +319,25 @@ public final class MetalVideoPresenter {
pixelBuffer, plane: 1, format: tenBit ? .rg16Unorm : .rg8Unorm, cache: textureCache)
else { return false }
// Size the drawable to the decoded frame so the fullscreen triangle samples 1:1 (pixel-exact);
// the layer's contentsGravity then scales it to the on-screen bounds via the system compositor
// (matching stage-1). drawableSize does NOT track bounds (defaults to 0), so set it BEFORE
// nextDrawable; re-set only on a change (first frame / Reconfigure / HDR flip).
// Size the drawable to the LAYER's pixels (bounds × contentsScale, both set by the hosting
// view's layout) so the Catmull-Rom shader performs the decodedon-screen scale in one pass:
// a native-mode session stays exactly 1:1 (the kernel reduces to the identity texel), and a
// window bigger than the host's mode gets bicubic luma instead of the compositor's bilinear.
// Before the first layout (empty bounds) fall back to the decoded size. drawableSize does NOT
// track bounds (defaults to 0), so set it BEFORE nextDrawable; re-set only on a change
// (layout / Reconfigure / HDR flip and every frame of a live resize, which is fine).
let decodedSize = CGSize(
width: CVPixelBufferGetWidth(pixelBuffer), height: CVPixelBufferGetHeight(pixelBuffer))
if layer.drawableSize != decodedSize { layer.drawableSize = decodedSize }
let scale = layer.contentsScale
let boundsSize = layer.bounds.size
let targetSize = (boundsSize.width > 0 && boundsSize.height > 0)
? CGSize(
width: (boundsSize.width * scale).rounded(),
height: (boundsSize.height * scale).rounded())
: decodedSize
if layer.drawableSize != targetSize { layer.drawableSize = targetSize }
#if DEBUG
logSizeIfChanged(decoded: decodedSize)
logSizeIfChanged(decoded: decodedSize, drawable: targetSize)
#endif
guard let drawable = layer.nextDrawable(),
let commandBuffer = queue.makeCommandBuffer()
@@ -325,6 +356,24 @@ public final class MetalVideoPresenter {
encoder.setFragmentTexture(CVMetalTextureGetTexture(chroma), index: 1)
encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3)
encoder.endEncoding()
if let onPresented {
#if targetEnvironment(simulator)
// The simulator SDK exposes neither addPresentedHandler nor presentedTime report
// nil so the caller stamps with its display-link estimate (the pre-presentedTime
// behavior; simulator numbers are indicative only anyway).
onPresented(nil)
#else
// Registered BEFORE present. presentedTime is CACurrentMediaTime-based; 0 means the
// system never put this drawable on glass (dropped) report nil, the caller falls
// back to its display-link estimate.
drawable.addPresentedHandler { d in
onPresented(
d.presentedTime > 0
? Stage2Pipeline.realtimeNs(forDisplayLinkTimestamp: d.presentedTime)
: nil)
}
#endif
}
commandBuffer.present(drawable) // present at the next vsync lowest latency
// Hold the CVMetalTextures + source pixel buffer (its IOSurface) alive until the GPU finishes
// sampling releasing them at scope exit could free the backing mid-read.
@@ -350,11 +399,12 @@ public final class MetalVideoPresenter {
}
#if DEBUG
private func logSizeIfChanged(decoded: CGSize) {
let sig = "\(Int(decoded.width))x\(Int(decoded.height))|hdr\(hdrActive ? 1 : 0)"
private func logSizeIfChanged(decoded: CGSize, drawable: CGSize) {
let sig = "\(Int(decoded.width))x\(Int(decoded.height))\(Int(drawable.width))x\(Int(drawable.height))|hdr\(hdrActive ? 1 : 0)"
if sig != lastSizeSig {
lastSizeSig = sig
let msg = "stage2: decoded \(Int(decoded.width))x\(Int(decoded.height)) hdr=\(hdrActive)"
let msg =
"stage2: decoded \(Int(decoded.width))x\(Int(decoded.height)) → drawable \(Int(drawable.width))x\(Int(drawable.height)) hdr=\(hdrActive)"
presenterLog.info("\(msg, privacy: .public)")
}
}
@@ -0,0 +1,153 @@
// Per-session presenter stack shared by the macOS and iOS/tvOS stream views: stage-2 (explicit
// VTDecompressionSession decode CAMetalLayer, driven by the hosting view's CADisplayLink) is the
// default; stage-1 (StreamPump AVSampleBufferDisplayLayer) is the Metal-unavailable / DEBUG
// fallback. The views own the platform bits capture, window/scale tracking, and constructing the
// display link and delegate the shared presenter lifecycle here.
//
// Main-thread only: start/layout/stop and the display-link tick all run on the main runloop.
#if canImport(Metal) && canImport(QuartzCore)
import AVFoundation
import Foundation
import QuartzCore
/// Weak-target wrapper for CADisplayLink. The link retains its target, so targeting a view or
/// presenter directly makes a `owner link owner` cycle that only `invalidate()` breaks if a
/// teardown is ever missed the owner leaks and keeps ticking. The proxy is what the link retains;
/// the handler closure captures the owner `[weak]`, so the owner can deallocate and its `deinit`
/// invalidate the link.
public final class DisplayLinkProxy: NSObject {
private let onTick: (CADisplayLink) -> Void
public init(_ onTick: @escaping (CADisplayLink) -> Void) { self.onTick = onTick }
@objc public func tick(_ link: CADisplayLink) { onTick(link) }
}
final class SessionPresenter {
private var pump: StreamPump?
private var stage2: Stage2Pipeline?
private var stage2Link: CADisplayLink?
private var metalLayer: CAMetalLayer?
private var connection: PunktfunkConnection?
/// Start the presenter for `connection`. `baseLayer` is the view's AVSampleBufferDisplayLayer:
/// stage-1 enqueues into it; stage-2 leaves it idle and composites an opaque CAMetalLayer
/// sublayer over it. `makeDisplayLink` supplies the platform link (macOS `NSView.displayLink`
/// tracks the view's display; iOS/tvOS uses the plain `CADisplayLink` init) only called when
/// stage-2 engages. Call `layout(in:contentsScale:)` right after so the sublayer has a frame
/// before the first tick.
func start(
connection: PunktfunkConnection,
baseLayer: AVSampleBufferDisplayLayer,
presentMeter: LatencyMeter?,
presentTailMeter: LatencyMeter? = nil,
makeDisplayLink: (AnyObject, Selector) -> CADisplayLink,
onFrame: (@Sendable (AccessUnit) -> Void)?,
onSessionEnd: (@Sendable () -> Void)?
) {
stop()
self.connection = connection
// Presenter choice stage-2 is the DEFAULT (explicit VTDecompressionSession decode + a
// CAMetalLayer/display-link present): it can detect + recover a wedged decoder where
// stage-1's AVSampleBufferDisplayLayer freezes hard on a lost HEVC reference. Stage-1 is
// reachable only via the DEBUG presenter toggle; release always takes stage-2 (the stage-1
// pump below stays the automatic fallback if Metal is missing).
#if DEBUG
let forceStage1 = UserDefaults.standard.string(forKey: DefaultsKey.presenter) == "stage1"
#else
let forceStage1 = false
#endif
if !forceStage1,
let pipeline = Stage2Pipeline(
presentMeter: presentMeter, presentTailMeter: presentTailMeter) {
let metal = pipeline.layer
// The opaque metal layer composites OVER the AVSampleBufferDisplayLayer base, which
// sits idle (un-enqueued) in stage-2. contentsScale + frame are set in layout().
baseLayer.addSublayer(metal)
metalLayer = metal
stage2 = pipeline
let proxy = DisplayLinkProxy { [weak self] link in
self?.stage2?.renderTick(
targetPresentNs: Stage2Pipeline.realtimeNs(
forDisplayLinkTimestamp: link.targetTimestamp))
}
let link = makeDisplayLink(proxy, #selector(DisplayLinkProxy.tick(_:)))
link.add(to: .main, forMode: .common)
stage2Link = link
syncFrameRate(hz: connection.currentMode().refreshHz)
pipeline.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
} else {
let pump = StreamPump()
pump.start(
connection: connection, layer: baseLayer,
onFrame: onFrame, onSessionEnd: onSessionEnd)
self.pump = pump
}
}
/// Ask the display link for the stream's own cadence. iOS/tvOS-only: without an explicit
/// range, ProMotion devices cap CADisplayLink at 60 Hz (iPhones additionally need
/// `CADisableMinimumFrameDurationOnPhone` in Info.plist), so a 120 fps stream would present
/// at half rate with the ring silently dropping every other frame. `maximum` allows up to
/// 120 so the system MAY tick faster than a sub-120 stream (each extra tick is a near-free
/// empty `renderTick`, and presenting on a denser grid shortens the decodeglass wait); the
/// macOS NSView link already tracks its display and must NOT be capped to the stream rate.
/// Re-applied from `layout` so a mid-session `Reconfigure` picks up a new refresh.
private func syncFrameRate(hz: UInt32) {
#if !os(macOS)
guard hz > 0, let link = stage2Link else { return }
let hzF = Float(hz)
if link.preferredFrameRateRange.preferred != hzF {
link.preferredFrameRateRange = CAFrameRateRange(
minimum: min(30, hzF), maximum: max(hzF, 120), preferred: hzF)
}
#endif
}
/// Position the stage-2 metal sublayer aspect-fit in the hosting view (the host streams at the
/// client's native mode, so this is usually the full bounds; it letterboxes a resized window).
/// The layer FRAME + contentsScale set here are what the presenter sizes its drawable from
/// (frame × scale) the shader then performs the decodedon-screen scale (bicubic luma), so a
/// native-mode session stays pixel-exact 1:1 and a mismatched window beats the compositor's
/// bilinear. No-op for stage-1 or before start.
func layout(in bounds: CGRect, contentsScale: CGFloat) {
guard let metalLayer, let connection else { return }
let mode = connection.currentMode()
syncFrameRate(hz: mode.refreshHz) // track a mid-session Reconfigure's new refresh
let fit: CGRect = (mode.width > 0 && mode.height > 0)
? AVMakeRect(
aspectRatio: CGSize(width: Int(mode.width), height: Int(mode.height)),
insideRect: bounds)
: bounds
// No implicit resize animation; contentsScale tracks the view's backing/display scale.
CATransaction.begin()
CATransaction.setDisableActions(true)
metalLayer.contentsScale = contentsScale
metalLayer.frame = fit
CATransaction.commit()
}
/// Stop the active pump/pipeline ( one poll timeout; stage-2 joins its pump) and detach the
/// stage-2 layer + link. Does not close the connection that stays with whoever owns it.
/// Idempotent.
func stop() {
pump?.stop()
pump = nil
stage2Link?.invalidate()
stage2Link = nil
stage2?.stop() // stops the pump (synchronous join) + drops the decode session
stage2 = nil
metalLayer?.removeFromSuperlayer()
metalLayer = nil
connection = nil
}
deinit {
// The owning view's stop() normally ran already; this covers a missed teardown so the
// display link can't keep ticking a deallocated pipeline.
stage2Link?.invalidate()
stage2?.stop()
pump?.stop()
}
}
#endif
@@ -12,16 +12,6 @@ import AVFoundation
import Foundation
import QuartzCore
/// Weak-target wrapper for CADisplayLink. The link retains its target, so targeting a view directly
/// makes a `view link view` cycle that only `invalidate()` breaks if a teardown is ever missed
/// the view leaks and keeps ticking. This proxy holds the handler weakly, so the view can deallocate
/// and its `deinit` invalidate the link.
public final class DisplayLinkProxy: NSObject {
private let onTick: (CADisplayLink) -> Void
public init(_ onTick: @escaping (CADisplayLink) -> Void) { self.onTick = onTick }
@objc public func tick(_ link: CADisplayLink) { onTick(link) }
}
/// Newest-ready 1-slot ring: the decoder overwrites (drops the older undisplayed frame lowest
/// latency, no smoothing buffer), the display link takes-and-clears. Sendable; lock-guarded.
private final class ReadyRing: @unchecked Sendable {
@@ -34,37 +24,15 @@ private final class ReadyRing: @unchecked Sendable {
lock.lock(); defer { lock.unlock() }
let f = frame; frame = nil; return f
}
}
/// Cancellation handle owned by one pump thread (same pattern as StreamPump).
private final class PumpToken: @unchecked Sendable {
private let lock = NSLock()
private var live = true
var isLive: Bool { lock.lock(); defer { lock.unlock() }; return live }
func cancel() { lock.lock(); live = false; lock.unlock() }
}
/// Throttled host keyframe requests for decode recovery. The decoder's async error callback (a VT
/// thread) and the pump thread (a submit failure) both signal a wedge; this coalesces them so the
/// control stream isn't flooded while the decode stays stalled for several frames until the requested
/// IDR lands. Bound to the live connection in `start`, unbound in `stop`.
private final class KeyframeRecovery: @unchecked Sendable {
private let lock = NSLock()
private var connection: PunktfunkConnection?
private var lastNs: UInt64 = 0
func bind(_ c: PunktfunkConnection?) {
lock.lock(); connection = c; lastNs = 0; lock.unlock()
}
func request() {
/// Return a frame the display link took but could not present (a transient `nextDrawable`
/// failure). Kept only while the slot is still empty a newer decoded frame wins, so
/// newest-ready ordering is preserved. Without this, a failed render silently LOSES the
/// frame, and under the host's infinite GOP a static scene sends no replacement until the
/// next damage the stale picture would persist.
func putBack(_ f: ReadyFrame) {
lock.lock()
let now = DispatchTime.now().uptimeNanoseconds
let due = lastNs == 0 || now &- lastNs > 100_000_000 // 100 ms since the last request
if due { lastNs = now }
let conn = due ? connection : nil
if frame == nil { frame = f }
lock.unlock()
conn?.requestKeyframe()
}
}
@@ -72,12 +40,13 @@ public final class Stage2Pipeline {
private let ring = ReadyRing()
private let presenter: MetalVideoPresenter
private let decoder: VideoDecoder
private let presentMeter: LatencyMeter
private let presentMeter: LatencyMeter?
private let presentTailMeter: LatencyMeter?
private let recovery = KeyframeRecovery()
private var token = PumpToken()
private var token = StopFlag()
private var offsetNs: Int64 = 0
/// Signalled when the pump thread exits, so `stop()` can join it (bounded) before `decoder.reset()`
/// otherwise a pump iteration already past its `token.isLive` check can rebuild a decode session
/// otherwise a pump iteration already past its `token.isStopped` check can rebuild a decode session
/// right after the reset (a brief orphan session). `pumpJoinable` is armed by `start`, consumed by
/// the first `stop` (so the idempotent second `stop`/deinit doesn't block on an already-drained
/// semaphore). start/stop are sequential lifecycle calls, so the plain flag is safe.
@@ -87,12 +56,15 @@ public final class Stage2Pipeline {
/// The Metal layer the hosting view installs + sizes.
public var layer: CAMetalLayer { presenter.layer }
/// `presentMeter` records capturepresent (the glass-to-glass term). Returns nil if Metal can't be
/// set up (headless / no GPU) caller falls back to the stage-1 presenter.
public init?(presentMeter: LatencyMeter) {
/// `presentMeter` records capturepresent (the glass-to-glass term); `presentTailMeter`
/// records decode-completionpresent (the ring wait + render the tail stage-2 exists to
/// shorten). Both optional: metering never gates the presenter choice. Returns nil if Metal
/// can't be set up (headless / no GPU) caller falls back to the stage-1 presenter.
public init?(presentMeter: LatencyMeter?, presentTailMeter: LatencyMeter? = nil) {
guard let presenter = MetalVideoPresenter.make() else { return nil }
self.presenter = presenter
self.presentMeter = presentMeter
self.presentTailMeter = presentTailMeter
let ring = ring
let recovery = recovery
self.decoder = VideoDecoder(
@@ -113,7 +85,7 @@ public final class Stage2Pipeline {
) {
offsetNs = connection.clockOffsetNs
recovery.bind(connection) // arm host-keyframe recovery for this session
token = PumpToken() // fresh token per start cancel is permanent (like StreamPump)
token = StopFlag() // fresh token per start a stop is permanent (like StreamPump)
// Configure the decoder's chroma + the layer's initial colorimetry before the first frame. The
// chroma subsampling drives only the decode pixel format (orthogonal to HDR/depth); the HDR
@@ -138,7 +110,7 @@ public final class Stage2Pipeline {
// decode 4:4:4 at the negotiated resolution (the HW probe clears the common case but not a
// resolution-ceiling miss). End cleanly instead of looping on a black screen.
var decodeFailRun = 0
while token.isLive {
while !token.isStopped {
do {
// Loss recovery (the primary path). The reassembler drops unrecoverable AUs and the
// decoder conceals the reference-missing deltas often WITHOUT an error callback
@@ -164,7 +136,7 @@ public final class Stage2Pipeline {
format = f // refreshed on every IDR (mode changes included)
awaitingIDR = false // a fresh IDR re-anchored decode recovery complete
}
guard let f = format, token.isLive else { continue }
guard let f = format, !token.isStopped else { continue }
if decoder.decode(au: au, format: f) {
decodeFailRun = 0
} else {
@@ -176,12 +148,12 @@ public final class Stage2Pipeline {
// ~3 s of solid failure in a 4:4:4 session (and only there a 4:2:0 loss
// recovers within a GOP) 4:4:4 isn't decodable here; end the session.
if connection.isChroma444, decodeFailRun >= 180 {
if token.isLive { onSessionEnd?() }
if !token.isStopped { onSessionEnd?() }
break
}
}
} catch {
if token.isLive { onSessionEnd?() }
if !token.isStopped { onSessionEnd?() }
break // session closed
}
}
@@ -192,19 +164,32 @@ public final class Stage2Pipeline {
thread.start()
}
/// MAIN thread, once per vsync. Present the newest ready frame (if any) and stamp capturepresent at
/// MAIN thread, once per vsync. Present the newest ready frame (if any). The latency stamps
/// use the drawable's ACTUAL on-glass instant (`addPresentedHandler`/`presentedTime` the
/// handler fires on a Metal callback thread; the meters are thread-safe), falling back to
/// `targetPresentNs` the display link's target present instant, already converted to
/// `CLOCK_REALTIME` (see `realtimeNs(forDisplayLinkTimestamp:)`).
/// `CLOCK_REALTIME` (see `realtimeNs(forDisplayLinkTimestamp:)`) when the system reports
/// no presented time (a dropped drawable). A frame that could not be rendered (no drawable
/// yet) goes back into the ring so the next tick retries it.
public func renderTick(targetPresentNs: Int64) {
guard let frame = ring.take() else { return }
guard presenter.render(frame.pixelBuffer, isHDR: frame.isHDR) else { return }
presentMeter.record(ptsNs: frame.ptsNs, atNs: targetPresentNs, offsetNs: offsetNs)
let offsetNs = offsetNs
let presentMeter = presentMeter
let presentTailMeter = presentTailMeter
let rendered = presenter.render(frame.pixelBuffer, isHDR: frame.isHDR) { presentedNs in
let atNs = presentedNs ?? targetPresentNs
presentMeter?.record(ptsNs: frame.ptsNs, atNs: atNs, offsetNs: offsetNs)
// Present tail = decode-completion on-glass. Both instants are client
// CLOCK_REALTIME, so no skew offset applies.
presentTailMeter?.record(ptsNs: UInt64(frame.decodedNs), atNs: atNs, offsetNs: 0)
}
if !rendered { ring.putBack(frame) }
}
/// Stop the pump ( one poll timeout) and drop the decode session. MAIN THREAD; idempotent. Does not
/// close the connection. A restart needs a fresh Stage2Pipeline (cancel is permanent).
/// close the connection. A restart needs a fresh Stage2Pipeline (the stop is permanent).
public func stop() {
token.cancel()
token.stop()
// Join the pump (bounded: one nextAU poll + an in-flight decode) before resetting the decoder,
// so the pump can't rebuild a session right after the reset. Only the first stop joins; a
// repeat/deinit stop skips the already-drained semaphore.
@@ -216,7 +201,7 @@ public final class Stage2Pipeline {
recovery.bind(nil) // stop requesting keyframes once the session is torn down
}
deinit { token.cancel() }
deinit { token.stop() }
/// Convert a `CADisplayLink.targetTimestamp` (CACurrentMediaTime basis) to a `CLOCK_REALTIME`
/// nanosecond instant the present clock the AU pts + skew offset live in. Projects to the target
@@ -43,7 +43,7 @@ public enum Stage444Probe {
au auBytes: [UInt8], want: OSType, fullRangeSibling: OSType
) -> Bool {
let data = Data(auBytes)
guard let format = AnnexB.formatDescription(fromIDR: data) else { return false }
guard let format = AnnexB.formatDescription(fromIDR: data, codec: .hevc) else { return false }
// Require a hardware decoder a software false-positive would make us advertise 4:4:4 and
// then decode every real frame on the CPU, blowing the latency budget.
let spec: [CFString: Any] = [
@@ -62,7 +62,7 @@ public enum Stage444Probe {
defer { VTDecompressionSessionInvalidate(session) }
let au = AccessUnit(data: data, ptsNs: 0, frameIndex: 0, flags: 0)
guard let sample = AnnexB.sampleBuffer(au: au, format: format) else { return false }
guard let sample = AnnexB.sampleBuffer(au: au, format: format, codec: .hevc) else { return false }
var produced: OSType = 0
let done = DispatchSemaphore(value: 0)
@@ -10,26 +10,10 @@ import os
private let pumpLog = Logger(subsystem: "io.unom.punktfunk", category: "video")
/// Cancellation handle owned by exactly one pump thread a restart hands the old pump
/// its own token, so it can never be revived by a newer start().
private final class PumpToken: @unchecked Sendable {
private let lock = NSLock()
private var live = true
var isLive: Bool {
lock.lock()
defer { lock.unlock() }
return live
}
func cancel() {
lock.lock()
live = false
lock.unlock()
}
}
/// One pump per instance; create a fresh StreamPump per start (cancel is permanent).
/// One pump per instance; create a fresh StreamPump per start (the stop is permanent
/// a restart hands the old pump its own token, so it can never be revived by a newer start()).
final class StreamPump {
private let token = PumpToken()
private let token = StopFlag()
/// Pump thread: pull AUs, wrap, enqueue. Non-IDR AUs before the first format
/// description are dropped. `onFrame`/`onSessionEnd` fire on the pump thread.
@@ -40,6 +24,9 @@ final class StreamPump {
onSessionEnd: (@Sendable () -> Void)?
) {
let token = token
// Coalesced host keyframe requests (100 ms throttle see KeyframeRecovery).
let recovery = KeyframeRecovery()
recovery.bind(connection)
// The layer is non-Sendable but its enqueue/flush are documented thread-safe, and after
// this point only the pump thread drives it assert that so the @Sendable Thread closure
// may capture it.
@@ -48,7 +35,6 @@ final class StreamPump {
let thread = Thread {
var format: CMVideoFormatDescription?
var lastKeyframeRequest = Date.distantPast
var lastFramesDropped = connection.framesDropped()
// Recovery is a persistent WANT, not a one-shot edge: set it on detected loss (or a
// decoder reset), retry the throttled request EVERY iteration, and clear it only when a
@@ -61,17 +47,7 @@ final class StreamPump {
var awaitingIDR = false
var awaitingSince = Date.distantPast // when the current recovery began (for the resume log)
var wasFailed = false
// Coalesced host keyframe request. 100 ms throttle (matches the working Android path):
// fast enough that a lost recovery IDR is re-requested promptly, bounded so a sustained
// freeze can't flood the control stream.
func requestKeyframeThrottled() {
let now = Date()
if now.timeIntervalSince(lastKeyframeRequest) > 0.1 {
connection.requestKeyframe()
lastKeyframeRequest = now
}
}
while token.isLive {
while !token.isStopped {
do {
// Loss recovery (the primary path). Under the host's infinite GOP the only
// recovery keyframe is one we request. The reassembler drops unrecoverable AUs
@@ -91,7 +67,7 @@ final class StreamPump {
lastFramesDropped = dropped
awaitingIDR = true
}
if awaitingIDR { requestKeyframeThrottled() }
if awaitingIDR { recovery.request() }
guard let au = try connection.nextAU(timeoutMs: 100) else { continue }
onFrame?(au)
@@ -120,11 +96,11 @@ final class StreamPump {
wasFailed = failed
guard let f = format,
let sample = AnnexB.sampleBuffer(au: au, format: f, codec: connection.videoCodec),
token.isLive // don't enqueue a stale frame after a restart
!token.isStopped // don't enqueue a stale frame after a restart
else { continue }
layer.enqueue(sample)
} catch {
if token.isLive {
if !token.isStopped {
onSessionEnd?()
}
break // session closed
@@ -138,8 +114,8 @@ final class StreamPump {
/// Stop pumping ( one poll timeout). Does not close the connection.
func stop() {
token.cancel()
token.stop()
}
deinit { token.cancel() }
deinit { token.stop() }
}
@@ -148,9 +148,9 @@ public final class VideoDecoder: @unchecked Sendable {
/// True when `newFormat` carries a PQ (SMPTE ST 2084) or HLG transfer function i.e. the host
/// is sending HDR (BT.2020). VideoToolbox populates the transfer-function extension from the
/// HEVC VUI, so this picks the decode bit depth (10-bit P010/x444 vs 8-bit NV12/444v) from the
/// stream. The present-side HDR config (colorspace/EDR/shader) is latched once per session from
/// the Welcome (`connection.isHDR`), which the host does NOT flip mid-session so this predicate
/// and that config agree for the session (a `#if DEBUG` assert in the presenter guards it).
/// stream and can flip mid-session (a game entering HDR re-inits the host encoder). The
/// presenter follows the decoded frame's resulting `isHDR`, not the Welcome's latched flag
/// (`render` reconciles the layer per frame via the idempotent `configure(hdr:)`).
static func isHDRFormat(_ format: CMVideoFormatDescription) -> Bool {
guard
let tf = CMFormatDescriptionGetExtension(
@@ -208,6 +208,11 @@ public final class VideoDecoder: @unchecked Sendable {
outputCallback: &callback,
decompressionSessionOut: &newSession)
guard status == noErr, let newSession else { return false }
// Real-time hint: schedule this session for live-streaming latency rather than
// throughput/efficiency. Best-effort decoders that don't support the property
// return an error, which is fine to ignore.
VTSessionSetProperty(
newSession, key: kVTDecompressionPropertyKey_RealTime, value: kCFBooleanTrue)
session = newSession
format = newFormat
return true
@@ -86,20 +86,22 @@ public struct StreamView: NSViewRepresentable {
private let onFrame: (@Sendable (AccessUnit) -> Void)?
private let onSessionEnd: (@Sendable () -> Void)?
private let presentMeter: LatencyMeter?
private let presentTailMeter: LatencyMeter?
/// `onFrame`/`onSessionEnd` fire on the pump thread hop to the main actor for UI.
/// `captureEnabled: false` disables input capture entirely while UI (e.g. a trust
/// prompt) is layered over the stream; flipping it to true auto-engages capture
/// once. `onCaptureChange` (main thread) reports engage/release drive the HUD's
/// "click to capture" / " releases" hint with it. `presentMeter` records capturepresent
/// when the stage-2 presenter is active (`punktfunk.presenter == "stage2"`).
/// and `presentTailMeter` decodepresent when the stage-2 presenter is active.
public init(
connection: PunktfunkConnection,
captureEnabled: Bool = true,
onCaptureChange: ((Bool) -> Void)? = nil,
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
onSessionEnd: (@Sendable () -> Void)? = nil,
presentMeter: LatencyMeter? = nil
presentMeter: LatencyMeter? = nil,
presentTailMeter: LatencyMeter? = nil
) {
self.connection = connection
self.captureEnabled = captureEnabled
@@ -107,6 +109,7 @@ public struct StreamView: NSViewRepresentable {
self.onFrame = onFrame
self.onSessionEnd = onSessionEnd
self.presentMeter = presentMeter
self.presentTailMeter = presentTailMeter
}
public func makeNSView(context: Context) -> StreamLayerView {
@@ -114,6 +117,7 @@ public struct StreamView: NSViewRepresentable {
view.onCaptureChange = onCaptureChange
view.captureEnabled = captureEnabled
view.presentMeter = presentMeter
view.presentTailMeter = presentTailMeter
view.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
return view
}
@@ -122,6 +126,7 @@ public struct StreamView: NSViewRepresentable {
view.onCaptureChange = onCaptureChange
view.captureEnabled = captureEnabled
view.presentMeter = presentMeter
view.presentTailMeter = presentTailMeter
// SwiftUI reuses the NSView across state changes repoint the pump only when the
// connection identity actually changed.
if view.connection !== connection {
@@ -136,13 +141,13 @@ public struct StreamView: NSViewRepresentable {
public final class StreamLayerView: NSView {
private let displayLayer = AVSampleBufferDisplayLayer()
private var pump: StreamPump?
/// Stage-2 presenter (default): a CAMetalLayer sublayer driven by a display link instead of the
/// StreamPump displayLayer path. nil = stage-1 (Metal-unavailable fallback / DEBUG toggle).
/// Record capturepresent / decodepresent when the stage-2 presenter is active.
/// Consulted at start().
var presentMeter: LatencyMeter?
private var stage2: Stage2Pipeline?
private var stage2Link: CADisplayLink?
private var metalLayer: CAMetalLayer?
var presentTailMeter: LatencyMeter?
/// The shared presenter stack: stage-2 (CAMetalLayer sublayer + display link) with the
/// stage-1 StreamPump displayLayer path as the Metal-unavailable / DEBUG fallback.
private let presenter = SessionPresenter()
public private(set) var connection: PunktfunkConnection?
private let cursorCapture = CursorCapture()
private var inputCapture: InputCapture?
@@ -242,16 +247,16 @@ public final class StreamLayerView: NSView {
public override func layout() {
super.layout()
attemptPendingCapture() // bounds become real here on first presentation
layoutMetalLayer() // keep the stage-2 sublayer aspect-fit to the view
layoutPresenter() // keep the stage-2 sublayer aspect-fit to the view
}
public override func setFrameSize(_ newSize: NSSize) {
super.setFrameSize(newSize)
// `layout()` isn't guaranteed on a manual-frame (no-Auto-Layout) live resize, so the
// stage-2 metal sublayer's drawableSize could stay at the old size while the view grows
// the compositor then upscales a too-small drawable and the video turns blocky. Resize the
// drawable here too so it always tracks the window's pixel size (no stale upscale).
layoutMetalLayer()
// stage-2 metal sublayer's frame could stay at the old size while the view grows
// the compositor then upscales a too-small layer and the video turns blocky. Re-fit
// here too so it always tracks the window's size (no stale upscale).
layoutPresenter()
}
// MARK: - Capture state machine
@@ -362,8 +367,7 @@ public final class StreamLayerView: NSView {
// A click is explicit intent AND may arrive mid-activation (acceptsFirstMouse:
// NSApp.isActive / isKeyWindow are still false for the click coming in from
// another app) only the auto-engage paths require already-held key status.
// `connection != nil` (not `pump`) is the session-active gate the stage-2 presenter
// runs without a StreamPump, and capture must still engage there.
// `connection != nil` is the session-active gate (presenter internals are opaque here).
guard captureEnabled, !captured, connection != nil, window != nil,
fromClick || (NSApp.isActive && window?.isKeyWindow == true)
else { return }
@@ -483,8 +487,10 @@ public final class StreamLayerView: NSView {
let u = (p.x - fit.minX) / fit.width
let v = (p.y - videoMinYTop) / fit.height
guard u >= 0, u <= 1, v >= 0, v <= 1 else { return nil } // letterbox bars
let hx = Int32((u * CGFloat(mode.width)).rounded().clamped(0, CGFloat(mode.width - 1)))
let hy = Int32((v * CGFloat(mode.height)).rounded().clamped(0, CGFloat(mode.height - 1)))
let hx = Int32((u * CGFloat(mode.width)).rounded()
.clamped(to: 0...CGFloat(mode.width - 1)))
let hy = Int32((v * CGFloat(mode.height)).rounded()
.clamped(to: 0...CGFloat(mode.height - 1)))
return HostPoint(x: hx, y: hy, w: mode.width, h: mode.height)
}
@@ -507,10 +513,10 @@ public final class StreamLayerView: NSView {
DispatchQueue.main.async { onCaptureChange(captured) }
}
// MARK: - Pump
// MARK: - Session start/stop
/// Pump thread: pull AUs from the connection, wrap, enqueue. The first IDR yields the
/// format description; non-IDR AUs before it are dropped (the host opens with an IDR).
/// Wire up input capture and start the presenter (see SessionPresenter for the
/// stage-2/stage-1 choice). `onFrame` fires per AU at receipt; `onSessionEnd` on close.
public func start(
connection: PunktfunkConnection,
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
@@ -558,90 +564,31 @@ public final class StreamLayerView: NSView {
cursorVisible = false
_ = connection.resolvedCompositor // (was: Auto gamescope; kept to document intent)
// Presenter choice stage-2 is the DEFAULT (explicit VTDecompressionSession decode + a
// CAMetalLayer/display-link present): it can detect + recover a wedged decoder where
// stage-1's AVSampleBufferDisplayLayer freezes hard on a lost HEVC reference. Stage-1 is
// reachable only via the DEBUG presenter toggle; release always takes stage-2 (the stage-1
// pump below stays the automatic fallback if Metal is missing).
#if DEBUG
let forceStage1 = UserDefaults.standard.string(forKey: DefaultsKey.presenter) == "stage1"
#else
let forceStage1 = false
#endif
if !forceStage1,
let meter = presentMeter,
let pipeline = Stage2Pipeline(presentMeter: meter) {
startStage2(pipeline, connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
} else {
let pump = StreamPump()
pump.start(
connection: connection, layer: displayLayer,
onFrame: onFrame, onSessionEnd: onSessionEnd)
self.pump = pump
}
// Presenter choice + lifecycle live in SessionPresenter (shared with iOS/tvOS): stage-2
// (explicit VTDecompressionSession decode + a CAMetalLayer/display-link present) by
// default, the stage-1 pump as the Metal-missing / DEBUG fallback. The link comes from
// NSView.displayLink so it tracks the display this view is on.
presenter.start(
connection: connection,
baseLayer: displayLayer,
presentMeter: presentMeter,
presentTailMeter: presentTailMeter,
makeDisplayLink: { displayLink(target: $0, selector: $1) },
onFrame: onFrame,
onSessionEnd: onSessionEnd)
layoutPresenter()
requestAutoCapture() // entering a session is the deliberate "capture me" moment
}
// MARK: - Stage-2 presenter (VTDecompressionSession CAMetalLayer + display link)
private func startStage2(
_ pipeline: Stage2Pipeline, connection: PunktfunkConnection,
onFrame: (@Sendable (AccessUnit) -> Void)?, onSessionEnd: (@Sendable () -> Void)?
) {
let metal = pipeline.layer
// The opaque metal layer composites OVER the AVSampleBufferDisplayLayer base, which sits
// idle (un-enqueued) in stage-2. contentsScale + frame are set in layoutMetalLayer().
displayLayer.addSublayer(metal)
metalLayer = metal
stage2 = pipeline
layoutMetalLayer()
// Weak-proxy target so the link doesn't form a retain cycle with the view (see
// DisplayLinkProxy) the link retains the proxy; the proxy holds the view weakly.
let proxy = DisplayLinkProxy { [weak self] link in self?.stage2Tick(link) }
let link = displayLink(target: proxy, selector: #selector(DisplayLinkProxy.tick(_:)))
link.add(to: .main, forMode: .common)
stage2Link = link
pipeline.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
}
private func stage2Tick(_ link: CADisplayLink) {
stage2?.renderTick(
targetPresentNs: Stage2Pipeline.realtimeNs(forDisplayLinkTimestamp: link.targetTimestamp))
}
/// Position the metal sublayer aspect-fit in the view (the host streams at the client's native
/// mode, so this is usually the full bounds; it letterboxes a resized window). Only the layer
/// FRAME is set here the presenter sizes the drawable to the decoded frame and the layer's
/// contentsGravity (.resizeAspect) scales it to this frame via the system compositor, so a
/// resized window rescales through the system's filter (matching stage-1) instead of the shader.
private func layoutMetalLayer() {
guard let metalLayer, let connection else { return }
let mode = connection.currentMode()
let fit: NSRect = (mode.width > 0 && mode.height > 0)
? AVMakeRect(
aspectRatio: CGSize(width: Int(mode.width), height: Int(mode.height)),
insideRect: bounds)
: bounds
// No implicit resize animation; refresh contentsScale on a retinanon-retina move.
CATransaction.begin()
CATransaction.setDisableActions(true)
metalLayer.contentsScale = window?.backingScaleFactor ?? 1
metalLayer.frame = fit
CATransaction.commit()
/// Aspect-fit the stage-2 metal sublayer to the view; refresh contentsScale on a
/// retinanon-retina move (see SessionPresenter.layout).
private func layoutPresenter() {
presenter.layout(in: bounds, contentsScale: window?.backingScaleFactor ?? 1)
}
public override func viewDidChangeBackingProperties() {
super.viewDidChangeBackingProperties()
layoutMetalLayer() // backing scale changed (e.g. moved to a non-retina display)
}
private func teardownStage2() {
stage2Link?.invalidate()
stage2Link = nil
stage2?.stop() // stops the pump (synchronous join) + drops the decode session
stage2 = nil
metalLayer?.removeFromSuperlayer()
metalLayer = nil
layoutPresenter() // backing scale changed (e.g. moved to a non-retina display)
}
/// Stop pumping ( one poll timeout). Does not close the connection that stays with
@@ -651,9 +598,7 @@ public final class StreamLayerView: NSView {
removeMouseMonitor() // belt-and-suspenders: releaseCapture no-ops if not captured
inputCapture?.stop()
inputCapture = nil
pump?.stop()
pump = nil
teardownStage2()
presenter.stop()
connection = nil
}
@@ -661,16 +606,7 @@ public final class StreamLayerView: NSView {
removeMouseMonitor()
appObservers.forEach(NotificationCenter.default.removeObserver(_:))
windowObservers.forEach(NotificationCenter.default.removeObserver(_:))
pump?.stop()
teardownStage2() // invalidate the display link + stop the pipeline if stop() was missed
}
}
extension CGFloat {
/// Clamp into a [lo, hi] range keeps the absolute-cursor mapping inside the host's
/// pixel bounds even if a stray event reports a point a hair past the video rect.
fileprivate func clamped(_ lo: CGFloat, _ hi: CGFloat) -> CGFloat {
Swift.min(Swift.max(self, lo), hi)
presenter.stop() // invalidate the display link + stop the pipeline if stop() was missed
}
}
#endif
@@ -51,6 +51,7 @@ public struct StreamView: UIViewControllerRepresentable {
private let onFrame: (@Sendable (AccessUnit) -> Void)?
private let onSessionEnd: (@Sendable () -> Void)?
private let presentMeter: LatencyMeter?
private let presentTailMeter: LatencyMeter?
public init(
connection: PunktfunkConnection,
@@ -58,7 +59,8 @@ public struct StreamView: UIViewControllerRepresentable {
onCaptureChange: ((Bool) -> Void)? = nil,
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
onSessionEnd: (@Sendable () -> Void)? = nil,
presentMeter: LatencyMeter? = nil
presentMeter: LatencyMeter? = nil,
presentTailMeter: LatencyMeter? = nil
) {
self.connection = connection
self.captureEnabled = captureEnabled
@@ -66,6 +68,7 @@ public struct StreamView: UIViewControllerRepresentable {
self.onFrame = onFrame
self.onSessionEnd = onSessionEnd
self.presentMeter = presentMeter
self.presentTailMeter = presentTailMeter
}
public func makeUIViewController(context: Context) -> StreamViewController {
@@ -73,6 +76,7 @@ public struct StreamView: UIViewControllerRepresentable {
controller.onCaptureChange = onCaptureChange
controller.captureEnabled = captureEnabled
controller.presentMeter = presentMeter
controller.presentTailMeter = presentTailMeter
controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
return controller
}
@@ -81,6 +85,7 @@ public struct StreamView: UIViewControllerRepresentable {
controller.onCaptureChange = onCaptureChange
controller.captureEnabled = captureEnabled
controller.presentMeter = presentMeter
controller.presentTailMeter = presentTailMeter
if controller.connection !== connection {
controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
}
@@ -95,14 +100,14 @@ public struct StreamView: UIViewControllerRepresentable {
public final class StreamViewController: UIViewController {
public private(set) var connection: PunktfunkConnection?
private var pump: StreamPump?
private var observers: [NSObjectProtocol] = []
/// Stage-2 presenter (default): a CAMetalLayer sublayer driven by a CADisplayLink instead of the
/// StreamPump displayLayer path. nil = stage-1 (Metal-unavailable fallback / DEBUG toggle).
/// Record capturepresent / decodepresent when the stage-2 presenter is active.
/// Consulted at start().
var presentMeter: LatencyMeter?
private var stage2: Stage2Pipeline?
private var stage2Link: CADisplayLink?
private var metalLayer: CAMetalLayer?
var presentTailMeter: LatencyMeter?
/// The shared presenter stack: stage-2 (CAMetalLayer sublayer + display link) with the
/// stage-1 StreamPump displayLayer path as the Metal-unavailable / DEBUG fallback.
private let presenter = SessionPresenter()
#if os(iOS)
private var inputCapture: InputCapture?
fileprivate var captured = false
@@ -274,27 +279,18 @@ public final class StreamViewController: UIViewController {
inputCapture = capture
#endif
// Presenter choice stage-2 is the DEFAULT (VTDecompressionSession decode + a
// CAMetalLayer/display-link present): it can detect + recover a wedged decoder, where
// stage-1's AVSampleBufferDisplayLayer freezes hard on a lost HEVC reference frame with no
// way to recover. Stage-1 is reachable only via the DEBUG presenter toggle; release always
// takes stage-2 (the stage-1 pump below stays the automatic fallback if Metal is missing).
#if DEBUG
let forceStage1 = UserDefaults.standard.string(forKey: DefaultsKey.presenter) == "stage1"
#else
let forceStage1 = false
#endif
if !forceStage1,
let meter = presentMeter,
let pipeline = Stage2Pipeline(presentMeter: meter) {
startStage2(pipeline, connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
} else {
let pump = StreamPump()
pump.start(
connection: connection, layer: streamView.displayLayer,
onFrame: onFrame, onSessionEnd: onSessionEnd)
self.pump = pump
}
// Presenter choice + lifecycle live in SessionPresenter (shared with macOS): stage-2
// (explicit VTDecompressionSession decode + a CAMetalLayer/display-link present) by
// default, the stage-1 pump as the Metal-missing / DEBUG fallback.
presenter.start(
connection: connection,
baseLayer: streamView.displayLayer,
presentMeter: presentMeter,
presentTailMeter: presentTailMeter,
makeDisplayLink: { CADisplayLink(target: $0, selector: $1) },
onFrame: onFrame,
onSessionEnd: onSessionEnd)
layoutMetalLayer()
#if os(iOS)
// GC only delivers while active; everything held is flushed by InputCapture's
@@ -349,39 +345,10 @@ public final class StreamViewController: UIViewController {
streamView.onScroll = nil
streamView.currentHostMode = nil
#endif
pump?.stop()
pump = nil
teardownStage2()
presenter.stop()
connection = nil
}
// MARK: - Stage-2 presenter (VTDecompressionSession CAMetalLayer + display link)
private func startStage2(
_ pipeline: Stage2Pipeline, connection: PunktfunkConnection,
onFrame: (@Sendable (AccessUnit) -> Void)?, onSessionEnd: (@Sendable () -> Void)?
) {
let metal = pipeline.layer
// Composites OVER the idle (un-enqueued in stage-2) AVSampleBufferDisplayLayer base.
// (contentsScale + frame are set by layoutMetalLayer() just below.)
streamView.layer.addSublayer(metal)
metalLayer = metal
stage2 = pipeline
layoutMetalLayer()
// Weak-proxy target so the link doesn't retain-cycle with the controller (see
// DisplayLinkProxy) the link retains the proxy; the proxy holds self weakly.
let proxy = DisplayLinkProxy { [weak self] link in self?.stage2Tick(link) }
let link = CADisplayLink(target: proxy, selector: #selector(DisplayLinkProxy.tick(_:)))
link.add(to: .main, forMode: .common)
stage2Link = link
pipeline.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
}
private func stage2Tick(_ link: CADisplayLink) {
stage2?.renderTick(
targetPresentNs: Stage2Pipeline.realtimeNs(forDisplayLinkTimestamp: link.targetTimestamp))
}
public override func viewDidLayoutSubviews() {
super.viewDidLayoutSubviews()
layoutMetalLayer()
@@ -397,40 +364,16 @@ public final class StreamViewController: UIViewController {
return s > 0 ? s : UIScreen.main.scale
}
/// Position the metal sublayer aspect-fit in the view (the host streams at the client's native
/// mode, so this is usually the full bounds). Only the layer FRAME is set here the presenter
/// sizes the drawable to the decoded frame and the layer's contentsGravity (.resizeAspect)
/// scales it to this frame via the system compositor (matching stage-1's videoGravity).
/// Aspect-fit the stage-2 metal sublayer to the view at the canonical render scale
/// (see SessionPresenter.layout).
private func layoutMetalLayer() {
guard let metalLayer, let connection else { return }
let mode = connection.currentMode()
let bounds = streamView.bounds
let fit: CGRect = (mode.width > 0 && mode.height > 0)
? AVMakeRect(
aspectRatio: CGSize(width: Int(mode.width), height: Int(mode.height)),
insideRect: bounds)
: bounds
CATransaction.begin()
CATransaction.setDisableActions(true) // don't animate the resize
metalLayer.contentsScale = renderScale
metalLayer.frame = fit
CATransaction.commit()
}
private func teardownStage2() {
stage2Link?.invalidate()
stage2Link = nil
stage2?.stop() // stops the pump (synchronous join) + drops the decode session
stage2 = nil
metalLayer?.removeFromSuperlayer()
metalLayer = nil
presenter.layout(in: streamView.bounds, contentsScale: renderScale)
}
#if os(iOS)
private func setCaptured(_ on: Bool) {
if on {
// `connection != nil` (not `pump`) is the session-active gate the stage-2 presenter
// runs without a StreamPump.
// `connection != nil` is the session-active gate (presenter internals are opaque here).
guard captureEnabled, !captured, connection != nil else { return }
inputCapture?.setForwarding(true)
captured = true
@@ -476,8 +419,7 @@ public final class StreamViewController: UIViewController {
deinit {
observers.forEach(NotificationCenter.default.removeObserver(_:))
pump?.stop()
teardownStage2() // invalidate the display link + stop the pipeline if stop() was missed
presenter.stop() // invalidate the display link + stop the pipeline if stop() was missed
}
}
@@ -675,12 +617,4 @@ final class StreamLayerUIView: UIView {
}
#endif
}
#if os(iOS)
extension CGFloat {
fileprivate func clamped(to range: ClosedRange<CGFloat>) -> CGFloat {
Swift.min(Swift.max(self, range.lowerBound), range.upperBound)
}
}
#endif
#endif