diff --git a/clients/apple/README.md b/clients/apple/README.md index c7c4fed2..27da0c40 100644 --- a/clients/apple/README.md +++ b/clients/apple/README.md @@ -22,8 +22,9 @@ Opus audio, cert pinning — lives in the shared Rust **`punktfunk-core`** (stat - **Full controller support** — one selected controller forwarded as pad 0, including **DualSense** feedback (rumble → CoreHaptics, lightbar, player LEDs, adaptive triggers) and touchpad/motion. The virtual pad type auto-resolves from your physical controller. -- **Mouse & keyboard** — `GCMouse`/`GCKeyboard` capture with click-to-capture and a ⌘⎋ release, plus - iPad pointer lock and touch input. +- **Mouse & keyboard** — `GCMouse`/`GCKeyboard` capture with click-to-capture and a ⌃⌥⇧Q release + (the cross-client Ctrl+Alt+Shift+Q; ⌘⎋ still works as the macOS/iPad toggle), plus iPad pointer + lock and touch input. - **Find hosts automatically** — mDNS discovery (`NWBrowser` over `_punktfunk._udp`); first connect does a one-time **SPAKE2 PIN pairing** (or TOFU on trusted LANs), then reconnects on a pinned, Keychain-stored identity. @@ -83,7 +84,9 @@ PUNKTFUNK_AUTOCONNECT= PUNKTFUNK_MODE=1280x720x60 swift run PunktfunkCli - **`PunktfunkClient`** (the app) — hosts grid with an *On this network* section, add-host sheet, the two trust flows (TOFU prompt + SPAKE2 `PairSheet`), the stream view with the HUD, a tabbed Settings pane (General / Display / Audio / Controllers / Advanced), and the network speed - test. A Scene-level **Stream** menu carries Disconnect (⌘D) and the HUD toggle (⌘⇧S). + test. A Scene-level **Stream** menu carries the cross-client shortcut set: Release Mouse (⌃⌥⇧Q), + Disconnect (⌃⌥⇧D) and the HUD toggle (⌃⌥⇧S) — the same Ctrl+Alt+Shift combos the Windows and + Linux clients reserve, also shown on a 6-second banner at stream start. On iOS/iPadOS **and macOS** a connected controller swaps the whole home for the **gamepad UI** (`Home/Gamepad*`, `Settings/GamepadSettingsView`): a console-style host carousel (A connect · Y library · X settings), a controller-navigable settings screen, an add-host flow with an diff --git a/clients/apple/Sources/PunktfunkClient/ContentView.swift b/clients/apple/Sources/PunktfunkClient/ContentView.swift index f4937733..9f0b8c56 100644 --- a/clients/apple/Sources/PunktfunkClient/ContentView.swift +++ b/clients/apple/Sources/PunktfunkClient/ContentView.swift @@ -55,6 +55,18 @@ struct ContentView: View { /// Wakes a sleeping host and waits for it to come back online before connecting (drives the /// "Waking…" overlay). macOS-only in practice — WoL is gated off on iOS/tvOS. @StateObject private var waker = HostWaker() + #if os(macOS) + /// Whether the hosting window is native-fullscreen right now (reported by + /// FullscreenController). Drives the session view's safe-area choice: fullscreen goes + /// edge-to-edge (behind the notch); windowed respects the top inset so the title bar + /// never covers the video. + @State private var isFullscreen = false + /// Shows the start-of-stream shortcut banner (the Windows client's discoverability + /// pattern): raised on every transition to `.streaming`, dropped by the banner's own + /// 6-second task. Independent of the stats HUD so the keys are discoverable even with + /// statistics off. + @State private var showShortcutHint = false + #endif #if !os(macOS) @State private var showSettings = false #endif @@ -89,6 +101,9 @@ struct ContentView: View { .onChange(of: model.phase) { _, phase in switch phase { case .streaming: + #if os(macOS) + showShortcutHint = true // the 6 s shortcut banner, per session start + #endif // A session actually started — remember it on the card ("Connected … ago" // plus the accent ring on the most recent host). guard let host = model.activeHost else { break } @@ -115,7 +130,7 @@ struct ContentView: View { } } .onDisappear { model.disconnect() } // window closed mid-session (Cmd+N spawns more) - // Expose the session to the Scene-level Stream menu (Disconnect ⌘D works even when + // Expose the session to the Scene-level Stream menu (Disconnect ⌃⌥⇧D works even when // the HUD is hidden). tvOS has no such menu. #if !os(tvOS) .focusedSceneValue(\.sessionFocus, SessionFocus( @@ -125,7 +140,12 @@ struct ContentView: View { #if os(macOS) // Fullscreen only while a session is up (incl. the trust prompt over the blurred stream), // windowed on the host list — so the picker isn't forced fullscreen. Opt-out in Settings. - .background(FullscreenController(active: fullscreenWhileStreaming && model.connection != nil)) + // The controller also reports the window's ACTUAL fullscreen state back into + // `isFullscreen` (the user can toggle it manually), which drives the session view's + // safe-area handling below. + .background(FullscreenController( + active: fullscreenWhileStreaming && model.connection != nil, + isFullscreen: $isFullscreen)) #endif // On the outer Group so the sheet survives the trust-prompt → home transition // (the "Pair with PIN instead" path disconnects first — the host's accept loop @@ -300,13 +320,17 @@ struct ContentView: View { #if os(macOS) .frame(minWidth: 640, minHeight: 360) .background(Color.black) - // Fill the whole display in fullscreen, INCLUDING behind the camera housing (notch). + // FULLSCREEN fills the whole display, INCLUDING behind the camera housing (notch). // Without this the stream is laid out in the safe area below the notch, so an // aspect-fit video at the display's native mode scales down and leaves black borders. // A fullscreen video behind the notch (a thin top-center strip occluded) is the - // expected behavior — same edge-to-edge intent as the iOS/tvOS branches below. Inert - // in windowed mode (no notch safe-area inset on a titled window). - .ignoresSafeArea() + // expected behavior — same edge-to-edge intent as the iOS/tvOS branches below. + // WINDOWED keeps the TOP inset: macOS 26 windows extend content under the (glass) + // title bar and report its height as top safe area — ignoring it there put the top of + // the video (and the HUD) underneath the title bar. The black `.background` above is a + // ShapeStyle background, which always extends under every inset, so the strip behind + // the title bar stays black rather than showing the video. + .ignoresSafeArea(edges: isFullscreen ? .all : [.horizontal, .bottom]) #elseif os(iOS) // Streaming is immersive: edge-to-edge under the status bar and home // indicator, both hidden for the session (they return with the hosts grid). @@ -335,6 +359,9 @@ struct ContentView: View { onCaptureChange: { [weak model] captured in model?.mouseCaptured = captured }, + onDisconnectRequest: { [weak model] in + model?.disconnect() // the captured-state ⌃⌥⇧D combo + }, onFrame: { [meter = model.meter, latency = model.latency, split = model.latencySplit, offset = conn.clockOffsetNs] au in meter.note(byteCount: au.data.count) @@ -356,6 +383,30 @@ struct ContentView: View { StreamHUDView(model: model, connection: conn, placement: placement) } } + #if os(macOS) + // The start-of-stream shortcut banner (Windows-client parity): the full + // reserved key set on a glass pill, bottom-centre, for the first 6 seconds of + // every session — independent of the stats HUD, so the keys are discoverable + // even with statistics off. The banner's own task drops it (cancelled cleanly + // if the session view goes away first). + .overlay(alignment: .bottom) { + if captureEnabled && showShortcutHint { + Text("Click the stream to capture · ⌃⌥⇧Q releases the mouse · " + + "⌃⌥⇧D disconnects · ⌃⌥⇧S stats") + .font(.geist(12, relativeTo: .caption)) + .foregroundStyle(.secondary) + .padding(.horizontal, 14) + .padding(.vertical, 8) + .glassBackground(Capsule()) + .padding(.bottom, 24) + .transition(.opacity) + .task { + try? await Task.sleep(for: .seconds(6)) + withAnimation(.easeOut(duration: 0.6)) { showShortcutHint = false } + } + } + } + #endif #if os(iOS) // Touch users have no menu / ⌘D, so when the HUD (and its Disconnect button) // is hidden, keep a minimal always-reachable exit in a corner. It rides a @@ -654,23 +705,62 @@ struct ContentView: View { } #if os(macOS) -/// Drives the hosting window in/out of native fullscreen from SwiftUI state. Mounted invisibly in -/// the view tree; on each `active` change it captures the window and toggles fullscreen only when -/// the current state differs (so it never fights a toggle already in flight, and never touches a -/// window the user fullscreened manually unless `active` says otherwise). +/// Drives the hosting window in/out of native fullscreen from SwiftUI state, and mirrors the +/// window's ACTUAL fullscreen state back into `isFullscreen` (the user can also toggle it with the +/// green button / ⌃⌘F — ContentView keys the session view's safe-area handling off the real state, +/// not the setting). Mounted invisibly in the view tree; on each `active` change it captures the +/// window and toggles fullscreen only when the current state differs (so it never fights a toggle +/// already in flight, and never touches a window the user fullscreened manually unless `active` +/// says otherwise). private struct FullscreenController: NSViewRepresentable { let active: Bool + @Binding var isFullscreen: Bool + + /// Holds the window's fullscreen-transition observers so they're rebound on a window change + /// and removed on dismantle. + final class Coordinator { + var observers: [NSObjectProtocol] = [] + weak var observedWindow: NSWindow? + deinit { observers.forEach(NotificationCenter.default.removeObserver(_:)) } + } + + func makeCoordinator() -> Coordinator { Coordinator() } func makeNSView(context: Context) -> NSView { NSView() } func updateNSView(_ view: NSView, context: Context) { let want = active + let isFullscreen = $isFullscreen + let coordinator = context.coordinator DispatchQueue.main.async { guard let window = view.window else { return } + observeTransitions(of: window, coordinator: coordinator) let isFull = window.styleMask.contains(.fullScreen) + if isFullscreen.wrappedValue != isFull { isFullscreen.wrappedValue = isFull } if want != isFull { window.toggleFullScreen(nil) } } } + + /// `willEnter` (not did) so the video goes edge-to-edge while the title bar is already + /// animating away; `didExit` so the top inset returns only once the title bar is back — + /// no black gap in either direction. + private func observeTransitions(of window: NSWindow, coordinator: Coordinator) { + guard coordinator.observedWindow !== window else { return } + coordinator.observers.forEach(NotificationCenter.default.removeObserver(_:)) + coordinator.observers.removeAll() + coordinator.observedWindow = window + let isFullscreen = $isFullscreen + for (name, value) in [ + (NSWindow.willEnterFullScreenNotification, true), + (NSWindow.didExitFullScreenNotification, false), + ] { + coordinator.observers.append(NotificationCenter.default.addObserver( + forName: name, object: window, queue: .main + ) { _ in + isFullscreen.wrappedValue = value + }) + } + } } #endif diff --git a/clients/apple/Sources/PunktfunkClient/PunktfunkClientApp.swift b/clients/apple/Sources/PunktfunkClient/PunktfunkClientApp.swift index a723b46d..002af27a 100644 --- a/clients/apple/Sources/PunktfunkClient/PunktfunkClientApp.swift +++ b/clients/apple/Sources/PunktfunkClient/PunktfunkClientApp.swift @@ -43,8 +43,9 @@ struct PunktfunkClientApp: App { // form row labels; views that pick an explicit size/weight use `.geist(…)` directly. .font(.geist(17, relativeTo: .body)) } - // The Stream menu (Disconnect ⌘D, Show/Hide Statistics ⌘⇧S) — a real menu bar on - // macOS, hardware-keyboard shortcuts on iPad. tvOS has neither. + // The Stream menu (Release Mouse ⌃⌥⇧Q, Disconnect ⌃⌥⇧D, Show/Hide Statistics ⌃⌥⇧S — + // the cross-client Ctrl+Alt+Shift set) — a real menu bar on macOS, hardware-keyboard + // shortcuts on iPad. tvOS has neither. #if !os(tvOS) .commands { StreamCommands() } #endif diff --git a/clients/apple/Sources/PunktfunkClient/Session/StreamCommands.swift b/clients/apple/Sources/PunktfunkClient/Session/StreamCommands.swift index d27db624..013b2263 100644 --- a/clients/apple/Sources/PunktfunkClient/Session/StreamCommands.swift +++ b/clients/apple/Sources/PunktfunkClient/Session/StreamCommands.swift @@ -1,6 +1,11 @@ // The app's "Stream" menu (macOS menu bar + iPad hardware-keyboard shortcuts). These live at -// the Scene level so they keep working when the HUD overlay is hidden — in particular ⌘D -// disconnect, which used to be reachable only via the HUD's button. The toggle just flips the +// the Scene level so they keep working when the HUD overlay is hidden. The shortcuts are the +// CROSS-CLIENT set every punktfunk client reserves — Ctrl+Alt+Shift+Q (release the captured +// mouse) / +D (disconnect) / +S (stats) — and the menu is their discoverable surface on macOS +// (the Linux client has its GTK Shortcuts window, Windows its start-of-stream banner). While +// input is CAPTURED these key equivalents never reach the menu (the stream view swallows +// keys); InputCapture's monitor detects the same combos there and performs the same actions — +// the menu covers the released state and discoverability. The stats toggle just flips the // shared `hudEnabled` setting; ContentView reads the same @AppStorage and reacts. // // tvOS has no menu bar / hardware-keyboard command surface (disconnect there is the Siri @@ -38,10 +43,19 @@ struct StreamCommands: Commands { Button(hudEnabled ? "Hide Statistics" : "Show Statistics") { hudEnabled.toggle() } - .keyboardShortcut("s", modifiers: [.command, .shift]) + .keyboardShortcut("s", modifiers: [.control, .option, .shift]) + // Reaches the key window's stream view via NotificationCenter — capture is view + // state the Scene can't touch directly. (Captured, the combo is handled by + // InputCapture's monitor before menus see it; this item is the released-state + // path and the shortcut's menu-bar documentation.) + Button("Release Mouse") { + NotificationCenter.default.post(name: .punktfunkReleaseCapture, object: nil) + } + .keyboardShortcut("q", modifiers: [.control, .option, .shift]) + .disabled(session?.isStreaming != true) Divider() Button("Disconnect") { session?.disconnect() } - .keyboardShortcut("d", modifiers: .command) + .keyboardShortcut("d", modifiers: [.control, .option, .shift]) .disabled(session?.isStreaming != true) } } diff --git a/clients/apple/Sources/PunktfunkClient/Session/StreamHUDView.swift b/clients/apple/Sources/PunktfunkClient/Session/StreamHUDView.swift index 22f4d6d7..e4793da0 100644 --- a/clients/apple/Sources/PunktfunkClient/Session/StreamHUDView.swift +++ b/clients/apple/Sources/PunktfunkClient/Session/StreamHUDView.swift @@ -64,10 +64,11 @@ struct StreamHUDView: View { .foregroundStyle(.secondary) } // While captured the cursor is hidden+frozen, so the button is keyboard-only - // (⌘⎋ or Cmd+Tab release the cursor; released, it's clickable again). + // (⌃⌥⇧Q — the cross-client Ctrl+Alt+Shift+Q — or ⌘⎋/Cmd+Tab release the cursor; + // released, it's clickable again). #if os(macOS) Text(model.mouseCaptured - ? "⌘⎋ releases the mouse" + ? "⌃⌥⇧Q releases the mouse" : "Click the stream to capture input") .font(.geist(11, relativeTo: .caption2)) .foregroundStyle(.secondary) @@ -87,10 +88,16 @@ struct StreamHUDView: View { .font(.geist(12, relativeTo: .caption)) .foregroundStyle(.secondary) #else - // ⌘D lives on the app's Stream menu (so it still works when the HUD is hidden); - // this button is the in-overlay, click-to-disconnect affordance. - Button("Disconnect (⌘D)") { model.disconnect() } + // ⌃⌥⇧D lives on the app's Stream menu (so it still works when the HUD is hidden) + // and in InputCapture's monitor while captured; this button is the in-overlay, + // click-to-disconnect affordance. + #if os(macOS) + Button("Disconnect (⌃⌥⇧D)") { model.disconnect() } .font(.geist(12, relativeTo: .caption)) + #else + Button("Disconnect") { model.disconnect() } + .font(.geist(12, relativeTo: .caption)) + #endif #endif } .padding(10) diff --git a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift index baa2ef0f..96ecda72 100644 --- a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift +++ b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift @@ -306,6 +306,25 @@ extension SettingsView { #endif } + // macOS-only: iOS/tvOS layers always present on the display's vsync, so the choice only + // exists on the Mac (where the layer's own sync must stay off — see MetalVideoPresenter). + @ViewBuilder var vsyncSection: some View { + #if os(macOS) + Section { + Toggle("V-Sync", isOn: $vsync) + } header: { + Text("Presentation") + } footer: { + Text("Off (default): each frame is shown as soon as it's ready — lowest latency, " + + "but frame timing can look uneven and fullscreen may tear. On: frames flip " + + "in step with the display's refresh — evenly paced, up to one refresh of " + + "added latency. Applies from the next session.") + .font(.geist(12, relativeTo: .caption)) + .foregroundStyle(.secondary) + } + #endif + } + // Stage-2 (Metal/VTDecompressionSession) is the default and only user-visible presenter — it // recovers from a wedged decoder, where stage-1's AVSampleBufferDisplayLayer freezes hard on a // lost HEVC reference. Stage-1 is kept reachable as a DEBUG-only override for diagnostics, like diff --git a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift index d66ed1d8..7ea6b83c 100644 --- a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift +++ b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift @@ -25,6 +25,9 @@ struct SettingsView: View { @AppStorage(DefaultsKey.gamepadType) var gamepadType = 0 @AppStorage(DefaultsKey.bitrateKbps) var bitrateKbps = 0 @AppStorage(DefaultsKey.presenter) var presenter = "stage2" + #if os(macOS) + @AppStorage(DefaultsKey.vsync) var vsync = false + #endif @AppStorage(DefaultsKey.hdrEnabled) var hdrEnabled = true @AppStorage(DefaultsKey.enable444) var enable444 = true @AppStorage(DefaultsKey.libraryEnabled) var libraryEnabled = false @@ -106,6 +109,7 @@ struct SettingsView: View { Form { presenterSection hdrSection + vsyncSection windowSection statisticsSection } diff --git a/clients/apple/Sources/PunktfunkKit/Input/InputCapture.swift b/clients/apple/Sources/PunktfunkKit/Input/InputCapture.swift index dfaa89f4..e1e9fd57 100644 --- a/clients/apple/Sources/PunktfunkKit/Input/InputCapture.swift +++ b/clients/apple/Sources/PunktfunkKit/Input/InputCapture.swift @@ -24,8 +24,9 @@ // // Forwarding is gated by `forwarding` (driven by StreamLayerView's capture state): the // handlers stay attached for the whole session, but while the user has released capture -// (⌘⎋, focus loss) nothing reaches the host and key events travel the responder chain -// normally. Everything held is flushed host-side on each transition to released. +// (⌃⌥⇧Q — the cross-client Ctrl+Alt+Shift+Q — or ⌘⎋, focus loss) nothing reaches the host +// and key events travel the responder chain normally. Everything held is flushed host-side +// on each transition to released. // // GCMouse.current/GCKeyboard.coalesced are process-global singletons with one handler // slot each: only one InputCapture can be live per process. `activeCapture` tracks @@ -108,6 +109,16 @@ public final class InputCapture { /// the absolute-vs-relative forwarding lives entirely in StreamLayerView. Main queue. public var onToggleCursor: (() -> Void)? + /// The cross-client combos (Windows/Linux parity: Ctrl+Alt+Shift+Q/D/S), fired from the macOS + /// keyDown monitor only WHILE FORWARDING — that's the state in which the app's menu (which + /// carries the same key equivalents for discoverability) can't see them, so the monitor is the + /// captured-state delivery path; released, the events pass through and the menu handles them. + /// ⌃⌥⇧Q releases the captured mouse/keyboard; ⌃⌥⇧D disconnects; ⌃⌥⇧S toggles the stats + /// overlay. Main queue. + public var onReleaseCapture: (() -> Void)? + public var onDisconnect: (() -> Void)? + public var onToggleStats: (() -> Void)? + /// Fired when a newer InputCapture takes the process-global GC handler slots (the /// singletons hold ONE handler each): the preempted owner must drop its capture /// state — its handlers are gone, so it would otherwise sit "captured" with dead @@ -215,6 +226,32 @@ public final class InputCapture { self.onToggleCursor?() return nil } + // The cross-client combos (Ctrl+Alt+Shift+Q/D/S — the same set every other + // punktfunk client reserves), intercepted only while forwarding so the host never + // sees the letter (the ⌃⌥⇧ modifiers were already forwarded as they went down; + // they're flushed by the release path / released by the user as usual). The letter + // is latched (suppressedVK) so its keyUp doesn't leak to the host either. While + // NOT forwarding the events pass through and the menu's identical key equivalents + // handle them (with the standard menu-flash feedback). keyCodes are kVK_ANSI_* — + // physical positions, layout-independent. + if self.forwarding, flags == [.control, .option, .shift] { + switch event.keyCode { + case 12 /* Q */: + self.suppressedVK = 0x51 + self.onReleaseCapture?() + return nil + case 2 /* D */: + self.suppressedVK = 0x44 + self.onDisconnect?() + return nil + case 1 /* S */: + self.suppressedVK = 0x53 + self.onToggleStats?() + return nil + default: + break + } + } return event } #endif diff --git a/clients/apple/Sources/PunktfunkKit/Support/DefaultsKeys.swift b/clients/apple/Sources/PunktfunkKit/Support/DefaultsKeys.swift index 94656ae2..47dc1091 100644 --- a/clients/apple/Sources/PunktfunkKit/Support/DefaultsKeys.swift +++ b/clients/apple/Sources/PunktfunkKit/Support/DefaultsKeys.swift @@ -31,6 +31,11 @@ public enum DefaultsKey { /// is higher up), so we fold to mono ourselves. Only meaningful for multi-channel devices. public static let micChannel = "punktfunk.micChannel" public static let presenter = "punktfunk.presenter" + /// macOS: V-Sync the stream's presents — each decoded frame flips on the next display vsync + /// (evenly paced, no tearing under direct scanout) instead of as soon as the GPU finishes + /// (lowest latency — the default, OFF). Resolved once per session; + /// PUNKTFUNK_PRESENT_MODE=immediate|vsync overrides it for A/B. See Stage2Pipeline's header. + public static let vsync = "punktfunk.vsync" /// Request a 10-bit BT.2020 PQ (HDR10) stream. On by default; only takes effect when the host /// has HDR content AND this display supports HDR — otherwise the stream stays 8-bit SDR. public static let hdrEnabled = "punktfunk.hdrEnabled" @@ -57,7 +62,7 @@ public enum DefaultsKey { /// macOS: take the window fullscreen while streaming and restore it on the host list. On by default. public static let fullscreenWhileStreaming = "punktfunk.fullscreenWhileStreaming" /// Show the streaming statistics overlay (mode/fps/throughput/latency). On by default; toggle - /// while streaming with ⌘⇧S (macOS / hardware keyboard). + /// while streaming with ⌃⌥⇧S (the cross-client Ctrl+Alt+Shift+S; macOS / hardware keyboard). public static let hudEnabled = "punktfunk.hudEnabled" /// Which corner the statistics overlay sits in — a `HUDPlacement` raw value /// ("topLeading"/"topTrailing"/"bottomLeading"/"bottomTrailing"). Default top-trailing. @@ -67,3 +72,12 @@ public enum DefaultsKey { /// whenever a gamepad is connected. On by default; see `GamepadUIEnvironment.isActive`. public static let gamepadUIEnabled = "punktfunk.gamepadUIEnabled" } + +extension Notification.Name { + /// Posted by the app's Stream menu ("Release Mouse", ⌃⌥⇧Q): the key window's stream view + /// releases input capture if it holds it. Only reachable while NOT captured (a captured + /// session swallows the combo in InputCapture's monitor and the frozen cursor can't click + /// menus) — it exists so the menu item is honest whenever it CAN fire, and as the shortcut's + /// discoverable menu-bar surface. + public static let punktfunkReleaseCapture = Notification.Name("io.unom.punktfunk.release-capture") +} diff --git a/clients/apple/Sources/PunktfunkKit/Video/MetalVideoPresenter.swift b/clients/apple/Sources/PunktfunkKit/Video/MetalVideoPresenter.swift index b16e9b4e..e94600b1 100644 --- a/clients/apple/Sources/PunktfunkKit/Video/MetalVideoPresenter.swift +++ b/clients/apple/Sources/PunktfunkKit/Video/MetalVideoPresenter.swift @@ -1,12 +1,15 @@ // Stage-2 presenter, present half: draw a decoded NV12 / P010 / 4:4:4 CVPixelBuffer into a CAMetalLayer -// drawable with a Y′CbCr→RGB shader. The hosting view's CADisplayLink drives `render` once per vsync -// (via Stage2Pipeline.renderTick) with the target present time, so a present can be stamped and the -// present tail hand-paced. See docs apple-stage2-presenter.md. +// drawable with a Y′CbCr→RGB shader. The hosting view's CADisplayLink still paces the pipeline once per +// vsync, but the actual `render` runs on Stage2Pipeline's dedicated RENDER THREAD (the link tick just +// signals it), so `nextDrawable()`'s blocking never lands on the main thread. See docs +// apple-stage2-presenter.md. // -// Main-thread only: created during view setup, `render`/`configure` called from the view's CADisplayLink -// (which fires on the main runloop). The Metal objects + texture cache are touched only here. The one -// exception is `setHdrMeta`, called from the pump thread — it hops the layer write to main so every -// CALayer mutation stays on one thread. +// Threading: created during view setup (main); `render`/`configure` run on the render thread — the +// layer's drawable/format/colour mutations all happen there (CAMetalLayer is designed for dedicated +// render threads; only the layer's GEOMETRY — frame/contentsScale — is touched from main, in +// SessionPresenter.layout, which also pushes the resulting pixel size here via `setDrawableTarget` +// so the render thread never reads layer geometry cross-thread). `setHdrMeta` (pump thread) and +// `setDrawableTarget` (main) only write lock-guarded staging state the render thread drains. #if canImport(Metal) && canImport(QuartzCore) import CoreGraphics @@ -144,14 +147,23 @@ public final class MetalVideoPresenter { private var textureCache: CVMetalTextureCache? /// Current layer configuration — switched in `configure(hdr:)` when a frame's HDR-ness differs. - /// Main-thread only (read + written from `render`/`configure`, all on the display-link runloop). + /// Render-thread confined once the pipeline runs (Stage2Pipeline.start's one pre-thread + /// `configure` call is ordered before the thread starts, so it doesn't race). private var hdrActive = false /// Last HDR mastering grade received via `setHdrMeta` (the host's 0xCE). Cached so a mid-session /// SDR→HDR flip's `configureColor` re-applies the real grade instead of clobbering it back to the /// bare reference-white anchor (an out-of-order race otherwise: `setHdrMeta` and the flip both write - /// `edrMetadata`). Main-thread only. + /// `edrMetadata`). Render-thread confined (drained from `pendingHdrMeta` at the top of `render`). private var lastHdrMeta: PunktfunkConnection.HdrMeta? + /// Cross-thread staging, all under `stagingLock`: the pump thread parks a fresh 0xCE grade in + /// `pendingHdrMeta` and the main thread parks the layout-derived drawable pixel size in + /// `drawableTarget`; the render thread drains both at the top of `render`, so every layer + /// format/colour mutation stays on the one thread that also calls `nextDrawable()`. + private let stagingLock = NSLock() + private var pendingHdrMeta: PunktfunkConnection.HdrMeta? + private var drawableTarget: CGSize = .zero + #if DEBUG /// Last logged "decoded→drawable" signature, so the diagnostic logs only on a size/HDR change. private var lastSizeSig = "" @@ -191,12 +203,18 @@ public final class MetalVideoPresenter { layer.framebufferOnly = true layer.isOpaque = true #if os(macOS) - // The display link already paces exactly one present per vsync. Leaving the layer's own vsync - // wait on means `commandBuffer.present` ALSO blocks for the hardware vsync, so `nextDrawable()` - // stalls the MAIN thread until a drawable frees — windowed, the WindowServer's looser - // compositing hides it; FULLSCREEN's tighter path serializes the main thread to the display and - // the stall surfaces as bad judder. Disabling the layer-level sync lets present return promptly - // (the display link is the pacing source) — the fix for the fullscreen stutter. macOS-only. + // displaySyncEnabled MUST stay false on macOS. It has flip-flopped, so the full history: + // sync ON was tried twice and starves the drawable pool both times — on macOS 26 a synced + // present only reaches glass when the WindowServer composites the window, and its FramePacing + // path does not treat our out-of-band image-queue presents as damage, so with a static scene + // the ONLY recurring damage is the 1 Hz stats HUD update: presents queue, all drawables stay + // held, `nextDrawable()` sleeps (sampled: ~70% of the render thread inside + // CAMetalLayerPrivateNextDrawableLocked → usleep), and the stream turns into a ~1 fps + // slideshow with normal-LOOKING stats (each rare frame is fresh, newest-wins ring). The + // 94fb7d1b fullscreen judder was the same starvation biting the then-main-thread render. + // With sync OFF the flip is immediate; the vsync alignment that sync was supposed to give + // (the HUD-off direct-scanout pacing fix) comes from scheduling the present at the display + // link's target time instead (`present(at:)` — see `render`). layer.displaySyncEnabled = false #endif // The drawable is rendered at the LAYER's pixel size (set per-frame in `render`), so the @@ -226,11 +244,12 @@ public final class MetalVideoPresenter { self.layer = layer } - /// Configure the layer + active pipeline for an SDR or HDR session. MAIN THREAD ONLY. Called once at - /// session start and again per-frame from `render` (idempotent — the guard makes a same-state call a - /// no-op), so a mid-session HDR toggle (the host re-inits its encoder; the decoded `frame.isHDR` - /// flips) reconfigures here automatically. HDR uses an rgba16Float drawable + BT.2020 PQ colour space - /// + EDR with a 203-nit reference-white anchor; SDR uses the plain 8-bit sRGB path. + /// Configure the layer + active pipeline for an SDR or HDR session. Called once at session start + /// (main, before the render thread exists) and again per-frame from `render` on the RENDER THREAD + /// (idempotent — the guard makes a same-state call a no-op), so a mid-session HDR toggle (the host + /// re-inits its encoder; the decoded `frame.isHDR` flips) reconfigures here automatically. HDR uses + /// an rgba16Float drawable + BT.2020 PQ colour space + EDR with a 203-nit reference-white anchor; + /// SDR uses the plain 8-bit sRGB path. public func configure(hdr: Bool) { guard hdr != hdrActive else { return } hdrActive = hdr @@ -273,36 +292,65 @@ public final class MetalVideoPresenter { #endif /// Update the HDR mastering metadata (drained from the host's 0xCE datagram) to refine the system - /// tone-map from the real grade. Called from the PUMP thread, so the layer write is hopped to MAIN - /// (every CALayer mutation stays on one thread). The grade is cached so a later SDR→HDR - /// `configureColor` re-applies it; the `edrMetadata` write is gated on `hdrActive` (setting it on an - /// SDR layer is harmless but pointless, and the flip will apply it anyway). + /// tone-map from the real grade. Called from the PUMP thread — the grade is only PARKED here (lock- + /// guarded); the render thread applies it at the top of the next `render`, keeping every layer + /// colour mutation on the one thread that also vends drawables. public func setHdrMeta(_ meta: PunktfunkConnection.HdrMeta) { - DispatchQueue.main.async { [weak self] in - guard let self else { return } - self.lastHdrMeta = meta - // tvOS has no edrMetadata — the cached grade is still kept above (harmless), it just can't - // be applied to the layer there. macOS/iOS refine the system tone-map from the real grade. - #if !os(tvOS) - if self.hdrActive { self.layer.edrMetadata = self.makeEDR(meta) } - #endif - } + stagingLock.lock() + pendingHdrMeta = meta + stagingLock.unlock() } - /// Draw one decoded frame to the next drawable and present it. MAIN THREAD (the display link). - /// `isHDR` selects the 10-bit BT.2020 PQ path vs the 8-bit BT.709 path and is reconciled with the + /// Park the drawable pixel size the shader should render at: the metal layer's laid-out frame × + /// contentsScale, both owned by the MAIN thread (SessionPresenter.layout pushes it on every layout/ + /// backing change). The render thread reads this instead of the layer's geometry so it never + /// touches main-owned CALayer state. Zero until the first layout → `render` falls back to the + /// decoded frame size. + public func setDrawableTarget(_ size: CGSize) { + stagingLock.lock() + drawableTarget = size + stagingLock.unlock() + } + + /// Draw one decoded frame to the next drawable and present it. RENDER THREAD (Stage2Pipeline's; + /// `nextDrawable()` may block up to a frame — that wait belongs here, never on main). `isHDR` + /// selects the 10-bit BT.2020 PQ path vs the 8-bit BT.709 path and is reconciled with the /// layer config via `configure`. Returns true on success; false when there's no drawable yet, a /// texture couldn't be made, or Metal errored — the caller then doesn't stamp a present (and can /// requeue the frame). `onPresented` fires once the drawable actually reached glass, with the /// `CLOCK_REALTIME` instant from the drawable's `presentedTime` — or nil when the system reports /// none (a dropped drawable). It runs on a Metal callback thread; keep the handler thread-safe. + /// + /// `presentAtMediaTime` (a `CACurrentMediaTime`-basis host time — the display link's + /// `targetTimestamp`) schedules the flip ON the vsync instead of "as soon as the GPU finishes": + /// with the layer's own sync disabled (mandatory on macOS — see init) an immediate present hits + /// glass mid-refresh whenever the layer is direct-scanout promoted (fullscreen, no HUD), which + /// is the "frametimes are off with the stats HUD closed" report. nil presents immediately + /// (`PUNKTFUNK_PRESENT_MODE=immediate` — the pre-fix behavior, kept as a diagnostic A/B). @discardableResult public func render( _ pixelBuffer: CVPixelBuffer, isHDR: Bool = false, + presentAtMediaTime: CFTimeInterval? = nil, onPresented: ((Int64?) -> Void)? = nil ) -> Bool { + // Drain the cross-thread staging (see `stagingLock`): the layout-derived drawable size and + // any freshly-arrived HDR grade, both applied from this thread. + stagingLock.lock() + let targetFromLayout = drawableTarget + let newHdrMeta = pendingHdrMeta + pendingHdrMeta = nil + stagingLock.unlock() + // Reconcile the layer with the decoded frame's HDR-ness (handles a mid-session SDR↔HDR flip). configure(hdr: isHDR) + if let newHdrMeta { + self.lastHdrMeta = newHdrMeta + // tvOS has no edrMetadata — the cached grade is still kept (a later HDR flip's + // configureColor is where it matters there). macOS/iOS refine the live tone-map now. + #if !os(tvOS) + if hdrActive { layer.edrMetadata = makeEDR(newHdrMeta) } + #endif + } // P010/x444 store 10-bit luma/chroma in 16-bit samples → R16/RG16; NV12/444v is 8-bit → R8/RG8. // Derived from the actual decoded buffer so a 4:4:4 (full chroma plane) frame just works. @@ -319,22 +367,18 @@ public final class MetalVideoPresenter { pixelBuffer, plane: 1, format: tenBit ? .rg16Unorm : .rg8Unorm, cache: textureCache) else { return false } - // Size the drawable to the LAYER's pixels (bounds × contentsScale, both set by the hosting - // view's layout) so the Catmull-Rom shader performs the decoded→on-screen scale in one pass: + // Size the drawable to the LAYER's pixels (its laid-out frame × contentsScale, pushed here by + // SessionPresenter.layout via `setDrawableTarget` — not read off the layer, whose geometry the + // main thread owns) so the Catmull-Rom shader performs the decoded→on-screen scale in one pass: // a native-mode session stays exactly 1:1 (the kernel reduces to the identity texel), and a // window bigger than the host's mode gets bicubic luma instead of the compositor's bilinear. - // Before the first layout (empty bounds) fall back to the decoded size. drawableSize does NOT + // Before the first layout (zero target) fall back to the decoded size. drawableSize does NOT // track bounds (defaults to 0), so set it BEFORE nextDrawable; re-set only on a change // (layout / Reconfigure / HDR flip — and every frame of a live resize, which is fine). let decodedSize = CGSize( width: CVPixelBufferGetWidth(pixelBuffer), height: CVPixelBufferGetHeight(pixelBuffer)) - let scale = layer.contentsScale - let boundsSize = layer.bounds.size - let targetSize = (boundsSize.width > 0 && boundsSize.height > 0) - ? CGSize( - width: (boundsSize.width * scale).rounded(), - height: (boundsSize.height * scale).rounded()) - : decodedSize + let targetSize = (targetFromLayout.width > 0 && targetFromLayout.height > 0) + ? targetFromLayout : decodedSize if layer.drawableSize != targetSize { layer.drawableSize = targetSize } #if DEBUG logSizeIfChanged(decoded: decodedSize, drawable: targetSize) @@ -374,7 +418,13 @@ public final class MetalVideoPresenter { } #endif } - commandBuffer.present(drawable) // present at the next vsync — lowest latency + // Scheduled on the vsync when the pipeline gave us the link's target (see the doc comment); + // immediate otherwise. A target already in the past presents immediately — same thing. + if let presentAtMediaTime { + commandBuffer.present(drawable, atTime: presentAtMediaTime) + } else { + commandBuffer.present(drawable) + } // Hold the CVMetalTextures + source pixel buffer (its IOSurface) alive until the GPU finishes // sampling — releasing them at scope exit could free the backing mid-read. commandBuffer.addCompletedHandler { _ in _ = (luma, chroma, pixelBuffer) } diff --git a/clients/apple/Sources/PunktfunkKit/Video/SessionPresenter.swift b/clients/apple/Sources/PunktfunkKit/Video/SessionPresenter.swift index ae305e2d..ff337a75 100644 --- a/clients/apple/Sources/PunktfunkKit/Video/SessionPresenter.swift +++ b/clients/apple/Sources/PunktfunkKit/Video/SessionPresenter.swift @@ -68,10 +68,13 @@ final class SessionPresenter { baseLayer.addSublayer(metal) metalLayer = metal stage2 = pipeline + // The link is the vsync CLOCK + putBack-retry nudge, not the presentation trigger + // (frame arrival is — see Stage2Pipeline's header). timestamp→targetTimestamp is the + // link's own report of the current refresh period (tracks VRR rate changes). let proxy = DisplayLinkProxy { [weak self] link in self?.stage2?.renderTick( - targetPresentNs: Stage2Pipeline.realtimeNs( - forDisplayLinkTimestamp: link.targetTimestamp)) + targetMediaTime: link.targetTimestamp, + period: link.targetTimestamp - link.timestamp) } let link = makeDisplayLink(proxy, #selector(DisplayLinkProxy.tick(_:))) link.add(to: .main, forMode: .common) @@ -127,6 +130,11 @@ final class SessionPresenter { metalLayer.contentsScale = contentsScale metalLayer.frame = fit CATransaction.commit() + // Hand the resulting pixel size to the render thread (it must not read layer geometry + // cross-thread) — this is what the presenter sizes its drawable to. + stage2?.setDrawableTarget(CGSize( + width: (fit.width * contentsScale).rounded(), + height: (fit.height * contentsScale).rounded())) } /// Stop the active pump/pipeline (≤ one poll timeout; stage-2 joins its pump) and detach the diff --git a/clients/apple/Sources/PunktfunkKit/Video/Stage2Pipeline.swift b/clients/apple/Sources/PunktfunkKit/Video/Stage2Pipeline.swift index 6634ca16..f2ae7df2 100644 --- a/clients/apple/Sources/PunktfunkKit/Video/Stage2Pipeline.swift +++ b/clients/apple/Sources/PunktfunkKit/Video/Stage2Pipeline.swift @@ -1,25 +1,60 @@ -// Stage-2 presenter orchestrator: a pump thread pulls AUs → VideoDecoder; the decoder's async output -// drops the newest decoded frame into a 1-slot ring; the hosting view's display link calls `renderTick` -// once per vsync to draw + present the newest ready frame and stamp the unified latency stages -// (end-to-end capture→on-glass, plus the decode and display stage terms — -// design/stats-unification.md). Mirrors StreamPump's lifecycle (one per start; cancel is permanent). +// Stage-2 presenter orchestrator. GOAL ARCHITECTURE (the result of the 2026-07 pacing saga — +// read this before touching presentation): // -// Threading: the pump runs on its own thread; the decoder callback on a VT thread; `renderTick` + -// `start`/`stop` on the MAIN thread (the view's CADisplayLink fires there). Only the ring (lock-guarded) -// and the decoder/presenter (internally locked / main-hopped) cross threads. +// net pump ──► VideoDecoder (VT async) ──► newest-wins 1-slot ring ──► RENDER THREAD ──► CAMetalLayer +// +// • The render thread is woken by FRAME ARRIVAL (the decoder callback signals it), never gated on +// the display link: on macOS the WindowServer's damage tracking / FramePacing does not count our +// out-of-band presents, so anything display-link-gated stalls exactly when the rest of the screen +// goes quiet (adaptive-sync displays idle the link down). A decoded frame is always presented +// promptly. The display link remains only as (a) a vsync CLOCK (phase + period, for the opt-in +// V-Sync policy below), (b) a retry tick for a frame that couldn't get a drawable (`putBack`), +// and (c) the iOS ProMotion rate hint. +// • The layer's own displaySyncEnabled stays FALSE on macOS — synced presents starve the drawable +// pool outright (see MetalVideoPresenter's init for the post-mortem). +// • Present policy is a USER SETTING (DefaultsKey.vsync; PUNKTFUNK_PRESENT_MODE=immediate|vsync +// overrides it for A/B), resolved once per session in start(): +// – V-Sync OFF (default): present immediately — lowest latency, the long-proven behavior. +// – V-Sync ON: present(at: next vsync) predicted from the link's last phase/period, at most one +// period ahead by construction, falling back to immediate when the link data is stale — a +// schedule can never sit far in the future holding drawables hostage. +// • Rendering lives on its own thread so any `nextDrawable()` wait lands off-main (input, SwiftUI). +// +// The render thread also stamps the unified latency stages (end-to-end capture→on-glass + decode and +// display stage terms — design/stats-unification.md). Mirrors StreamPump's lifecycle (one per start; +// cancel is permanent). PUNKTFUNK_PRESENT_DEBUG=1 prints per-second pacing stats (see +// PresentDebugStats). +// +// Threading: the pump runs on its own thread; the decoder callback on a VT thread; the render loop on +// the render thread; `renderTick` + `start`/`stop` on the MAIN thread (the view's CADisplayLink fires +// there). Only the ring (lock-guarded), the vsync clock (lock-guarded), and the decoder/presenter +// (internally locked / staged) cross threads. #if canImport(Metal) && canImport(QuartzCore) import AVFoundation import Foundation import QuartzCore +/// PUNKTFUNK_PRESENT_DEBUG=1: the render thread prints a once-per-second line with the decode +/// (ring-submit) rate, present rate, failed/empty wakes and the slowest render call — for +/// diagnosing pacing regressions without instruments. Plain print: the unbundled CLI client's +/// stdout is the cheapest reliable capture channel. +let presentDebug = ProcessInfo.processInfo.environment["PUNKTFUNK_PRESENT_DEBUG"] == "1" + /// Newest-ready 1-slot ring: the decoder overwrites (drops the older undisplayed frame — lowest /// latency, no smoothing buffer), the display link takes-and-clears. Sendable; lock-guarded. private final class ReadyRing: @unchecked Sendable { private let lock = NSLock() private var frame: ReadyFrame? + /// Ring submissions since the last `drainSubmitted` — the decode rate for the + /// PUNKTFUNK_PRESENT_DEBUG stat line. + private var submitted = 0 func submit(_ f: ReadyFrame) { - lock.lock(); frame = f; lock.unlock() + lock.lock(); frame = f; submitted += 1; lock.unlock() + } + func drainSubmitted() -> Int { + lock.lock(); defer { lock.unlock() } + let n = submitted; submitted = 0; return n } func take() -> ReadyFrame? { lock.lock(); defer { lock.unlock() } @@ -37,6 +72,86 @@ private final class ReadyRing: @unchecked Sendable { } } +/// The display's vsync grid as last reported by the display link (target timestamp + period, +/// `CACurrentMediaTime` basis), written on main by `renderTick`, read by the render thread to +/// schedule V-Sync-mode presents. A shared box (like `ReadyRing`) so neither thread captures the +/// pipeline itself. Sendable; lock-guarded. +private final class VsyncClock: @unchecked Sendable { + private let lock = NSLock() + private var target: CFTimeInterval = 0 + private var period: CFTimeInterval = 0 + + func set(target t: CFTimeInterval, period p: CFTimeInterval) { + lock.lock(); target = t; period = p; lock.unlock() + } + + /// The next vsync at or after `now`, extrapolated from the last reported phase/period — by + /// construction less than one period ahead, so a scheduled present can never sit far in the + /// future holding its drawable. nil (⇒ present immediately) when the link has reported nothing + /// yet, its period is nonsense, or its data is STALE (an idle/suspended link on an + /// adaptive-sync display — exactly the case where scheduling onto its grid stalls the stream). + func nextVsync(after now: CFTimeInterval) -> CFTimeInterval? { + lock.lock(); defer { lock.unlock() } + guard period > 0.0005, target > 0, now - target < 0.25 else { return nil } + if target >= now { return target } + return target + ceil((now - target) / period) * period + } +} + +/// PUNKTFUNK_PRESENT_DEBUG=1 aggregation: one printed line per second from the render thread with +/// the decode rate, render outcomes, the slowest render call (≈ nextDrawable wait) and the deltas +/// between system-reported on-glass times (vsync-aligned presents show clean refresh-period +/// multiples; immediate flips scatter). Lock-guarded — `presented` lands on a Metal callback thread. +private final class PresentDebugStats: @unchecked Sendable { + private let lock = NSLock() + private var last = CACurrentMediaTime() + private var ok = 0, failed = 0, empty = 0, dropped = 0 + private var maxRenderMs = 0.0 + private var lastGlassNs: Int64 = 0 + private var glassDeltasMs: [Double] = [] + + func emptyWake() { lock.lock(); empty += 1; lock.unlock() } + + func renderReturned(ok rendered: Bool, tookMs: Double) { + lock.lock() + if rendered { ok += 1 } else { failed += 1 } + maxRenderMs = max(maxRenderMs, tookMs) + lock.unlock() + } + + func presented(atNs: Int64?) { + lock.lock() + if let atNs { + if lastGlassNs > 0 { glassDeltasMs.append(Double(atNs - lastGlassNs) / 1e6) } + lastGlassNs = atNs + } else { + dropped += 1 + } + lock.unlock() + } + + func flushIfDue(ring: ReadyRing) { + lock.lock() + let now = CACurrentMediaTime() + guard now - last >= 1 else { lock.unlock(); return } + last = now + let decoded = ring.drainSubmitted() + let deltas = glassDeltasMs.sorted() + let p50 = deltas.isEmpty ? 0 : deltas[deltas.count / 2] + let dMax = deltas.last ?? 0 + let line = String( + format: "pf-present decoded=%d ok=%d fail=%d empty=%d dropped=%d " + + "maxRenderMs=%.1f glassDeltaMs p50=%.2f max=%.2f n=%d", + decoded, ok, failed, empty, dropped, maxRenderMs, p50, dMax, deltas.count) + ok = 0; failed = 0; empty = 0; dropped = 0 + maxRenderMs = 0 + glassDeltasMs.removeAll(keepingCapacity: true) + lock.unlock() + print(line) + fflush(stdout) // stdout is a pipe when captured — flush per line or nothing shows + } +} + public final class Stage2Pipeline { private let ring = ReadyRing() private let presenter: MetalVideoPresenter @@ -54,6 +169,19 @@ public final class Stage2Pipeline { private let pumpStopped = DispatchSemaphore(value: 0) private var pumpJoinable = false + /// Render-thread plumbing. `renderSignal` wakes the render thread — signalled by the DECODER + /// callback on every frame (the primary trigger: presentation must never be gated on the + /// display link, see the header) and by each display-link tick (the `putBack` retry + the + /// vsync-clock refresh). Signals coalesce harmlessly (an extra wake finds an empty ring and + /// goes back to sleep). `vsyncClock` is the link's last phase/period for V-Sync-mode + /// scheduling. Lock-guarded boxes — the render thread, like the pump thread, must not capture + /// `self`, or a missed stop() would leak a spinning pipeline. `renderStopped`/`renderJoinable` + /// mirror the pump's bounded join. + private let renderSignal = DispatchSemaphore(value: 0) + private let vsyncClock = VsyncClock() + private let renderStopped = DispatchSemaphore(value: 0) + private var renderJoinable = false + /// The Metal layer the hosting view installs + sizes. public var layer: CAMetalLayer { presenter.layer } @@ -74,6 +202,7 @@ public final class Stage2Pipeline { self.displayMeter = displayMeter let ring = ring let recovery = recovery + let renderSignal = renderSignal self.decoder = VideoDecoder( onDecoded: { frame in // Decode stage = received→decoded, both client CLOCK_REALTIME (offset 0 — no @@ -82,6 +211,8 @@ public final class Stage2Pipeline { decodeMeter?.record( ptsNs: UInt64(frame.receivedNs), atNs: frame.decodedNs, offsetNs: 0) ring.submit(frame) + // FRAME ARRIVAL is the render trigger (never the display link — see the header). + renderSignal.signal() }, // Async decode failure (a bad P-frame referencing a lost/corrupt IDR): the pump resets to // re-gate on the next IDR, and we ask the host to send one now (infinite GOP — it wouldn't @@ -176,34 +307,89 @@ public final class Stage2Pipeline { thread.qualityOfService = .userInteractive pumpJoinable = true thread.start() - } - /// MAIN thread, once per vsync. Present the newest ready frame (if any). The latency stamps - /// use the drawable's ACTUAL on-glass instant (`addPresentedHandler`/`presentedTime` — the - /// handler fires on a Metal callback thread; the meters are thread-safe), falling back to - /// `targetPresentNs` — the display link's target present instant, already converted to - /// `CLOCK_REALTIME` (see `realtimeNs(forDisplayLinkTimestamp:)`) — when the system reports - /// no presented time (a dropped drawable). A frame that could not be rendered (no drawable - /// yet) goes back into the ring so the next tick retries it. - public func renderTick(targetPresentNs: Int64) { - guard let frame = ring.take() else { return } - let offsetNs = offsetNs + // The render thread: one present per display-link signal. It owns every layer format/colour/ + // drawable interaction (see MetalVideoPresenter's threading notes); with displaySyncEnabled on, + // nextDrawable's up-to-a-frame wait lands here instead of on main. The 100 ms timed wait is + // only the stop-flag poll for a session whose link stopped ticking. + let ring = ring let endToEndMeter = endToEndMeter let displayMeter = displayMeter - let rendered = presenter.render(frame.pixelBuffer, isHDR: frame.isHDR) { presentedNs in - let atNs = presentedNs ?? targetPresentNs - // End-to-end = capture→on-glass, measured directly (skew-corrected via the - // connect-time clock offset) — the HUD headline. - endToEndMeter?.record(ptsNs: frame.ptsNs, atNs: atNs, offsetNs: offsetNs) - // Display stage = decoded → on-glass. Both instants are client CLOCK_REALTIME, - // so no skew offset applies. - displayMeter?.record(ptsNs: UInt64(frame.decodedNs), atNs: atNs, offsetNs: 0) + let offsetNs = offsetNs + let renderSignal = renderSignal + let renderStopped = renderStopped + // Present policy — the user's V-Sync setting (default OFF = immediate, the long-proven + // lowest-latency behavior); PUNKTFUNK_PRESENT_MODE=immediate|vsync overrides it for A/B. + // Resolved once per session. + let presentMode = ProcessInfo.processInfo.environment["PUNKTFUNK_PRESENT_MODE"] + let vsyncEnabled = presentMode == "vsync" + || (presentMode != "immediate" + && UserDefaults.standard.bool(forKey: DefaultsKey.vsync)) + let debugStats = presentDebug ? PresentDebugStats() : nil + let vsyncClock = vsyncClock + let renderThread = Thread { + defer { renderStopped.signal() } + while !token.isStopped { + if renderSignal.wait(timeout: .now() + .milliseconds(100)) == .timedOut { + debugStats?.flushIfDue(ring: ring) + continue + } + guard !token.isStopped, let frame = ring.take() else { + debugStats?.emptyWake() + debugStats?.flushIfDue(ring: ring) + continue + } + // V-Sync ON: flip on the next predicted vsync (< one period out, stale link ⇒ + // immediate — see VsyncClock). OFF: flip as soon as the GPU finishes. + let presentAt = vsyncEnabled + ? vsyncClock.nextVsync(after: CACurrentMediaTime()) : nil + let renderStarted = CACurrentMediaTime() + let rendered = presenter.render( + frame.pixelBuffer, isHDR: frame.isHDR, presentAtMediaTime: presentAt + ) { presentedNs in + // Fallback stamp for a dropped drawable (no system presentedTime): "now" on + // the Metal callback, converted to the CLOCK_REALTIME the meters live in. + let atNs = presentedNs + ?? Stage2Pipeline.realtimeNs(forDisplayLinkTimestamp: CACurrentMediaTime()) + // End-to-end = capture→on-glass, measured directly (skew-corrected via the + // connect-time clock offset) — the HUD headline. + endToEndMeter?.record(ptsNs: frame.ptsNs, atNs: atNs, offsetNs: offsetNs) + // Display stage = decoded → on-glass. Both instants are client CLOCK_REALTIME, + // so no skew offset applies. + displayMeter?.record(ptsNs: UInt64(frame.decodedNs), atNs: atNs, offsetNs: 0) + debugStats?.presented(atNs: presentedNs) + } + debugStats?.renderReturned( + ok: rendered, tookMs: (CACurrentMediaTime() - renderStarted) * 1000) + if !rendered { ring.putBack(frame) } + debugStats?.flushIfDue(ring: ring) + } } - if !rendered { ring.putBack(frame) } + renderThread.name = "punktfunk-stage2-render" + renderThread.qualityOfService = .userInteractive + renderJoinable = true + renderThread.start() } - /// Stop the pump (≤ one poll timeout) and drop the decode session. MAIN THREAD; idempotent. Does not - /// close the connection. A restart needs a fresh Stage2Pipeline (the stop is permanent). + /// MAIN thread, once per display-link tick: refresh the vsync clock (V-Sync-mode scheduling) + /// and nudge the render thread. The nudge is NOT the presentation trigger — frame arrival is + /// (see the header) — it only retries a frame a transient `nextDrawable` failure put back into + /// the ring, which matters under the host's infinite GOP where a static scene sends no + /// replacement frame. + public func renderTick(targetMediaTime: CFTimeInterval, period: CFTimeInterval) { + vsyncClock.set(target: targetMediaTime, period: period) + renderSignal.signal() + } + + /// Forward the layout-derived drawable pixel size to the presenter (MAIN thread — see + /// `MetalVideoPresenter.setDrawableTarget`). + public func setDrawableTarget(_ size: CGSize) { + presenter.setDrawableTarget(size) + } + + /// Stop the pump + render thread (≤ one poll timeout each) and drop the decode session. MAIN + /// THREAD; idempotent. Does not close the connection. A restart needs a fresh Stage2Pipeline + /// (the stop is permanent). public func stop() { token.stop() // Join the pump (bounded: ≤ one nextAU poll + an in-flight decode) before resetting the decoder, @@ -213,11 +399,22 @@ public final class Stage2Pipeline { pumpJoinable = false _ = pumpStopped.wait(timeout: .now() + 0.5) } + // Wake + join the render thread (bounded: it may sit in `nextDrawable` for up to ~a frame; a + // timed-out join is fine — the loop exits at its next stop-flag check, and a final present on + // the detached layer is harmless). + if renderJoinable { + renderJoinable = false + renderSignal.signal() + _ = renderStopped.wait(timeout: .now() + 0.5) + } decoder.reset() recovery.bind(nil) // stop requesting keyframes once the session is torn down } - deinit { token.stop() } + deinit { + token.stop() + renderSignal.signal() // wake the render thread so it can observe the stop and exit + } /// Convert a `CADisplayLink.targetTimestamp` (CACurrentMediaTime basis) to a `CLOCK_REALTIME` /// nanosecond instant — the present clock the AU pts + skew offset live in. Projects to the target diff --git a/clients/apple/Sources/PunktfunkKit/Views/StreamView.swift b/clients/apple/Sources/PunktfunkKit/Views/StreamView.swift index 27a428d4..d245a943 100644 --- a/clients/apple/Sources/PunktfunkKit/Views/StreamView.swift +++ b/clients/apple/Sources/PunktfunkKit/Views/StreamView.swift @@ -7,10 +7,11 @@ // // The view also owns the input-capture state machine (Moonlight-style): capture is a // deliberate, reversible state — engaged when the stream starts and when the user clicks -// into the video, released by ⌘⎋ or focus loss, and NEVER engaged by mere app -// activation (the click that activates the window may be a title-bar drag or a resize — -// warping the cursor there is exactly the intrusiveness this design removes). While -// released, nothing is forwarded to the host and the local cursor is free. +// into the video, released by ⌃⌥⇧Q (the cross-client Ctrl+Alt+Shift+Q), ⌘⎋, or focus +// loss, and NEVER engaged by mere app activation (the click that activates the window may +// be a title-bar drag or a resize — warping the cursor there is exactly the intrusiveness +// this design removes). While released, nothing is forwarded to the host and the local +// cursor is free. // // macOS-first (NSViewRepresentable); the iOS variant is the same layer under // UIViewRepresentable. @@ -83,6 +84,7 @@ public struct StreamView: NSViewRepresentable { private let connection: PunktfunkConnection private let captureEnabled: Bool private let onCaptureChange: ((Bool) -> Void)? + private let onDisconnectRequest: (() -> Void)? private let onFrame: (@Sendable (AccessUnit) -> Void)? private let onSessionEnd: (@Sendable () -> Void)? private let endToEndMeter: LatencyMeter? @@ -93,14 +95,17 @@ public struct StreamView: NSViewRepresentable { /// `captureEnabled: false` disables input capture entirely while UI (e.g. a trust /// prompt) is layered over the stream; flipping it to true auto-engages capture /// once. `onCaptureChange` (main thread) reports engage/release — drive the HUD's - /// "click to capture" / "⌘⎋ releases" hint with it. The meters record the unified latency - /// stages when the stage-2 presenter is active (design/stats-unification.md): - /// `endToEndMeter` capture→on-glass, `decodeMeter` received→decoded, `displayMeter` - /// decoded→on-glass. + /// "click to capture" / "⌃⌥⇧Q releases" hint with it. `onDisconnectRequest` (main + /// thread) fires on the reserved ⌃⌥⇧D combo while captured — the owner ends the + /// session (released, the same combo reaches the Stream menu instead). The meters + /// record the unified latency stages when the stage-2 presenter is active + /// (design/stats-unification.md): `endToEndMeter` capture→on-glass, `decodeMeter` + /// received→decoded, `displayMeter` decoded→on-glass. public init( connection: PunktfunkConnection, captureEnabled: Bool = true, onCaptureChange: ((Bool) -> Void)? = nil, + onDisconnectRequest: (() -> Void)? = nil, onFrame: (@Sendable (AccessUnit) -> Void)? = nil, onSessionEnd: (@Sendable () -> Void)? = nil, endToEndMeter: LatencyMeter? = nil, @@ -110,6 +115,7 @@ public struct StreamView: NSViewRepresentable { self.connection = connection self.captureEnabled = captureEnabled self.onCaptureChange = onCaptureChange + self.onDisconnectRequest = onDisconnectRequest self.onFrame = onFrame self.onSessionEnd = onSessionEnd self.endToEndMeter = endToEndMeter @@ -120,6 +126,7 @@ public struct StreamView: NSViewRepresentable { public func makeNSView(context: Context) -> StreamLayerView { let view = StreamLayerView() view.onCaptureChange = onCaptureChange + view.onDisconnectRequest = onDisconnectRequest view.captureEnabled = captureEnabled view.endToEndMeter = endToEndMeter view.decodeMeter = decodeMeter @@ -130,6 +137,7 @@ public struct StreamView: NSViewRepresentable { public func updateNSView(_ view: StreamLayerView, context: Context) { view.onCaptureChange = onCaptureChange + view.onDisconnectRequest = onDisconnectRequest view.captureEnabled = captureEnabled view.endToEndMeter = endToEndMeter view.decodeMeter = decodeMeter @@ -189,6 +197,10 @@ public final class StreamLayerView: NSView { /// Reports engage/release on the main thread. public var onCaptureChange: ((Bool) -> Void)? + /// Fired (main thread) when the captured-state ⌃⌥⇧D combo asks to end the session — the + /// view can't do that itself (the connection's owner disconnects). + public var onDisconnectRequest: (() -> Void)? + /// Main-thread only. False = input capture disabled outright (UI layered over the /// stream); flipping to true auto-engages once. public var captureEnabled = true { @@ -215,6 +227,16 @@ public final class StreamLayerView: NSView { ) { [weak self] _ in self?.releaseCapture() }) + // The Stream menu's "Release Mouse" item (⌃⌥⇧Q's discoverable menu-bar surface). Only + // the key window's stream may act — same ownership rule as the ⌘⎋ toggle. (While + // captured the combo never reaches the menu — InputCapture's monitor handles it — so + // in practice this fires only as a not-captured no-op; wired for honesty.) + appObservers.append(NotificationCenter.default.addObserver( + forName: .punktfunkReleaseCapture, object: nil, queue: .main + ) { [weak self] _ in + guard let self, self.window?.isKeyWindow == true else { return } + self.releaseCapture() + }) } public required init?(coder: NSCoder) { fatalError("not used") } @@ -562,6 +584,24 @@ public final class StreamLayerView: NSView { // (see the cursorVisible resolution below): toggling it on under gamescope's relative-only // input traps the pointer. Restore this body when absolute/synthetic-cursor support lands. capture.onToggleCursor = {} + // The cross-client combos (⌃⌥⇧Q/D/S — Ctrl+Alt+Shift on the other clients), delivered by + // the monitor only while captured; the same key-window ownership rule as ⌘⎋ throughout. + capture.onReleaseCapture = { [weak self] in + guard let self, self.window?.isKeyWindow == true else { return } + self.releaseCapture() + } + capture.onDisconnect = { [weak self] in + guard let self, self.window?.isKeyWindow == true else { return } + self.onDisconnectRequest?() + } + capture.onToggleStats = { [weak self] in + guard self?.window?.isKeyWindow == true else { return } + // Flip the shared setting directly — every @AppStorage reader (the HUD's visibility, + // the menu item's title) observes UserDefaults, so this is the same as the menu path. + let defaults = UserDefaults.standard + let current = defaults.object(forKey: DefaultsKey.hudEnabled) as? Bool ?? true + defaults.set(!current, forKey: DefaultsKey.hudEnabled) + } capture.start() inputCapture = capture diff --git a/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift b/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift index 8c32aabf..db90a414 100644 --- a/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift +++ b/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift @@ -54,10 +54,15 @@ public struct StreamView: UIViewControllerRepresentable { private let decodeMeter: LatencyMeter? private let displayMeter: LatencyMeter? + /// `onDisconnectRequest` exists for call-site parity with the macOS StreamView (the + /// captured-state ⌃⌥⇧D combo is detected by the macOS NSEvent monitor only); on iOS a + /// hardware keyboard reaches Disconnect through the Stream menu's key equivalent instead, + /// so the parameter is accepted and unused here. public init( connection: PunktfunkConnection, captureEnabled: Bool = true, onCaptureChange: ((Bool) -> Void)? = nil, + onDisconnectRequest: (() -> Void)? = nil, onFrame: (@Sendable (AccessUnit) -> Void)? = nil, onSessionEnd: (@Sendable () -> Void)? = nil, endToEndMeter: LatencyMeter? = nil,