feat(apple): stage-2 presenter — explicit decode + Metal present + glass-to-glass
ci / web (push) Failing after 38s
ci / rust (push) Successful in 53s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 3s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 4s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 16s
ci / docs-site (push) Failing after 39s
docker / deploy-docs (push) Successful in 16s
apple / swift (push) Successful in 1m17s

Opt-in (Settings -> Presenter; `punktfunk.presenter`, default stage-1). Stage-1's
AVSampleBufferDisplayLayer decodes AND presents internally with no per-frame
callback, so neither decode nor present can be stamped or hand-paced. Stage-2
takes explicit control:

- VideoDecoder: VTDecompressionSession, async output callback stamps
  decode-completion, session rebuilt on every IDR / format change. Unit-tested
  (testVideoDecoderAsyncCallbackDeliversPixels).
- MetalVideoPresenter: CAMetalLayer + CVMetalTextureCache + a runtime-compiled
  BT.709 limited-range NV12->RGB shader, present at the next vsync. The
  CVMetalTextures + pixel buffer are held until the GPU completes.
- Stage2Pipeline: pump thread -> decoder -> newest-ready 1-slot ring; the hosting
  view's display link drains it once per vsync and stamps capture->present
  (the display-link target time projected into CLOCK_REALTIME).
- LatencyMeter gains record(ptsNs:atNs:offsetNs:); the HUD shows a capture->present
  (glass-to-glass, modulo host render->capture) line, skew-corrected via
  clockOffsetNs. Measured live ~11 ms p50 vs ~2.2 ms capture->client.
- StreamView / StreamViewIOS host the CAMetalLayer as a sublayer + a CADisplayLink
  (NSView.displayLink on macOS) when stage-2; input capture + HUD unchanged. The
  session-active gates switch from `pump != nil` to `connection != nil` so capture
  engages without a StreamPump.

Validated: builds macOS/iOS/tvOS; the decode half is unit-tested; the Metal
present is live-validated on glass (correct image + the capture->present number).
Colorspace is BT.709 SDR for now; 10-bit/HDR + a pacing policy are later.
Plan: docs-site/content/docs/apple-stage2-presenter.md.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-12 15:28:23 +02:00
parent 848738ed00
commit 7b10714b62
12 changed files with 737 additions and 30 deletions
@@ -610,7 +610,8 @@ struct ContentView: View {
},
onSessionEnd: { [weak model] in
Task { @MainActor in model?.sessionEnded() }
}
},
presentMeter: model.presentLatency
)
.overlay(alignment: .topTrailing) {
if captureEnabled { hud(conn) }
@@ -635,6 +636,13 @@ struct ContentView: View {
.font(.system(.caption2, design: .monospaced))
.foregroundStyle(.secondary)
}
if model.presentLatencyValid {
// Capturepresent (glass-to-glass, modulo host rendercapture) stage-2 presenter
// only; stage-1's layer presents internally with no per-frame stamp.
Text("capture→present \(model.presentLatencyP50Ms, specifier: "%.1f")/\(model.presentLatencyP95Ms, specifier: "%.1f") ms p50/p95\(model.presentLatencySkewCorrected ? "" : " (same-host)")")
.font(.system(.caption2, design: .monospaced))
.foregroundStyle(.secondary)
}
// While captured the cursor is hidden+frozen, so the button is keyboard-only
// ( or Cmd+Tab release the cursor; released, it's clickable again).
#if os(macOS)
@@ -61,12 +61,21 @@ final class SessionModel: ObservableObject {
@Published var latencyP95Ms = 0.0
@Published var latencyValid = false
@Published var latencySkewCorrected = false
/// Capturepresent (glass-to-glass, modulo the host rendercapture term) only the stage-2
/// presenter can stamp this (it owns decode + a CAMetalLayer/display-link present). Stays
/// invalid under stage-1, where the layer presents internally with no per-frame callback.
@Published var presentLatencyP50Ms = 0.0
@Published var presentLatencyP95Ms = 0.0
@Published var presentLatencyValid = false
@Published var presentLatencySkewCorrected = false
/// Mirrors StreamView's capture state (it owns the input capture; this drives the
/// HUD's "click to capture" / " releases" hint).
@Published var mouseCaptured = false
let meter = FrameMeter()
let latency = LatencyMeter()
/// Fed by the stage-2 presenter's display link (capturepresent). Passed to StreamView.
let presentLatency = LatencyMeter()
private var statsTimer: Timer?
private var audio: SessionAudio?
private var gamepadCapture: GamepadCapture?
@@ -230,6 +239,14 @@ final class SessionModel: ObservableObject {
} else {
self.latencyValid = false
}
if let p = self.presentLatency.drain() {
self.presentLatencyP50Ms = p.p50Ms
self.presentLatencyP95Ms = p.p95Ms
self.presentLatencySkewCorrected = p.skewCorrected
self.presentLatencyValid = true
} else {
self.presentLatencyValid = false
}
}
}
// .common so the HUD keeps updating during window drags / menu tracking.
@@ -17,6 +17,7 @@ struct SettingsView: View {
@AppStorage("punktfunk.compositor") private var compositor = 0
@AppStorage("punktfunk.gamepadType") private var gamepadType = 0
@AppStorage("punktfunk.bitrateKbps") private var bitrateKbps = 0
@AppStorage("punktfunk.presenter") private var presenter = "stage1"
@AppStorage("punktfunk.micEnabled") private var micEnabled = true
@ObservedObject private var gamepads = GamepadManager.shared
#if os(macOS)
@@ -88,6 +89,10 @@ struct SettingsView: View {
}
TVSelectionRow(
title: "Compositor", options: compositors, selection: $compositor)
TVSelectionRow(
title: "Presenter",
options: [("Stage 1 (default)", "stage1"), ("Stage 2 (experimental)", "stage2")],
selection: $presenter)
Text("The host creates a virtual output at exactly this mode — native "
+ "resolution, no scaling. \(Self.bitrateFooter) A specific compositor "
+ "is honored only if available on the host.")
@@ -366,6 +371,21 @@ struct SettingsView: View {
.font(.caption)
.foregroundStyle(.secondary)
}
Section {
Picker("Presenter", selection: $presenter) {
Text("Stage 1 (default)").tag("stage1")
Text("Stage 2 (experimental)").tag("stage2")
}
} header: {
Text("Video presenter")
} footer: {
Text("Stage 1 feeds compressed video to the system display layer (known-good). "
+ "Stage 2 decodes explicitly and presents through Metal with a display "
+ "link — it adds a capture→present (glass-to-glass) latency line in the HUD "
+ "and shortens the present tail. Applies from the next session.")
.font(.caption)
.foregroundStyle(.secondary)
}
Section {
if gamepads.controllers.isEmpty {
Text("No controllers detected")