feat(clients): unified stats vocabulary across every client + Moonlight comparison docs

One stat model everywhere (design/stats-unification.md): four measurement
points (capture/received/decoded/displayed), three stages that tile the
interval exactly, and a HUD that shows the addition explicitly —

  end-to-end 14.2 ms p50 · 19.8 p95 · capture→on-glass
  = host+network 9.8 + decode 2.1 + display 2.3

replacing each client's ad-hoc mix of overlapping absolutes (the Apple HUD's
three arrow lines that looked sequential but weren't), mean-vs-median decode
times (Windows/Linux), missing same-host-clock flags (Windows/Linux), and
three different names for the same capture→received measurement (probe's
"reassembled", Apple/Android's "client", Windows/Linux's post-decode "lat").

Per client: Apple threads receivedNs through the VT decode via the frame
refcon bit pattern so the decode stage exists at all (stage-1 fallback
honestly degrades to a capture→received headline); Windows carries
FrameTimes through the existing frame channel to the render thread and adds
e2e p50/p95 post-Present; Linux stamps received at AU pop and rides
decoded_ns on DecodedFrame to the paintable-set site; Android pairs receipt
stamps with MediaCodec output buffers via the codec's pts round-trip (JNI
stats array 14→16 doubles, indexes 0-13 unchanged). fps now uniformly counts
received AUs; lost/(received+lost) per window, hidden at zero.

docs-site gains "Understanding the Stats Overlay": what each line means, why
the equation only approximately sums (percentiles), and a line-by-line
Moonlight/Sunshine matrix — including that Moonlight has no end-to-end
number and its "network latency" is an ENet control RTT, so punktfunk's
headline must not be compared against any single Moonlight line.

Verified here: linux client + probe + core check/clippy/fmt green, android
native cargo-ndk arm64 check green. Pending: Windows CI + on-glass, swift
test on the mac, on-device Android.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-03 21:01:29 +00:00
parent c7630ff5dc
commit 09a5957c6d
38 changed files with 1122 additions and 380 deletions
@@ -333,8 +333,9 @@ struct ContentView: View {
onSessionEnd: { [weak model] in
Task { @MainActor in model?.sessionEnded() }
},
presentMeter: model.presentLatency,
presentTailMeter: model.presentTail
endToEndMeter: model.endToEnd,
decodeMeter: model.decodeStage,
displayMeter: model.displayStage
)
.overlay(alignment: placement.alignment) {
if captureEnabled && hudEnabled {
@@ -170,7 +170,10 @@ private struct ShotHUD: View {
Text("5120×1440@240 240 fps 812.4 Mb/s")
.font(.system(.caption, design: .monospaced))
}
Text("capture→client 1.3/2.1 ms p50/p95")
Text("end-to-end 2.9 ms p50 · 3.8 p95 · capture→on-glass")
.font(.system(.caption2, design: .monospaced))
.foregroundStyle(.secondary)
Text("= host+network 1.3 + decode 0.7 + display 0.9")
.font(.system(.caption2, design: .monospaced))
.foregroundStyle(.secondary)
#if os(macOS)
@@ -59,36 +59,50 @@ final class SessionModel: ObservableObject {
@Published var fps = 0
@Published var mbps = 0.0
@Published var totalFrames = 0
/// Captureclient-receipt latency (ms), skew-corrected across machines via the connect-time
/// clock offset p50/p95 for the HUD. `latencyValid` is false until the first sample drains
/// (and whenever no host frames arrived in the last interval). `latencySkewCorrected` = the host
/// The unified latency stages (design/stats-unification.md), ms per 1 s window. `host+network`
/// = capturereceived, skew-corrected across machines via the connect-time clock offset: the
/// stage-2 HUD shows its p50 in the equation line; the stage-1 fallback shows p50/p95 as its
/// `capturereceived` headline. `hostNetworkValid` is false until the first sample drains (and
/// whenever no host frames arrived in the last interval). `hostNetworkSkewCorrected` = the host
/// answered the skew handshake (the number is cross-machine valid, not just same-host).
@Published var latencyP50Ms = 0.0
@Published var latencyP95Ms = 0.0
@Published var latencyValid = false
@Published var latencySkewCorrected = false
/// Capturepresent (glass-to-glass, modulo the host rendercapture term) only the stage-2
/// presenter can stamp this (it owns decode + a CAMetalLayer/display-link present). Stays
/// invalid under stage-1, where the layer presents internally with no per-frame callback.
@Published var presentLatencyP50Ms = 0.0
@Published var presentLatencyP95Ms = 0.0
@Published var presentLatencyValid = false
@Published var presentLatencySkewCorrected = false
/// Decode-completionpresent (the "present tail": ring wait + render + vsync) the term the
/// stage-2 presenter exists to shorten. Both instants are client-side, so no skew applies.
@Published var presentTailP50Ms = 0.0
@Published var presentTailP95Ms = 0.0
@Published var presentTailValid = false
@Published var hostNetworkP50Ms = 0.0
@Published var hostNetworkP95Ms = 0.0
@Published var hostNetworkValid = false
@Published var hostNetworkSkewCorrected = false
/// End-to-end = captureon-glass, measured directly per frame (never summed from the stages)
/// the HUD headline. Only the stage-2 presenter can stamp it (it owns decode + a
/// CAMetalLayer/display-link present); stays invalid under stage-1, where the layer presents
/// internally with no per-frame callback.
@Published var endToEndP50Ms = 0.0
@Published var endToEndP95Ms = 0.0
@Published var endToEndValid = false
@Published var endToEndSkewCorrected = false
/// The client-local stage terms of the HUD's equation line (single clock, no skew; p50 only):
/// decode = receiveddecoded, display = decodedon-glass (ring wait + render + vsync the
/// term the stage-2 presenter exists to shorten).
@Published var decodeP50Ms = 0.0
@Published var decodeValid = false
@Published var displayP50Ms = 0.0
@Published var displayValid = false
/// Unrecoverable network frame drops in the last window (FEC couldn't rebuild them) and their
/// share of frames offered, `lost/(received+lost)`. The HUD hides the line while zero.
@Published var lostFrames = 0
@Published var lostPct = 0.0
/// Mirrors StreamView's capture state (it owns the input capture; this drives the
/// HUD's "click to capture" / " releases" hint).
@Published var mouseCaptured = false
let meter = FrameMeter()
/// Capturereceived (the host+network stage), fed per AU at receipt by the stream view's
/// onFrame under both presenters.
let latency = LatencyMeter()
/// Fed by the stage-2 presenter's display link (capturepresent). Passed to StreamView.
let presentLatency = LatencyMeter()
/// Fed by the same present stamp (decode-completionpresent). Passed to StreamView.
let presentTail = LatencyMeter()
/// The stage-2 meters, passed to StreamView: end-to-end (captureon-glass, stamped at
/// present), decode (receiveddecoded), display (decodedon-glass).
let endToEnd = LatencyMeter()
let decodeStage = LatencyMeter()
let displayStage = LatencyMeter()
/// Cumulative reassembler-drop counter at the last stats drain (per-window `lost` delta).
private var lastFramesDropped: UInt64 = 0
private var statsTimer: Timer?
private var audio: SessionAudio?
private var gamepadCapture: GamepadCapture?
@@ -281,7 +295,12 @@ final class SessionModel: ObservableObject {
phase = .idle
fps = 0
mbps = 0
latencyValid = false
hostNetworkValid = false
endToEndValid = false
decodeValid = false
displayValid = false
lostFrames = 0
lostPct = 0
mouseCaptured = false
}
@@ -321,6 +340,7 @@ final class SessionModel: ObservableObject {
}
private func startStatsTimer() {
lastFramesDropped = 0 // a fresh connection's cumulative drop counter starts at 0
let timer = Timer(timeInterval: 1.0, repeats: true) { [weak self] _ in
guard let self else { return }
Task { @MainActor in
@@ -328,28 +348,41 @@ final class SessionModel: ObservableObject {
self.fps = frames
self.mbps = Double(bytes) * 8 / 1_000_000
self.totalFrames = total
// Per-window `lost` = the delta of the connector's cumulative reassembler-drop
// counter (0 after close treat a rewind as no loss rather than underflowing).
let dropped = self.connection?.framesDropped() ?? 0
let lost = dropped >= self.lastFramesDropped
? Int(dropped - self.lastFramesDropped) : 0
self.lastFramesDropped = dropped
self.lostFrames = lost
self.lostPct = lost > 0 ? Double(lost) / Double(frames + lost) * 100 : 0
if let lat = self.latency.drain() {
self.latencyP50Ms = lat.p50Ms
self.latencyP95Ms = lat.p95Ms
self.latencySkewCorrected = lat.skewCorrected
self.latencyValid = true
self.hostNetworkP50Ms = lat.p50Ms
self.hostNetworkP95Ms = lat.p95Ms
self.hostNetworkSkewCorrected = lat.skewCorrected
self.hostNetworkValid = true
} else {
self.latencyValid = false
self.hostNetworkValid = false
}
if let p = self.presentLatency.drain() {
self.presentLatencyP50Ms = p.p50Ms
self.presentLatencyP95Ms = p.p95Ms
self.presentLatencySkewCorrected = p.skewCorrected
self.presentLatencyValid = true
if let e = self.endToEnd.drain() {
self.endToEndP50Ms = e.p50Ms
self.endToEndP95Ms = e.p95Ms
self.endToEndSkewCorrected = e.skewCorrected
self.endToEndValid = true
} else {
self.presentLatencyValid = false
self.endToEndValid = false
}
if let t = self.presentTail.drain() {
self.presentTailP50Ms = t.p50Ms
self.presentTailP95Ms = t.p95Ms
self.presentTailValid = true
if let d = self.decodeStage.drain() {
self.decodeP50Ms = d.p50Ms
self.decodeValid = true
} else {
self.presentTailValid = false
self.decodeValid = false
}
if let d = self.displayStage.drain() {
self.displayP50Ms = d.p50Ms
self.displayValid = true
} else {
self.displayValid = false
}
}
}
@@ -1,5 +1,7 @@
// The streaming overlay HUD: mode + fps/throughput, the captureclient (and, under the stage-2
// presenter, capturepresent) latency lines, the platform input hint, and disconnect.
// The streaming overlay HUD: mode + fps/throughput, the unified latency lines
// (design/stats-unification.md end-to-end headline + the stage equation under stage-2, the
// capturereceived headline under the stage-1 fallback), the loss counter, the platform input
// hint, and disconnect.
import PunktfunkKit
import SwiftUI
@@ -18,24 +20,32 @@ struct StreamHUDView: View {
Text("\(connection.width)×\(connection.height)@\(connection.refreshHz) \(model.fps) fps \(model.mbps, specifier: "%.1f") Mb/s")
.font(.system(.caption, design: .monospaced))
}
if model.latencyValid {
// Captureclient-receipt (skew-corrected); excludes the layer's decode+present
// see LatencyMeter. "(same-host)" when the host didn't answer the skew handshake.
Text("capture→client \(model.latencyP50Ms, specifier: "%.1f")/\(model.latencyP95Ms, specifier: "%.1f") ms p50/p95\(model.latencySkewCorrected ? "" : " (same-host)")")
if model.endToEndValid {
// Stage-2: the end-to-end headline (captureon-glass, measured directly, skew-
// corrected) "(same-host clock)" when the host didn't answer the skew handshake.
Text("end-to-end \(model.endToEndP50Ms, specifier: "%.1f") ms p50 · \(model.endToEndP95Ms, specifier: "%.1f") p95 · capture→on-glass\(model.endToEndSkewCorrected ? "" : " (same-host clock)")")
.font(.system(.caption2, design: .monospaced))
.foregroundStyle(.secondary)
// The equation: the three stages tiling the headline interval (per-window p50s
// they only approximately sum to the directly-measured total).
if model.hostNetworkValid && model.decodeValid && model.displayValid {
Text("= host+network \(model.hostNetworkP50Ms, specifier: "%.1f") + decode \(model.decodeP50Ms, specifier: "%.1f") + display \(model.displayP50Ms, specifier: "%.1f")")
.font(.system(.caption2, design: .monospaced))
.foregroundStyle(.secondary)
}
} else if model.hostNetworkValid {
// Stage-1 fallback presenter: the layer decodes + presents internally with no
// per-frame stamp, so the honest headline ends at receipt and there is no
// equation line (host+network is the whole measured interval).
Text("capture→received \(model.hostNetworkP50Ms, specifier: "%.1f") ms p50 · \(model.hostNetworkP95Ms, specifier: "%.1f") p95\(model.hostNetworkSkewCorrected ? "" : " (same-host clock)")")
.font(.system(.caption2, design: .monospaced))
.foregroundStyle(.secondary)
}
if model.presentLatencyValid {
// Capturepresent (glass-to-glass, modulo host rendercapture) stage-2 presenter
// only; stage-1's layer presents internally with no per-frame stamp.
Text("capture→present \(model.presentLatencyP50Ms, specifier: "%.1f")/\(model.presentLatencyP95Ms, specifier: "%.1f") ms p50/p95\(model.presentLatencySkewCorrected ? "" : " (same-host)")")
.font(.system(.caption2, design: .monospaced))
.foregroundStyle(.secondary)
}
if model.presentTailValid {
// Decodepresent (the client-local "present tail": ring wait + render + vsync)
// the term the stage-2 presenter shortens; no skew applies (one clock).
Text("decode→present \(model.presentTailP50Ms, specifier: "%.1f")/\(model.presentTailP95Ms, specifier: "%.1f") ms p50/p95")
if model.lostFrames > 0 {
// Unrecoverable network drops this window; hidden while the link is clean.
// String(format:) rather than specifier interpolation: the literal % would
// otherwise land in the LocalizedStringKey's format string as a bogus conversion.
Text(String(format: "lost %d (%.1f%%)", model.lostFrames, model.lostPct))
.font(.system(.caption2, design: .monospaced))
.foregroundStyle(.secondary)
}
@@ -310,10 +310,11 @@ extension SettingsView {
Text("Video presenter · debug")
} footer: {
Text("Stage 2 (default) decodes explicitly and presents through Metal with a display "
+ "link — it adds a capture→present (glass-to-glass) latency line in the HUD and "
+ "self-recovers from decode stalls. Stage 1 feeds compressed video straight to the "
+ "system display layer; it freezes on a lost HEVC reference frame, so it's a debug "
+ "fallback only. Applies from the next session.")
+ "link — it gives the HUD the end-to-end (capture→on-glass) headline with the "
+ "host+network/decode/display stage equation and self-recovers from decode "
+ "stalls. Stage 1 feeds compressed video straight to the system display layer; "
+ "it freezes on a lost HEVC reference frame, so it's a debug fallback only. "
+ "Applies from the next session.")
.font(.geist(12, relativeTo: .caption))
.foregroundStyle(.secondary)
}
@@ -35,6 +35,10 @@ public struct AccessUnit: Sendable {
public let ptsNs: UInt64
public let frameIndex: UInt32
public let flags: UInt32
/// Client `CLOCK_REALTIME` instant the AU was handed over by the core (post-FEC, decrypted)
/// the **received** measurement point of design/stats-unification.md. The decode stage is
/// `decodedNs - receivedNs`, both client-local (no skew offset applies).
public let receivedNs: Int64
}
/// One Opus audio packet (48 kHz stereo, 5 ms frames) decode with AVAudioConverter
@@ -419,9 +423,13 @@ public final class PunktfunkConnection {
case statusOK:
guard let base = frame.data, frame.len > 0 else { return nil }
let data = Data(bytes: base, count: Int(frame.len)) // copy: ptr valid only until next call
var ts = timespec()
clock_gettime(CLOCK_REALTIME, &ts)
let receivedNs = Int64(ts.tv_sec) * 1_000_000_000 + Int64(ts.tv_nsec)
return AccessUnit(
data: data, ptsNs: frame.pts_ns,
frameIndex: frame.frame_index, flags: frame.flags)
frameIndex: frame.frame_index, flags: frame.flags,
receivedNs: receivedNs)
case statusNoFrame:
return nil
case statusClosed:
@@ -1,23 +1,25 @@
// Per-frame latency sampler for the live HUD: records capture->client-receipt latency and drains
// percentiles on demand. NSLock rather than an actor the writer is the non-async pump/arrival
// path (same pattern as the app's FrameMeter).
// Per-frame latency-stage sampler for the live HUD: records one interval per frame (an end
// instant minus a start instant, both CLOCK_REALTIME ns) and drains percentiles on demand.
// NSLock rather than an actor the writers are the non-async pump/decode/present paths (same
// pattern as the app's FrameMeter).
import Foundation
/// Samples the **capture->client-receipt** latency of each access unit and reports percentiles.
/// Samples one **latency stage** per frame and reports percentiles. One instance per stage of the
/// unified stats model (design/stats-unification.md):
///
/// The latency is `now - pts_ns`, where `pts_ns` is the host's capture wall clock (the AU's pts) and
/// `now` is the client's `CLOCK_REALTIME` instant the AU was received, shifted by the connect-time
/// **clock-skew offset** (`PunktfunkConnection.clockOffsetNs`, host minus client) so the difference
/// is valid across machines. `offsetNs == 0` means an old host that didn't answer the skew handshake
/// (or genuinely synced clocks) the number is then only meaningful same-host.
/// - `host+network` = capturereceived: `record(ptsNs:offsetNs:)` at AU receipt.
/// - `decode` = receiveddecoded and `display` = decodeddisplayed: client-local single-clock
/// stages `record(ptsNs:atNs:offsetNs:)` with the start instant as `ptsNs` and `offsetNs: 0`.
/// - `end-to-end` = capturedisplayed, measured directly (never summed from the stages):
/// `record(ptsNs:atNs:offsetNs:)` at present.
///
/// SCOPE (stage-1 presenter): this covers host capture -> encode -> FEC -> network -> reassembly ->
/// decrypt -> handed to the presenter. It does **not** include the on-device VideoToolbox decode or
/// the `AVSampleBufferDisplayLayer` present that layer decodes and presents compressed samples
/// internally with no per-frame callback. True decode->present (the full glass-to-glass) needs the
/// stage-2 presenter (`VTDecompressionSession` decode-completion + `CAMetalLayer`/display-link
/// present); this meter is the substrate it will extend.
/// For the host-anchored intervals (capture) the sample is `end + offset - pts_ns`, where
/// `pts_ns` is the host's capture wall clock (the AU's pts) and the connect-time **clock-skew
/// offset** (`PunktfunkConnection.clockOffsetNs`, host minus client) makes the difference valid
/// across machines. `offsetNs == 0` means an old host that didn't answer the skew handshake (or
/// genuinely synced clocks) the number is then only meaningful same-host, and the HUD tags the
/// end-to-end line `(same-host clock)`.
public final class LatencyMeter: @unchecked Sendable {
private let lock = NSLock()
private var samplesUs: [Int64] = []
@@ -34,12 +36,16 @@ public final class LatencyMeter: @unchecked Sendable {
record(ptsNs: ptsNs, atNs: nowNs, offsetNs: offsetNs)
}
/// Record one frame whose latency is `atNs + offsetNs - ptsNs` an EXPLICIT client instant
/// rather than now. The stage-2 presenter uses this to stamp capturepresent at the display
/// link's target present time (not the moment the present call ran). All in `CLOCK_REALTIME`.
/// Record one frame whose sample is `atNs + offsetNs - ptsNs` an EXPLICIT end instant
/// rather than now. `ptsNs` is the stage's start point: the AU pts for the host-anchored
/// intervals, or a client stamp (receivedNs / decodedNs, with `offsetNs: 0`) for the local
/// decode/display stages. The stage-2 presenter stamps its present-side samples at the
/// display link's target present time (not the moment the present call ran). All in
/// `CLOCK_REALTIME`.
public func record(ptsNs: UInt64, atNs: Int64, offsetNs: Int64) {
let latNs = atNs &+ offsetNs &- Int64(bitPattern: ptsNs)
// Drop absurd values (a clock step, a wildly wrong offset, or garbage pts).
// Drop absurd values (a clock step, a wildly wrong offset, garbage pts, or a stage whose
// start stamp is missing/after its end) samples are clamped to (0, 10 s).
guard latNs > 0, latNs < 10_000_000_000 else { return }
lock.lock()
samplesUs.append(latNs / 1000)
@@ -38,8 +38,9 @@ final class SessionPresenter {
func start(
connection: PunktfunkConnection,
baseLayer: AVSampleBufferDisplayLayer,
presentMeter: LatencyMeter?,
presentTailMeter: LatencyMeter? = nil,
endToEndMeter: LatencyMeter?,
decodeMeter: LatencyMeter? = nil,
displayMeter: LatencyMeter? = nil,
makeDisplayLink: (AnyObject, Selector) -> CADisplayLink,
onFrame: (@Sendable (AccessUnit) -> Void)?,
onSessionEnd: (@Sendable () -> Void)?
@@ -59,7 +60,8 @@ final class SessionPresenter {
#endif
if !forceStage1,
let pipeline = Stage2Pipeline(
presentMeter: presentMeter, presentTailMeter: presentTailMeter) {
endToEndMeter: endToEndMeter, decodeMeter: decodeMeter,
displayMeter: displayMeter) {
let metal = pipeline.layer
// The opaque metal layer composites OVER the AVSampleBufferDisplayLayer base, which
// sits idle (un-enqueued) in stage-2. contentsScale + frame are set in layout().
@@ -1,7 +1,8 @@
// Stage-2 presenter orchestrator: a pump thread pulls AUs VideoDecoder; the decoder's async output
// drops the newest decoded frame into a 1-slot ring; the hosting view's display link calls `renderTick`
// once per vsync to draw + present the newest ready frame and stamp capturepresent. Mirrors
// StreamPump's lifecycle (one per start; cancel is permanent).
// once per vsync to draw + present the newest ready frame and stamp the unified latency stages
// (end-to-end captureon-glass, plus the decode and display stage terms
// design/stats-unification.md). Mirrors StreamPump's lifecycle (one per start; cancel is permanent).
//
// Threading: the pump runs on its own thread; the decoder callback on a VT thread; `renderTick` +
// `start`/`stop` on the MAIN thread (the view's CADisplayLink fires there). Only the ring (lock-guarded)
@@ -40,8 +41,8 @@ public final class Stage2Pipeline {
private let ring = ReadyRing()
private let presenter: MetalVideoPresenter
private let decoder: VideoDecoder
private let presentMeter: LatencyMeter?
private let presentTailMeter: LatencyMeter?
private let endToEndMeter: LatencyMeter?
private let displayMeter: LatencyMeter?
private let recovery = KeyframeRecovery()
private var token = StopFlag()
private var offsetNs: Int64 = 0
@@ -56,28 +57,41 @@ public final class Stage2Pipeline {
/// The Metal layer the hosting view installs + sizes.
public var layer: CAMetalLayer { presenter.layer }
/// `presentMeter` records capturepresent (the glass-to-glass term); `presentTailMeter`
/// records decode-completionpresent (the ring wait + render the tail stage-2 exists to
/// shorten). Both optional: metering never gates the presenter choice. Returns nil if Metal
/// can't be set up (headless / no GPU) caller falls back to the stage-1 presenter.
public init?(presentMeter: LatencyMeter?, presentTailMeter: LatencyMeter? = nil) {
/// Unified-stats meters (design/stats-unification.md): `endToEndMeter` records the headline
/// end-to-end (captureon-glass, skew-corrected); `decodeMeter` the decode stage
/// (receiveddecoded); `displayMeter` the display stage (decodedon-glass, the ring wait +
/// render + vsync the tail stage-2 exists to shorten). All optional: metering never gates
/// the presenter choice. Returns nil if Metal can't be set up (headless / no GPU) caller
/// falls back to the stage-1 presenter.
public init?(
endToEndMeter: LatencyMeter?,
decodeMeter: LatencyMeter? = nil,
displayMeter: LatencyMeter? = nil
) {
guard let presenter = MetalVideoPresenter.make() else { return nil }
self.presenter = presenter
self.presentMeter = presentMeter
self.presentTailMeter = presentTailMeter
self.endToEndMeter = endToEndMeter
self.displayMeter = displayMeter
let ring = ring
let recovery = recovery
self.decoder = VideoDecoder(
onDecoded: { ring.submit($0) },
onDecoded: { frame in
// Decode stage = receiveddecoded, both client CLOCK_REALTIME (offset 0 no
// skew applies). Stamped at decode completion, so it covers every decoded frame,
// including ones the newest-wins ring drops before present.
decodeMeter?.record(
ptsNs: UInt64(frame.receivedNs), atNs: frame.decodedNs, offsetNs: 0)
ring.submit(frame)
},
// Async decode failure (a bad P-frame referencing a lost/corrupt IDR): the pump resets to
// re-gate on the next IDR, and we ask the host to send one now (infinite GOP it wouldn't
// otherwise come soon). Throttled in KeyframeRecovery.
onDecodeError: { _ in recovery.request() })
}
/// Start pulling AUs into the decoder. MAIN THREAD. `onFrame` fires per AU at receipt (captureclient
/// meter, exactly as stage-1); `onSessionEnd` on close. `clockOffsetNs` (host minus client) makes the
/// present stamp cross-machine valid.
/// Start pulling AUs into the decoder. MAIN THREAD. `onFrame` fires per AU at receipt (the
/// host+network / capturereceived meter, exactly as stage-1); `onSessionEnd` on close.
/// `clockOffsetNs` (host minus client) makes the end-to-end stamp cross-machine valid.
public func start(
connection: PunktfunkConnection,
onFrame: (@Sendable (AccessUnit) -> Void)?,
@@ -174,14 +188,16 @@ public final class Stage2Pipeline {
public func renderTick(targetPresentNs: Int64) {
guard let frame = ring.take() else { return }
let offsetNs = offsetNs
let presentMeter = presentMeter
let presentTailMeter = presentTailMeter
let endToEndMeter = endToEndMeter
let displayMeter = displayMeter
let rendered = presenter.render(frame.pixelBuffer, isHDR: frame.isHDR) { presentedNs in
let atNs = presentedNs ?? targetPresentNs
presentMeter?.record(ptsNs: frame.ptsNs, atNs: atNs, offsetNs: offsetNs)
// Present tail = decode-completion on-glass. Both instants are client
// CLOCK_REALTIME, so no skew offset applies.
presentTailMeter?.record(ptsNs: UInt64(frame.decodedNs), atNs: atNs, offsetNs: 0)
// End-to-end = captureon-glass, measured directly (skew-corrected via the
// connect-time clock offset) the HUD headline.
endToEndMeter?.record(ptsNs: frame.ptsNs, atNs: atNs, offsetNs: offsetNs)
// Display stage = decoded on-glass. Both instants are client CLOCK_REALTIME,
// so no skew offset applies.
displayMeter?.record(ptsNs: UInt64(frame.decodedNs), atNs: atNs, offsetNs: 0)
}
if !rendered { ring.putBack(frame) }
}
@@ -61,7 +61,7 @@ public enum Stage444Probe {
guard created == noErr, let session else { return false }
defer { VTDecompressionSessionInvalidate(session) }
let au = AccessUnit(data: data, ptsNs: 0, frameIndex: 0, flags: 0)
let au = AccessUnit(data: data, ptsNs: 0, frameIndex: 0, flags: 0, receivedNs: 0)
guard let sample = AnnexB.sampleBuffer(au: au, format: format, codec: .hevc) else { return false }
var produced: OSType = 0
@@ -15,6 +15,10 @@ import VideoToolbox
public struct ReadyFrame: @unchecked Sendable {
/// Host capture clock (the AU's pts), in nanoseconds.
public let ptsNs: UInt64
/// Client `CLOCK_REALTIME` instant the AU was received (`AccessUnit.receivedNs`, threaded
/// through the decode via the frame refcon), in nanoseconds. 0 when unknown (a caller that
/// didn't stamp receipt) the decode-stage meter then drops the sample via its sanity guard.
public let receivedNs: Int64
/// Client `CLOCK_REALTIME` instant decode completed, in nanoseconds.
public let decodedNs: Int64
/// The decoded image 8-bit NV12 biplanar (SDR) or 10-bit P010 biplanar (HDR), Metal-compatible.
@@ -25,13 +29,16 @@ public struct ReadyFrame: @unchecked Sendable {
}
/// The C output callback can't capture context, so VideoToolbox hands it the refcon we set at
/// session creation a pointer back to the owning `VideoDecoder`.
/// session creation a pointer back to the owning `VideoDecoder`. The per-frame refcon carries
/// the AU's `receivedNs` as a pointer bit pattern (a scalar smuggled through the C void*, never
/// dereferenced) so the decode stage can be computed against decode-completion.
private let decoderOutputCallback: VTDecompressionOutputCallback = {
refcon, _, status, _, imageBuffer, pts, _ in
refcon, frameRefcon, status, _, imageBuffer, pts, _ in
guard let refcon else { return }
let receivedNs = frameRefcon.map { Int64(Int(bitPattern: $0)) } ?? 0
Unmanaged<VideoDecoder>.fromOpaque(refcon)
.takeUnretainedValue()
.handleDecoded(status: status, imageBuffer: imageBuffer, pts: pts)
.handleDecoded(status: status, imageBuffer: imageBuffer, pts: pts, receivedNs: receivedNs)
}
/// Owns a `VTDecompressionSession` rebuilt whenever the format description changes (every IDR /
@@ -112,7 +119,9 @@ public final class VideoDecoder: @unchecked Sendable {
session,
sampleBuffer: sample,
flags: [._EnableAsynchronousDecompression],
frameRefcon: nil,
// The AU's receipt instant rides through as a bit pattern (nil for 0 the output
// callback maps that back to 0); the callback needs it to stamp the decode stage.
frameRefcon: UnsafeMutableRawPointer(bitPattern: Int(au.receivedNs)),
infoFlagsOut: &infoOut)
lock.unlock()
if status != noErr {
@@ -218,8 +227,11 @@ public final class VideoDecoder: @unchecked Sendable {
return true
}
/// VT thread. Stamp decode-completion and enqueue, or report the error.
fileprivate func handleDecoded(status: OSStatus, imageBuffer: CVImageBuffer?, pts: CMTime) {
/// VT thread. Stamp decode-completion and enqueue, or report the error. `receivedNs` is the
/// AU's receipt instant threaded through the frame refcon (0 = unknown).
fileprivate func handleDecoded(
status: OSStatus, imageBuffer: CVImageBuffer?, pts: CMTime, receivedNs: Int64
) {
guard status == noErr, let imageBuffer else {
onDecodeError(status)
return
@@ -242,6 +254,8 @@ public final class VideoDecoder: @unchecked Sendable {
|| fmt == kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange
|| fmt == kCVPixelFormatType_444YpCbCr10BiPlanarFullRange
onDecoded(
ReadyFrame(ptsNs: ptsNs, decodedNs: decodedNs, pixelBuffer: imageBuffer, isHDR: isHDR))
ReadyFrame(
ptsNs: ptsNs, receivedNs: receivedNs, decodedNs: decodedNs,
pixelBuffer: imageBuffer, isHDR: isHDR))
}
}
@@ -85,39 +85,45 @@ public struct StreamView: NSViewRepresentable {
private let onCaptureChange: ((Bool) -> Void)?
private let onFrame: (@Sendable (AccessUnit) -> Void)?
private let onSessionEnd: (@Sendable () -> Void)?
private let presentMeter: LatencyMeter?
private let presentTailMeter: LatencyMeter?
private let endToEndMeter: LatencyMeter?
private let decodeMeter: LatencyMeter?
private let displayMeter: LatencyMeter?
/// `onFrame`/`onSessionEnd` fire on the pump thread hop to the main actor for UI.
/// `captureEnabled: false` disables input capture entirely while UI (e.g. a trust
/// prompt) is layered over the stream; flipping it to true auto-engages capture
/// once. `onCaptureChange` (main thread) reports engage/release drive the HUD's
/// "click to capture" / " releases" hint with it. `presentMeter` records capturepresent
/// and `presentTailMeter` decodepresent when the stage-2 presenter is active.
/// "click to capture" / " releases" hint with it. The meters record the unified latency
/// stages when the stage-2 presenter is active (design/stats-unification.md):
/// `endToEndMeter` captureon-glass, `decodeMeter` receiveddecoded, `displayMeter`
/// decodedon-glass.
public init(
connection: PunktfunkConnection,
captureEnabled: Bool = true,
onCaptureChange: ((Bool) -> Void)? = nil,
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
onSessionEnd: (@Sendable () -> Void)? = nil,
presentMeter: LatencyMeter? = nil,
presentTailMeter: LatencyMeter? = nil
endToEndMeter: LatencyMeter? = nil,
decodeMeter: LatencyMeter? = nil,
displayMeter: LatencyMeter? = nil
) {
self.connection = connection
self.captureEnabled = captureEnabled
self.onCaptureChange = onCaptureChange
self.onFrame = onFrame
self.onSessionEnd = onSessionEnd
self.presentMeter = presentMeter
self.presentTailMeter = presentTailMeter
self.endToEndMeter = endToEndMeter
self.decodeMeter = decodeMeter
self.displayMeter = displayMeter
}
public func makeNSView(context: Context) -> StreamLayerView {
let view = StreamLayerView()
view.onCaptureChange = onCaptureChange
view.captureEnabled = captureEnabled
view.presentMeter = presentMeter
view.presentTailMeter = presentTailMeter
view.endToEndMeter = endToEndMeter
view.decodeMeter = decodeMeter
view.displayMeter = displayMeter
view.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
return view
}
@@ -125,8 +131,9 @@ public struct StreamView: NSViewRepresentable {
public func updateNSView(_ view: StreamLayerView, context: Context) {
view.onCaptureChange = onCaptureChange
view.captureEnabled = captureEnabled
view.presentMeter = presentMeter
view.presentTailMeter = presentTailMeter
view.endToEndMeter = endToEndMeter
view.decodeMeter = decodeMeter
view.displayMeter = displayMeter
// SwiftUI reuses the NSView across state changes repoint the pump only when the
// connection identity actually changed.
if view.connection !== connection {
@@ -141,10 +148,11 @@ public struct StreamView: NSViewRepresentable {
public final class StreamLayerView: NSView {
private let displayLayer = AVSampleBufferDisplayLayer()
/// Record capturepresent / decodepresent when the stage-2 presenter is active.
/// Consulted at start().
var presentMeter: LatencyMeter?
var presentTailMeter: LatencyMeter?
/// Record the unified latency stages (end-to-end / decode / display) when the stage-2
/// presenter is active. Consulted at start().
var endToEndMeter: LatencyMeter?
var decodeMeter: LatencyMeter?
var displayMeter: LatencyMeter?
/// The shared presenter stack: stage-2 (CAMetalLayer sublayer + display link) with the
/// stage-1 StreamPump displayLayer path as the Metal-unavailable / DEBUG fallback.
private let presenter = SessionPresenter()
@@ -571,8 +579,9 @@ public final class StreamLayerView: NSView {
presenter.start(
connection: connection,
baseLayer: displayLayer,
presentMeter: presentMeter,
presentTailMeter: presentTailMeter,
endToEndMeter: endToEndMeter,
decodeMeter: decodeMeter,
displayMeter: displayMeter,
makeDisplayLink: { displayLink(target: $0, selector: $1) },
onFrame: onFrame,
onSessionEnd: onSessionEnd)
@@ -50,8 +50,9 @@ public struct StreamView: UIViewControllerRepresentable {
private let onCaptureChange: ((Bool) -> Void)?
private let onFrame: (@Sendable (AccessUnit) -> Void)?
private let onSessionEnd: (@Sendable () -> Void)?
private let presentMeter: LatencyMeter?
private let presentTailMeter: LatencyMeter?
private let endToEndMeter: LatencyMeter?
private let decodeMeter: LatencyMeter?
private let displayMeter: LatencyMeter?
public init(
connection: PunktfunkConnection,
@@ -59,24 +60,27 @@ public struct StreamView: UIViewControllerRepresentable {
onCaptureChange: ((Bool) -> Void)? = nil,
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
onSessionEnd: (@Sendable () -> Void)? = nil,
presentMeter: LatencyMeter? = nil,
presentTailMeter: LatencyMeter? = nil
endToEndMeter: LatencyMeter? = nil,
decodeMeter: LatencyMeter? = nil,
displayMeter: LatencyMeter? = nil
) {
self.connection = connection
self.captureEnabled = captureEnabled
self.onCaptureChange = onCaptureChange
self.onFrame = onFrame
self.onSessionEnd = onSessionEnd
self.presentMeter = presentMeter
self.presentTailMeter = presentTailMeter
self.endToEndMeter = endToEndMeter
self.decodeMeter = decodeMeter
self.displayMeter = displayMeter
}
public func makeUIViewController(context: Context) -> StreamViewController {
let controller = StreamViewController()
controller.onCaptureChange = onCaptureChange
controller.captureEnabled = captureEnabled
controller.presentMeter = presentMeter
controller.presentTailMeter = presentTailMeter
controller.endToEndMeter = endToEndMeter
controller.decodeMeter = decodeMeter
controller.displayMeter = displayMeter
controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
return controller
}
@@ -84,8 +88,9 @@ public struct StreamView: UIViewControllerRepresentable {
public func updateUIViewController(_ controller: StreamViewController, context: Context) {
controller.onCaptureChange = onCaptureChange
controller.captureEnabled = captureEnabled
controller.presentMeter = presentMeter
controller.presentTailMeter = presentTailMeter
controller.endToEndMeter = endToEndMeter
controller.decodeMeter = decodeMeter
controller.displayMeter = displayMeter
if controller.connection !== connection {
controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
}
@@ -101,10 +106,11 @@ public struct StreamView: UIViewControllerRepresentable {
public final class StreamViewController: UIViewController {
public private(set) var connection: PunktfunkConnection?
private var observers: [NSObjectProtocol] = []
/// Record capturepresent / decodepresent when the stage-2 presenter is active.
/// Consulted at start().
var presentMeter: LatencyMeter?
var presentTailMeter: LatencyMeter?
/// Record the unified latency stages (end-to-end / decode / display) when the stage-2
/// presenter is active. Consulted at start().
var endToEndMeter: LatencyMeter?
var decodeMeter: LatencyMeter?
var displayMeter: LatencyMeter?
/// The shared presenter stack: stage-2 (CAMetalLayer sublayer + display link) with the
/// stage-1 StreamPump displayLayer path as the Metal-unavailable / DEBUG fallback.
private let presenter = SessionPresenter()
@@ -285,8 +291,9 @@ public final class StreamViewController: UIViewController {
presenter.start(
connection: connection,
baseLayer: streamView.displayLayer,
presentMeter: presentMeter,
presentTailMeter: presentTailMeter,
endToEndMeter: endToEndMeter,
decodeMeter: decodeMeter,
displayMeter: displayMeter,
makeDisplayLink: { CADisplayLink(target: $0, selector: $1) },
onFrame: onFrame,
onSessionEnd: onSessionEnd)
@@ -1,6 +1,10 @@
// Unit tests for LatencyMeter: percentiles, the skew-corrected flag, reset-on-drain, and the
// absurd-value guard. Latencies are constructed by stamping a pts a known interval in the past, so
// the result is that interval plus the (tiny) clock advance between reads asserted with tolerance.
// Unit tests for LatencyMeter (one instance per unified-stats stage see
// design/stats-unification.md): percentiles, the skew-corrected flag, reset-on-drain, the
// absurd-value guard, and the explicit-instant stage form (record(ptsNs:atNs:offsetNs:), used for
// the client-local decode/display stages and the at-present end-to-end stamp). Receipt-path
// latencies are constructed by stamping a pts a known interval in the past, so the result is that
// interval plus the (tiny) clock advance between reads asserted with tolerance; the explicit
// form is exact.
import Foundation
import XCTest
@@ -38,6 +42,26 @@ final class LatencyMeterTests: XCTestCase {
XCTAssertEqual(m.drain()?.skewCorrected, true)
}
func testExplicitStageRecordIsExact() {
let m = LatencyMeter()
// A client-local stage (decode: receiveddecoded) start instant as ptsNs, offset 0.
let receivedNs: Int64 = 1_000_000_000_000
m.record(ptsNs: UInt64(receivedNs), atNs: receivedNs + 3_000_000, offsetNs: 0)
guard let s = m.drain() else { return XCTFail("expected a sample") }
XCTAssertEqual(s.count, 1)
XCTAssertEqual(s.p50Ms, 3.0, "explicit instants make the sample exact")
XCTAssertFalse(s.skewCorrected, "local stages record with offset 0")
}
func testExplicitStageDropsNonPositiveInterval() {
let m = LatencyMeter()
// A stage whose start stamp is missing (0) or after its end must not pollute the window.
let decodedNs: Int64 = 1_000_000_000_000
m.record(ptsNs: 0, atNs: decodedNs, offsetNs: 0) // "start unknown" > 10 s dropped
m.record(ptsNs: UInt64(decodedNs + 1), atNs: decodedNs, offsetNs: 0) // negative dropped
XCTAssertNil(m.drain())
}
func testDropsAbsurdValues() {
let m = LatencyMeter()
let now = nowRealtimeNs()
@@ -31,7 +31,7 @@ final class Stage444Tests: XCTestCase {
let data = Data(Probe444Blobs.au444_8bit)
let format = try XCTUnwrap(
AnnexB.formatDescription(fromIDR: data, codec: .hevc), "the 4:4:4 blob must yield a format description")
let au = AccessUnit(data: data, ptsNs: 7_000_000, frameIndex: 0, flags: 0)
let au = AccessUnit(data: data, ptsNs: 7_000_000, frameIndex: 0, flags: 0, receivedNs: 0)
let box = FrameBox()
let done = DispatchSemaphore(value: 0)
@@ -38,7 +38,8 @@ final class VideoToolboxRoundTripTests: XCTestCase {
XCTAssertEqual(AnnexB.avcc(from: annexB, codec: .hevc), avccSample)
// 3) Sample buffer real decoder pixels.
let au = AccessUnit(data: annexB, ptsNs: 1_000_000, frameIndex: 0, flags: 0)
let au = AccessUnit(
data: annexB, ptsNs: 1_000_000, frameIndex: 0, flags: 0, receivedNs: 0)
let sample = try XCTUnwrap(AnnexB.sampleBuffer(au: au, format: rebuilt, codec: .hevc))
var session: VTDecompressionSession?
@@ -67,13 +68,14 @@ final class VideoToolboxRoundTripTests: XCTestCase {
}
/// Stage-2 decode half: the same known IDR through `VideoDecoder` assert its async output
/// callback fires with a CVPixelBuffer of the right dimensions, the pts round-trips, and
/// decode-completion is stamped.
/// callback fires with a CVPixelBuffer of the right dimensions, the pts and the receipt stamp
/// round-trip (the latter rides the frame refcon), and decode-completion is stamped.
func testVideoDecoderAsyncCallbackDeliversPixels() throws {
let (formatDesc, avccSample) = try encodeOneHEVCKeyframe()
let annexB = try annexBAU(formatDesc: formatDesc, avccSample: avccSample)
let format = try XCTUnwrap(AnnexB.formatDescription(fromIDR: annexB, codec: .hevc))
let au = AccessUnit(data: annexB, ptsNs: 42_000_000, frameIndex: 0, flags: 0)
let au = AccessUnit(
data: annexB, ptsNs: 42_000_000, frameIndex: 0, flags: 0, receivedNs: 41_000_000)
let box = FrameBox()
let done = DispatchSemaphore(value: 0)
@@ -100,6 +102,8 @@ final class VideoToolboxRoundTripTests: XCTestCase {
XCTAssertEqual(CVPixelBufferGetWidth(ready.pixelBuffer), width)
XCTAssertEqual(CVPixelBufferGetHeight(ready.pixelBuffer), height)
XCTAssertEqual(ready.ptsNs, 42_000_000, "pts round-trips through the decoder")
XCTAssertEqual(
ready.receivedNs, 41_000_000, "receivedNs round-trips through the frame refcon")
XCTAssertGreaterThan(ready.decodedNs, 0, "decode-completion is stamped")
}