From ff38933312d63e9a748de80739937b6f86f29398 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Wed, 15 Jul 2026 17:43:54 +0200 Subject: [PATCH] feat(core,apple,session): report decode latency from the Apple + Windows/Linux clients too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends 56f9c8c4 (the Automatic-bitrate decode signal, core + Android) to the remaining clients, so every platform caps Automatic at its real decoder limit instead of the network link ceiling — the fix for a fast LAN feeding a slower hardware decoder. - core/abi: punktfunk_connection_report_decode_us + _wants_decode_latency expose the NativeClient methods to the C-ABI embedders (regenerated punktfunk_core.h, additive only). - apple: PunktfunkConnection wrappers + Stage2Pipeline reports received→decoded from the VideoToolbox decode-completion callback — every decoded frame, before the newest-wins ring can drop the backlog. Stage-1 (AVSampleBufferDisplayLayer, no per-frame decode callback) stays network-only; stage-2 is the metered path. - windows/linux: the shared punktfunk-session client (pf-client-core) links core directly, so it calls the NativeClient methods — report received→decoded from the pump, gated on wants_decode_latency. Exact for the synchronous D3D11VA/software decode; received→submit (still the decoder-input backpressure signal) for the async Vulkan-Video path. Co-Authored-By: Claude Opus 4.8 --- .../Connection/PunktfunkConnection.swift | 24 ++++++++ .../PunktfunkKit/Video/Stage2Pipeline.swift | 31 ++++++++++ crates/pf-client-core/src/session.rs | 12 ++++ crates/punktfunk-core/src/abi.rs | 57 +++++++++++++++++++ include/punktfunk_core.h | 31 ++++++++++ 5 files changed, 155 insertions(+) diff --git a/clients/apple/Sources/PunktfunkKit/Connection/PunktfunkConnection.swift b/clients/apple/Sources/PunktfunkKit/Connection/PunktfunkConnection.swift index 8adc5955..807204a2 100644 --- a/clients/apple/Sources/PunktfunkKit/Connection/PunktfunkConnection.swift +++ b/clients/apple/Sources/PunktfunkKit/Connection/PunktfunkConnection.swift @@ -578,6 +578,30 @@ public final class PunktfunkConnection { return out } + /// Report one decoded frame's decode-stage latency, in microseconds (the AU leaving `nextAU` + /// through its VideoToolbox output). This feeds the Automatic bitrate controller's decode + /// signal — the only one that sees this device's decoder — so the rate is capped at the real + /// decode limit instead of climbing to the network link ceiling and choking the decoder. Cheap; + /// silently dropped after close. Only worth calling when `wantsDecodeLatency()` is true. + public func reportDecodeUs(_ us: UInt32) { + abiLock.lock() + defer { abiLock.unlock() } + guard let h = handle, !closeRequested else { return } + _ = punktfunk_connection_report_decode_us(h, us) + } + + /// Whether `reportDecodeUs` is worth calling this session: true only when the adaptive-bitrate + /// controller is armed (Automatic bitrate, non-PyroWave). Query once — constant for the session + /// — and skip the per-frame decode measurement entirely when it's false. False after close. + public func wantsDecodeLatency() -> Bool { + abiLock.lock() + defer { abiLock.unlock() } + guard let h = handle, !closeRequested else { return false } + var out = false + _ = punktfunk_connection_wants_decode_latency(h, &out) + return out + } + /// The currently active session mode (updated by accepted `requestMode` switches). public func currentMode() -> (width: UInt32, height: UInt32, refreshHz: UInt32) { abiLock.lock() diff --git a/clients/apple/Sources/PunktfunkKit/Video/Stage2Pipeline.swift b/clients/apple/Sources/PunktfunkKit/Video/Stage2Pipeline.swift index f3a09419..b829cdb0 100644 --- a/clients/apple/Sources/PunktfunkKit/Video/Stage2Pipeline.swift +++ b/clients/apple/Sources/PunktfunkKit/Video/Stage2Pipeline.swift @@ -250,6 +250,28 @@ private final class PresentDebugStats: @unchecked Sendable { } } +/// Bridges the VideoToolbox decode-completion callback to the core Automatic-bitrate controller's +/// decode signal. Created as a pipeline property so the decoder's `onDecoded` callback (built in +/// `init`, before the connection exists) can capture it, then `start` binds the live connection + +/// the arming flag once known — the same "reference captured in init, configured in start" shape as +/// `recovery`/`gate`. `record` runs on VideoToolbox's callback thread; `bind` runs once on the main +/// thread before the pump feeds the first AU, so the plain fields are safe (set-once, then read). +private final class DecodeReport: @unchecked Sendable { + private weak var connection: PunktfunkConnection? + private var enabled = false + func bind(_ connection: PunktfunkConnection) { + self.connection = connection + self.enabled = connection.wantsDecodeLatency() + } + /// Report received→decoded for one frame, in µs. Both stamps are client `CLOCK_REALTIME` + /// (no skew). Skips when the controller isn't armed, so it's free to call on every decode. + func record(receivedNs: Int64, decodedNs: Int64) { + guard enabled, let c = connection else { return } + let us = (decodedNs - receivedNs) / 1000 + if us > 0 { c.reportDecodeUs(UInt32(min(us, Int64(UInt32.max)))) } + } +} + public final class Stage2Pipeline { private let ring = ReadyRing() private let presenter: MetalVideoPresenter @@ -261,6 +283,9 @@ public final class Stage2Pipeline { private let decodeMeter: LatencyMeter? private let displayMeter: LatencyMeter? private let recovery = KeyframeRecovery() + /// Feeds the core Automatic-bitrate controller's decode signal from the decode callback; `start` + /// binds the live connection + arming flag (see DecodeReport). + private let decodeReport = DecodeReport() /// Post-loss freeze-until-reanchor gate (shared core policy via the C ABI). Created here seeded 0; /// `start` reseeds it to the live connection's drop count. Captured by the decoder callbacks /// (which withhold concealed frames) and driven by the pump (arm on a gap, poll per iteration). @@ -314,6 +339,7 @@ public final class Stage2Pipeline { let recovery = recovery let renderSignal = renderSignal let gate = gate + let decodeReport = decodeReport self.decoder = VideoDecoder( onDecoded: { frame in // Decode stage = received→decoded, both client CLOCK_REALTIME (offset 0 — no @@ -321,6 +347,10 @@ public final class Stage2Pipeline { // including ones the re-anchor gate withholds or the newest-wins ring drops. decodeMeter?.record( ptsNs: UInt64(frame.receivedNs), atNs: frame.decodedNs, offsetNs: 0) + // Same interval, reported to the core bitrate controller so Automatic caps at this + // device's real decode limit instead of the network link ceiling. Every decoded + // frame (not just presented ones), so a newest-wins drop can't hide the backlog. + decodeReport.record(receivedNs: frame.receivedNs, decodedNs: frame.decodedNs) // Freeze-until-reanchor: WITHHOLD a decoder-concealed post-loss frame (the gray/ // garbage VideoToolbox returns Ok for a reference-missing delta) — don't submit it, // so the CAMetalLayer keeps its last good drawable on glass. The gate lifts (returns @@ -349,6 +379,7 @@ public final class Stage2Pipeline { ) { offsetNs = connection.clockOffsetNs recovery.bind(connection) // arm host-keyframe recovery for this session + decodeReport.bind(connection) // arm the Automatic-bitrate decode signal for this session gate.reseed(framesDropped: connection.framesDropped()) // baseline the freeze to this session token = StopFlag() // fresh token per start — a stop is permanent (like StreamPump) diff --git a/crates/pf-client-core/src/session.rs b/crates/pf-client-core/src/session.rs index d6036e69..816412fc 100644 --- a/crates/pf-client-core/src/session.rs +++ b/crates/pf-client-core/src/session.rs @@ -352,6 +352,9 @@ fn pump( // corrected), `decode` = received→decoded (client-local). p50 per 1 s window. let mut hostnet_us: Vec = Vec::with_capacity(256); let mut decode_us: Vec = Vec::with_capacity(256); + // Adaptive bitrate: report the decode stage back to the core controller only when it's armed + // (Automatic, non-PyroWave). Constant for the session — resolve once, gate the per-frame call. + let wants_decode = connector.wants_decode_latency(); // Host/network split (Phase 2): frames awaiting their per-AU 0xCF host timing, // correlated by pts_ns. Bounded — an old host never sends any, so entries just age out. let mut pending_split: std::collections::VecDeque<(u64, u64)> = @@ -574,6 +577,15 @@ fn pump( decode_us.push(decoded_ns.saturating_sub(received_ns) / 1000); } } + // Adaptive bitrate: feed the decoder-backlog signal every frame (the network + // signals can't see the client's decoder). Uses the CPU-side decoded stamp: + // exact for the synchronous D3D11VA/software path; received→submit for the + // async Vulkan-Video path — still the decoder-input backpressure the rate + // controller needs, without the per-frame fence wait the HUD stat avoids. + if wants_decode { + let us = decoded_ns.saturating_sub(received_ns) / 1000; + connector.report_decode_us(us.min(u32::MAX as u64) as u32); + } } // The decoder produced nothing — under zero-reorder LOW_DELAY (one-in/one-out) that // means it's wedged on missing references with no reassembler drop to trigger diff --git a/crates/punktfunk-core/src/abi.rs b/crates/punktfunk-core/src/abi.rs index a1343825..be8757fc 100644 --- a/crates/punktfunk-core/src/abi.rs +++ b/crates/punktfunk-core/src/abi.rs @@ -2686,6 +2686,63 @@ pub unsafe extern "C" fn punktfunk_connection_frames_dropped( }) } +/// Report one decoded frame's decode-stage latency, in microseconds: the wall-clock elapsed from +/// the access unit leaving [`punktfunk_connection_next_au`] to its decoded output becoming +/// available (VideoToolbox/D3D11VA/… produced the frame). This feeds the "Automatic" bitrate +/// controller's decode signal — the only one that sees the client's own decoder, so the rate is +/// capped at the real decode limit instead of climbing to the network link ceiling and choking a +/// slower hardware decoder (a fast LAN feeding a mobile-class decoder). Measure from the AU pull, +/// NOT from the decoder-submit call, so decoder-input backpressure (the backlog) is included; +/// exclude the presenter's vsync wait so a paced/capped frame rate doesn't read as decode latency. +/// Cheap — the client may call it every frame; the controller ignores it unless armed (query +/// [`punktfunk_connection_wants_decode_latency`] once to skip the measurement entirely when it's not). +/// +/// # Safety +/// `c` is a valid connection handle. +#[cfg(feature = "quic")] +#[no_mangle] +pub unsafe extern "C" fn punktfunk_connection_report_decode_us( + c: *const PunktfunkConnection, + us: u32, +) -> PunktfunkStatus { + guard(|| { + let c = match unsafe { c.as_ref() } { + Some(c) => c, + None => return PunktfunkStatus::NullPointer, + }; + c.inner.report_decode_us(us); + PunktfunkStatus::Ok + }) +} + +/// Whether [`punktfunk_connection_report_decode_us`] is worth calling this session: writes 1 to +/// `out` only when the adaptive-bitrate controller is armed (Automatic bitrate, non-PyroWave), so a +/// client can skip the per-frame decode-latency measurement entirely for explicit-bitrate and +/// PyroWave sessions (where the signal is ignored). Constant for the session — query once. Writes 0 +/// on a NULL connection. +/// +/// # Safety +/// `c` is a valid connection handle; `out` is writable (NULL is skipped). +#[cfg(feature = "quic")] +#[no_mangle] +pub unsafe extern "C" fn punktfunk_connection_wants_decode_latency( + c: *const PunktfunkConnection, + out: *mut bool, +) -> PunktfunkStatus { + guard(|| { + let c = match unsafe { c.as_ref() } { + Some(c) => c, + None => return PunktfunkStatus::NullPointer, + }; + unsafe { + if !out.is_null() { + *out = c.inner.wants_decode_latency(); + } + } + PunktfunkStatus::Ok + }) +} + /// A speed-test measurement, filled by [`punktfunk_connection_probe_result`]. `done` is 0 until /// the host's end-of-burst report lands, then 1 (the numbers are final). `throughput_kbps` is the /// delivered wire throughput to drive a bitrate choice from; `loss_pct` is the link loss and diff --git a/include/punktfunk_core.h b/include/punktfunk_core.h index 91268555..edd65e9a 100644 --- a/include/punktfunk_core.h +++ b/include/punktfunk_core.h @@ -1884,6 +1884,37 @@ PunktfunkStatus punktfunk_connection_note_frame_index(const PunktfunkConnection PunktfunkStatus punktfunk_connection_frames_dropped(const PunktfunkConnection *c, uint64_t *out); #endif +#if defined(PUNKTFUNK_FEATURE_QUIC) +// Report one decoded frame's decode-stage latency, in microseconds: the wall-clock elapsed from +// the access unit leaving [`punktfunk_connection_next_au`] to its decoded output becoming +// available (VideoToolbox/D3D11VA/… produced the frame). This feeds the "Automatic" bitrate +// controller's decode signal — the only one that sees the client's own decoder, so the rate is +// capped at the real decode limit instead of climbing to the network link ceiling and choking a +// slower hardware decoder (a fast LAN feeding a mobile-class decoder). Measure from the AU pull, +// NOT from the decoder-submit call, so decoder-input backpressure (the backlog) is included; +// exclude the presenter's vsync wait so a paced/capped frame rate doesn't read as decode latency. +// Cheap — the client may call it every frame; the controller ignores it unless armed (query +// [`punktfunk_connection_wants_decode_latency`] once to skip the measurement entirely when it's not). +// +// # Safety +// `c` is a valid connection handle. +PunktfunkStatus punktfunk_connection_report_decode_us(const PunktfunkConnection *c, + uint32_t us); +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// Whether [`punktfunk_connection_report_decode_us`] is worth calling this session: writes 1 to +// `out` only when the adaptive-bitrate controller is armed (Automatic bitrate, non-PyroWave), so a +// client can skip the per-frame decode-latency measurement entirely for explicit-bitrate and +// PyroWave sessions (where the signal is ignored). Constant for the session — query once. Writes 0 +// on a NULL connection. +// +// # Safety +// `c` is a valid connection handle; `out` is writable (NULL is skipped). +PunktfunkStatus punktfunk_connection_wants_decode_latency(const PunktfunkConnection *c, + bool *out); +#endif + #if defined(PUNKTFUNK_FEATURE_QUIC) // Start a bandwidth speed test: ask the host to burst filler over the data plane at // `target_kbps` of goodput for `duration_ms` (each clamped host-side to ≤ 3 Gbps / ≤ 5 s),