From 8a18e130a2619b0b8054c366f588d3fd1fa0ce21 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Mon, 13 Jul 2026 01:21:25 +0200 Subject: [PATCH] feat(client): freeze-until-reanchor loss recovery on Android + Apple via shared core gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After unrecoverable loss the host keeps sending delta frames that reference a picture the client never received; hardware decoders conceal these as gray/ garbage with a success status. Linux already withheld them and held the last good frame until a proven clean re-anchor — this brings that behavior to the Android and Apple clients. Extract the Linux pump's freeze state machine into a shared `ReanchorGate` in punktfunk-core (reanchor.rs, 18 tests) exposed over the C ABI (ABI v6, additive — no wire change) for the Swift clients. Migrate the Linux/Deck pump (pf-client-core) onto it as the parity proof (no-op refactor). Then wire: - Android (decode.rs, both sync + async loops): arm on the frame-index gap, a pts-keyed flag map carries the wire flags to the output-buffer release, fold the gate per drained output, gate.poll replaces the dropped-climb block. - Apple Stage2Pipeline (default): arm on a gap (new noteFrameIndexGap), withhold at the ring-submit seam (CAMetalLayer holds its last drawable), poll framesDropped, fold VT decode errors through the no-output streak. - Apple StreamPump (stage-1): fold at enqueue, withhold via kCMSampleAttachmentKey_DoNotDisplay so the layer keeps decoding (reference chain intact) but holds the last displayed frame. - Apple VideoDecoder: thread the AU's wire flags to the async decode callback via a retained FrameContext refcon (replaces the receivedNs bit-pattern scalar). Lifts only on a proven re-anchor (IDR / RFI anchor / 2nd recovery mark) with a 500 ms backstop so a lost re-anchor can never freeze forever. Apple: swift build clean, 123/123 tests pass (incl. VideoToolboxRoundTripTests). On-glass loss-injection validation still owed. Co-Authored-By: Claude Opus 4.8 (1M context) --- clients/android/native/src/decode.rs | 179 +++++-- .../Connection/PunktfunkConnection.swift | 15 + .../PunktfunkKit/Video/ReanchorGate.swift | 99 ++++ .../PunktfunkKit/Video/Stage2Pipeline.swift | 31 +- .../PunktfunkKit/Video/StreamPump.swift | 34 +- .../PunktfunkKit/Video/VideoDecoder.swift | 49 +- crates/pf-client-core/src/session.rs | 384 +++------------ crates/punktfunk-core/src/abi.rs | 140 ++++++ crates/punktfunk-core/src/lib.rs | 6 +- crates/punktfunk-core/src/reanchor.rs | 456 ++++++++++++++++++ include/punktfunk_core.h | 91 +++- 11 files changed, 1104 insertions(+), 380 deletions(-) create mode 100644 clients/apple/Sources/PunktfunkKit/Video/ReanchorGate.swift create mode 100644 crates/punktfunk-core/src/reanchor.rs diff --git a/clients/android/native/src/decode.rs b/clients/android/native/src/decode.rs index 72cb923f..d63ba14b 100644 --- a/clients/android/native/src/decode.rs +++ b/clients/android/native/src/decode.rs @@ -15,6 +15,7 @@ use ndk::media::media_format::MediaFormat; use ndk::native_window::NativeWindow; use punktfunk_core::client::NativeClient; use punktfunk_core::error::PunktfunkError; +use punktfunk_core::reanchor::{GateVerdict, ReanchorGate}; use punktfunk_core::session::Frame; use std::collections::VecDeque; use std::ffi::c_void; @@ -208,9 +209,15 @@ fn run_sync( // pressure the AU stays parked here instead of being dropped (a drop forces a keyframe // round-trip) and we only pop the next one once it's queued. let mut pending: Option = None; - // Loss recovery: watch the host→client unrecoverable-drop count and ask for an IDR when it - // climbs. - let mut last_dropped = client.frames_dropped(); + // Freeze-until-reanchor: the shared post-loss gate ([`punktfunk_core::reanchor::ReanchorGate`]). + // Armed on a frame-index gap or a dropped-count climb, it withholds the decoder's concealed output + // (released WITHOUT rendering — the SurfaceView keeps the last rendered frame on glass) until a + // proven clean re-anchor lifts it: an IDR (wire FLAG_SOF), an RFI anchor, or the 2nd recovery mark. + // `last_kf_req` throttles the keyframe intents it emits; `recovery_flags` carries each AU's + // user_flags from feed to present (keyed by the codec-echoed pts) so `on_decoded` reads the + // re-anchor signalling the platform decoder doesn't expose. + let mut gate = ReanchorGate::new(client.frames_dropped()); + let mut recovery_flags: VecDeque<(u64, u32)> = VecDeque::new(); let mut last_kf_req: Option = None; // Skew-corrected latency stats (spec: design/stats-unification.md) use the negotiated // host-minus-client clock offset (0 if the host didn't answer the skew handshake — then the @@ -245,9 +252,18 @@ fn run_sync( Ok(frame) => { // Loss recovery (RFI): feed the frame index so a forward gap fires a throttled // reference-frame-invalidation request — an RFI-capable host (AMD LTR / NVENC) - // recovers with a cheap clean P-frame instead of a full IDR. The frames_dropped - // keyframe path below stays the backstop when the recovery frame itself is lost. - let _ = client.note_frame_index(frame.frame_index); + // recovers with a cheap clean P-frame instead of a full IDR. The same forward gap + // arms the freeze gate so the decoder's concealment is held off the screen until the + // recovery re-anchors. The frames_dropped keyframe path below stays the backstop. + if client.note_frame_index(frame.frame_index) { + gate.arm(Instant::now()); + } + // Park this AU's re-anchor flags for the present side (keyed by the pts the codec + // echoes on the output buffer) — unconditional, unlike the HUD's `in_flight` map. + recovery_flags.push_back((frame.pts_ns / 1000, frame.flags)); + if recovery_flags.len() > IN_FLIGHT_CAP { + recovery_flags.pop_front(); + } if fed == 0 { let p = &frame.data; log::info!( @@ -336,6 +352,8 @@ fn run_sync( &mut in_flight, clock_offset.load(Ordering::Relaxed), &tracker, + &mut gate, + &mut recovery_flags, ); rendered += r; discarded += d; @@ -375,21 +393,19 @@ fn run_sync( work_accum_ns = 0; } - // Loss recovery: under infinite GOP the only recovery keyframe is one we request. The - // reassembler drops unrecoverable AUs (frames_dropped); the decoder then conceals the - // reference-missing delta frames that follow and renders them without error, so keying off - // a decode error rarely fires. Request an IDR when the drop count climbs, throttled — the - // decode stays wedged for several frames until the IDR lands, so requesting every frame - // would flood the control stream. - let dropped = client.frames_dropped(); - if dropped > last_dropped { - last_dropped = dropped; - let now = Instant::now(); - if last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) { - last_kf_req = Some(now); - let _ = client.request_keyframe(); - log::debug!("decode: requested keyframe (loss recovery, dropped={dropped})"); - } + // Loss recovery + overdue backstop, folded through the gate. Under infinite GOP the only + // recovery keyframe is one we request; the reassembler drops unrecoverable AUs (frames_dropped) + // and the decoder then conceals the reference-missing deltas and renders them without error, so + // a decode-error trigger rarely fires — the gate arms the freeze on the drop-count climb + // instead. An overdue freeze (held REANCHOR_FREEZE_MAX with no clean re-anchor) re-asks while it + // keeps holding: never resume to gray — a dead stream is the QUIC idle-timeout watchdog's job. + let now = Instant::now(); + if gate.poll(client.frames_dropped(), now) + && last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) + { + last_kf_req = Some(now); + let _ = client.request_keyframe(); + log::debug!("decode: requested keyframe (loss recovery / overdue re-anchor)"); } } @@ -707,8 +723,10 @@ struct OutputReady { /// internal looper thread) push the codec ones; the feeder thread pushes `Au`. Each carries only /// owned/`Copy` data so the callback closures satisfy the `Send` bound and never touch the codec. enum DecodeEvent { - /// A received access unit from the feeder, ready to queue into the decoder. - Au(Frame), + /// A received access unit from the feeder, ready to queue into the decoder. The `bool` is the + /// feeder's [`NativeClient::note_frame_index`] verdict — `true` when this AU revealed a forward + /// frame-index gap, so the loop arms the freeze gate (the feeder already fired the RFI request). + Au(Frame, bool), /// An input buffer slot freed (index) — we can queue an AU into it. InputAvailable(usize), /// A decoded frame is ready (buffer index + echoed pts + the callback-time `decoded` stamp). @@ -894,7 +912,12 @@ fn run_async( let mut discarded: u64 = 0; // AUs larger than the codec input buffer, dropped whole (see `feed`/`feed_ready`). let mut oversized_dropped: u64 = 0; - let mut last_dropped = client.frames_dropped(); + // Freeze-until-reanchor gate (see the sync loop for the rationale). Armed on a frame-index gap + // (the feeder's Au verdict), a parked-AU overflow drop, a dropped-count climb, or a recoverable + // codec error; `recovery_flags` carries each AU's user_flags from `dispatch_event` (feed) to + // `present_ready` (present), keyed by the codec-echoed pts. + let mut gate = ReanchorGate::new(client.frames_dropped()); + let mut recovery_flags: VecDeque<(u64, u32)> = VecDeque::new(); let mut last_kf_req: Option = None; // Productive (dispatch+feed+present) time between displayed frames; reported to ADPF once one is // presented. The blocking event wait is excluded (idle, not work) — same accounting as the sync loop. @@ -920,6 +943,8 @@ fn run_async( &mut ready, &mut fmt_dirty, &mut fatal, + &mut gate, + &mut recovery_flags, )); } // Coalesce every other event already queued into this one work pass — correct newest-only @@ -932,6 +957,8 @@ fn run_async( &mut ready, &mut fmt_dirty, &mut fatal, + &mut gate, + &mut recovery_flags, )); } stats.note_skipped(aus_dropped); // parked-AU overflow drops are client-side skips too @@ -956,6 +983,8 @@ fn run_async( &tracker, &mut rendered, &mut discarded, + &mut gate, + &mut recovery_flags, ); work_accum_ns += work_t0.elapsed().as_nanos() as i64; @@ -987,17 +1016,19 @@ fn run_async( log::info!("decode: fed={fed} rendered={rendered} discarded={discarded}"); } } - // Loss recovery: request an IDR when the reassembler's unrecoverable-drop count climbs (or we - // dropped a parked AU on overflow), throttled so a multi-frame recovery gap doesn't flood the - // control stream. - let dropped = client.frames_dropped(); - if dropped > last_dropped || aus_dropped > 0 { - last_dropped = dropped; - let now = Instant::now(); - if last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) { - last_kf_req = Some(now); - let _ = client.request_keyframe(); - } + // Loss recovery + overdue backstop, folded through the gate. A parked-AU overflow drop is itself + // a loss, so it arms the freeze directly; the gate's `poll` then arms on a dropped-count climb + // and re-asks on an overdue freeze. All keyframe intents route through the shared 100 ms + // throttle so a multi-frame recovery gap can't flood the control stream. + let now = Instant::now(); + if aus_dropped > 0 { + gate.arm(now); + } + if (gate.poll(client.frames_dropped(), now) || aus_dropped > 0) + && last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) + { + last_kf_req = Some(now); + let _ = client.request_keyframe(); } } @@ -1033,8 +1064,9 @@ fn feeder_loop( Ok(frame) => { // Loss recovery (RFI): a forward frame-index gap fires a throttled reference-frame- // invalidation request so an RFI-capable host recovers with a cheap clean P-frame - // instead of a full IDR (the frames_dropped keyframe path is the backstop). - let _ = client.note_frame_index(frame.frame_index); + // instead of a full IDR (the frames_dropped keyframe path is the backstop). The gap + // verdict rides the Au event so the decode loop arms its freeze gate on the same signal. + let gap = client.note_frame_index(frame.frame_index); if stats.enabled() { let received_ns = now_realtime_ns(); let clock_offset = clock_offset.load(Ordering::Relaxed) as i128; @@ -1067,7 +1099,7 @@ fn feeder_loop( } } } - if ev_tx.send(DecodeEvent::Au(frame)).is_err() { + if ev_tx.send(DecodeEvent::Au(frame, gap)).is_err() { break; // the decode loop is gone } } @@ -1079,6 +1111,7 @@ fn feeder_loop( /// Route one [`DecodeEvent`] into the loop's working sets. Returns `true` only when a parked AU was /// dropped on overflow (the caller then requests a keyframe). +#[allow(clippy::too_many_arguments)] // two call sites; the freeze gate + flag map are threaded in fn dispatch_event( ev: DecodeEvent, pending_aus: &mut VecDeque, @@ -1086,9 +1119,20 @@ fn dispatch_event( ready: &mut Vec, fmt_dirty: &mut bool, fatal: &mut bool, + gate: &mut ReanchorGate, + recovery_flags: &mut VecDeque<(u64, u32)>, ) -> bool { match ev { - DecodeEvent::Au(f) => { + DecodeEvent::Au(f, gap) => { + // A forward frame-index gap arms the freeze; park this AU's flags for the present side to + // fold `on_decoded` (keyed by the pts the codec will echo). + if gap { + gate.arm(Instant::now()); + } + recovery_flags.push_back((f.pts_ns / 1000, f.flags)); + if recovery_flags.len() > IN_FLIGHT_CAP { + recovery_flags.pop_front(); + } pending_aus.push_back(f); if pending_aus.len() > FRAME_PARK_CAP { pending_aus.pop_front(); // sustained overflow — drop oldest, signal a keyframe request @@ -1109,6 +1153,10 @@ fn dispatch_event( DecodeEvent::Error { fatal: f } => { if f { *fatal = true; + } else { + // A recoverable/transient codec error is a decode hiccup on a broken reference chain — + // arm the freeze so the concealed output it recovers into is held off the screen. + gate.arm(Instant::now()); } } } @@ -1180,6 +1228,8 @@ fn present_ready( tracker: &DisplayTracker, rendered: &mut u64, discarded: &mut u64, + gate: &mut ReanchorGate, + recovery_flags: &mut VecDeque<(u64, u32)>, ) { if ready.is_empty() { return; @@ -1192,10 +1242,16 @@ fn present_ready( note_decoded_pts(stats, &mut g, clock_offset, o.pts_us, o.decoded_ns); } } + // Fold EVERY output through the gate in pts (== decode) order — even the ones newest-wins discards — + // so the two-mark re-anchor count stays correct; the newest's verdict decides whether it reaches + // glass (`false` = withheld concealment; the SurfaceView keeps the last rendered frame frozen on). + let now = Instant::now(); let last = ready.len() - 1; let mut skipped: u64 = 0; for (i, o) in ready.drain(..).enumerate() { - let render = i == last; + let flags = take_flags(recovery_flags, o.pts_us); + let present = gate.on_decoded(flags, false, now) == GateVerdict::Present; + let render = i == last && present; match codec.release_output_buffer_by_index(o.index, render) { Ok(()) if render => { *rendered += 1; @@ -1215,7 +1271,7 @@ fn present_ready( } } } - stats.note_skipped(skipped); // HUD `skipped` counter (newest-wins drops); no-op while hidden + stats.note_skipped(skipped); // HUD `skipped` counter (newest-wins + held-off drops); no-op hidden } /// React to an output-format change by signalling the stream's HDR dataspace on the Surface (SDR @@ -1411,19 +1467,28 @@ fn drain( in_flight: &mut VecDeque<(u64, i128)>, clock_offset: i64, tracker: &DisplayTracker, + gate: &mut ReanchorGate, + recovery_flags: &mut VecDeque<(u64, u32)>, ) -> (u64, u64) { // Newest ready buffer so far (presented after the loop) with its HUD metadata — - // `Some((pts_us, decoded_ns))` only while the HUD is visible (the stamp read is gated). + // `Some((pts_us, decoded_ns))` only while the HUD is visible. `held_present` is the freeze gate's + // verdict for that newest buffer (`false` = a post-loss concealment to withhold). let mut held: Option<(OutputBuffer<'_>, Option<(u64, i128)>)> = None; + let mut held_present = true; let mut discarded: u64 = 0; let mut wait = first_wait; loop { match codec.dequeue_output_buffer(wait) { Ok(DequeuedOutputBufferInfoResult::Buffer(buf)) => { wait = Duration::ZERO; // only the first dequeue may block + // Fold every dequeued frame through the gate in pts (== decode) order — even the ones + // the newest-wins policy discards — so the two-mark re-anchor count stays correct; the + // verdict of the newest (last folded) buffer decides whether it reaches glass. + let pts_us = buf.info().presentation_time_us().max(0) as u64; + let flags = take_flags(recovery_flags, pts_us); + held_present = gate.on_decoded(flags, false, Instant::now()) == GateVerdict::Present; let meta = if stats.enabled() { // The dequeue IS the sync loop's decoded-availability instant. - let pts_us = buf.info().presentation_time_us().max(0) as u64; let decoded_ns = now_realtime_ns(); note_decoded_pts(stats, in_flight, clock_offset, pts_us, decoded_ns); Some((pts_us, decoded_ns)) @@ -1469,16 +1534,19 @@ fn drain( } } } - // Present the newest ready frame, if any, and park its metadata for the render callback. + // Present the newest ready frame — UNLESS the gate is withholding it as a post-loss concealment, + // in which case release it without rendering (the SurfaceView keeps the last rendered frame frozen + // on glass) and count it as a discard rather than a display. let mut rendered = 0; if let Some((buf, meta)) = held { - match codec.release_output_buffer(buf, true) { - Ok(()) => { + match codec.release_output_buffer(buf, held_present) { + Ok(()) if held_present => { rendered = 1; if let Some((pts_us, decoded_ns)) = meta { tracker.note_rendered(pts_us, decoded_ns); } } + Ok(()) => discarded += 1, // held off the screen — awaiting a clean re-anchor Err(e) => log::warn!("decode: release_output_buffer: {e}"), } } @@ -1520,6 +1588,25 @@ fn note_decoded_pts( stats.note_decoded(e2e_us, decode_us); } +/// The AU `user_flags` for a decoded output, keyed by the echoed `presentationTimeUs`. Recovery +/// signalling (FLAG_SOF IDR marker / RECOVERY_ANCHOR / RECOVERY_POINT) rides the AU's flags, which are +/// only in scope at feed time — so the feed side parks `(pts_us, flags)` here and the present side +/// looks them up to fold [`ReanchorGate::on_decoded`]. Decode order == input order (low-latency, no +/// B-frames), so this evicts entries older than `pts_us` as it goes; a miss (probe filler, or an entry +/// aged past the cap) reads `0` — no recovery flags, decoded normally. +fn take_flags(map: &mut VecDeque<(u64, u32)>, pts_us: u64) -> u32 { + while let Some(&(p, f)) = map.front() { + if p > pts_us { + break; // future frame — leave it for its own output buffer + } + map.pop_front(); + if p == pts_us { + return f; + } + } + 0 +} + /// Map the decoder's reported output colour to a BT.2020 HDR dataspace, or `None` for SDR. The /// integer values are the Android MediaFormat colour constants the NDK shares: COLOR_TRANSFER /// ST2084 = 6 (PQ/HDR10), HLG = 7; COLOR_RANGE FULL = 1, LIMITED = 2 (the host encodes limited). diff --git a/clients/apple/Sources/PunktfunkKit/Connection/PunktfunkConnection.swift b/clients/apple/Sources/PunktfunkKit/Connection/PunktfunkConnection.swift index 7a8e1195..6b8d3f69 100644 --- a/clients/apple/Sources/PunktfunkKit/Connection/PunktfunkConnection.swift +++ b/clients/apple/Sources/PunktfunkKit/Connection/PunktfunkConnection.swift @@ -450,6 +450,21 @@ public final class PunktfunkConnection { _ = punktfunk_connection_note_frame_index(h, frameIndex, nil) } + /// Like `noteFrameIndex`, but also reports whether the core saw a FORWARD frame-index gap — the + /// signal that intervening frames were lost and the following AUs reference a picture that never + /// arrived. The post-loss re-anchor gate arms its display freeze on a gap (the earliest, most + /// precise loss trigger — ahead of the `framesDropped` climb). Same core side effect as + /// `noteFrameIndex` (the throttled RFI request); call it for every received AU. Returns false + /// after close. + public func noteFrameIndexGap(_ frameIndex: UInt32) -> Bool { + abiLock.lock() + defer { abiLock.unlock() } + guard let h = handle, !closeRequested else { return false } + var gap = false + _ = punktfunk_connection_note_frame_index(h, frameIndex, &gap) + return gap + } + /// Cumulative access units the host→client reassembler dropped as unrecoverable (FEC couldn't /// rebuild them). The video pump polls this and calls `requestKeyframe()` when it climbs — the /// correct loss trigger under the host's infinite GOP, where unrecoverable loss yields diff --git a/clients/apple/Sources/PunktfunkKit/Video/ReanchorGate.swift b/clients/apple/Sources/PunktfunkKit/Video/ReanchorGate.swift new file mode 100644 index 00000000..95fdf680 --- /dev/null +++ b/clients/apple/Sources/PunktfunkKit/Video/ReanchorGate.swift @@ -0,0 +1,99 @@ +// Swift wrapper around the punktfunk-core C ABI's post-loss re-anchor gate +// (`punktfunk_reanchor_gate_*`, ABI v6). The shared Rust gate (crates/punktfunk-core/src/reanchor.rs) +// is what the Linux/Windows desktop pump and the Android client use directly; the Swift clients reach +// it across the C ABI so the freeze-until-reanchor policy is defined ONCE for every platform. +// +// Why a freeze at all: after unrecoverable loss the host keeps sending delta frames that reference a +// picture the client never got. Hardware decoders (VideoToolbox included) don't reliably error on +// that — they CONCEAL, returning a gray/garbage frame with a success status. Presenting those is the +// visible "gray flash with motion" of the loss reports. The gate withholds concealed frames and holds +// the last good picture on glass until a PROVEN clean re-anchor lands — an IDR (wire `FLAG_SOF`), an +// RFI recovery anchor (`USER_FLAG_RECOVERY_ANCHOR`), or the 2nd of two intra-refresh recovery marks +// (`USER_FLAG_RECOVERY_POINT`) — with a bounded backstop so a lost re-anchor can never freeze forever. +// See punktfunk-planning design/client-reanchor-freeze-parity.md. +// +// Threading: one gate per session. Its calls arrive from two threads — the pump thread (`arm` on a +// frame-index gap / a submit failure, `poll` per iteration) and a VideoToolbox decode thread +// (`onDecoded` per decoded frame, `onNoOutput` on a decode error). The raw Rust gate is a plain +// struct behind an opaque pointer with no internal synchronization, so every call is serialized under +// `lock` here — the calls are cheap field updates, so contention is negligible. `@unchecked Sendable`: +// the lock enforces the contract. + +import Foundation +import PunktfunkCore + +final class ReanchorGate: @unchecked Sendable { + private let lock = NSLock() + /// The opaque `ReanchorGate *`. `var` so `reseed` can swap it at session start. Never NULL + /// (`punktfunk_reanchor_gate_new` never returns NULL). + private var ptr: OpaquePointer + + /// Seed the baseline with the connection's current `framesDropped` so the first `poll` doesn't + /// read the session's starting drop count as a fresh loss. + init(framesDropped: UInt64) { + ptr = punktfunk_reanchor_gate_new(framesDropped) + } + + deinit { punktfunk_reanchor_gate_free(ptr) } + + /// Re-anchor the drop-count baseline to `framesDropped` for a (re)started session. The gate is + /// created in the pipeline's init (before a connection exists, seeded 0); `start` calls this once + /// the live connection's count is known so a mid-life connection's non-zero baseline isn't + /// mistaken for loss on the first poll. + func reseed(framesDropped: UInt64) { + lock.lock() + defer { lock.unlock() } + punktfunk_reanchor_gate_free(ptr) + ptr = punktfunk_reanchor_gate_new(framesDropped) + } + + /// Arm the freeze: a loss was detected (a frame-index gap, or a decoder wedge). Zeroes the + /// recovery-mark count and (re)sets the backstop deadline. + func arm() { + lock.lock() + punktfunk_reanchor_gate_arm(ptr) + lock.unlock() + } + + /// Fold one decoded frame. `flags` is the AU's wire `user_flags`. Returns true to PRESENT the + /// frame, false to WITHHOLD it as a post-loss concealment (hold the last good picture). Pass + /// `decoderKeyframe: false` — VideoToolbox doesn't flag IDRs, so the wire `FLAG_SOF` covers it. + func onDecoded(flags: UInt32, decoderKeyframe: Bool = false) -> Bool { + lock.lock() + defer { lock.unlock() } + var present = false + _ = punktfunk_reanchor_gate_on_decoded(ptr, flags, decoderKeyframe, &present) + return present + } + + /// A received AU produced no decoded frame (a VideoToolbox decode error). Returns true when the + /// no-output streak has tripped (the gate armed the freeze) and the caller should — throttled — + /// request a keyframe. + func onNoOutput() -> Bool { + lock.lock() + defer { lock.unlock() } + var requestKf = false + _ = punktfunk_reanchor_gate_on_no_output(ptr, &requestKf) + return requestKf + } + + /// Periodic fold of the session's `framesDropped` plus the overdue backstop. Returns true when the + /// caller should — throttled — request a keyframe (a drop-count climb armed a fresh freeze, or the + /// freeze is overdue and re-asks while it keeps holding). + func poll(framesDropped: UInt64) -> Bool { + lock.lock() + defer { lock.unlock() } + var requestKf = false + _ = punktfunk_reanchor_gate_poll(ptr, framesDropped, &requestKf) + return requestKf + } + + /// Whether the gate is currently withholding concealed frames (frozen on the last good picture). + var isHolding: Bool { + lock.lock() + defer { lock.unlock() } + var holding = false + _ = punktfunk_reanchor_gate_is_holding(ptr, &holding) + return holding + } +} diff --git a/clients/apple/Sources/PunktfunkKit/Video/Stage2Pipeline.swift b/clients/apple/Sources/PunktfunkKit/Video/Stage2Pipeline.swift index bdaa6203..3d41de91 100644 --- a/clients/apple/Sources/PunktfunkKit/Video/Stage2Pipeline.swift +++ b/clients/apple/Sources/PunktfunkKit/Video/Stage2Pipeline.swift @@ -259,6 +259,10 @@ public final class Stage2Pipeline { private let endToEndMeter: LatencyMeter? private let displayMeter: LatencyMeter? private let recovery = KeyframeRecovery() + /// Post-loss freeze-until-reanchor gate (shared core policy via the C ABI). Created here seeded 0; + /// `start` reseeds it to the live connection's drop count. Captured by the decoder callbacks + /// (which withhold concealed frames) and driven by the pump (arm on a gap, poll per iteration). + private let gate = ReanchorGate(framesDropped: 0) private var token = StopFlag() private var offsetNs: Int64 = 0 /// Signalled when the pump thread exits, so `stop()` can join it (bounded) before `decoder.reset()` @@ -306,21 +310,29 @@ public final class Stage2Pipeline { let ring = ring let recovery = recovery let renderSignal = renderSignal + let gate = gate self.decoder = VideoDecoder( onDecoded: { frame in // Decode stage = received→decoded, both client CLOCK_REALTIME (offset 0 — no // skew applies). Stamped at decode completion, so it covers every decoded frame, - // including ones the newest-wins ring drops before present. + // including ones the re-anchor gate withholds or the newest-wins ring drops. decodeMeter?.record( ptsNs: UInt64(frame.receivedNs), atNs: frame.decodedNs, offsetNs: 0) + // Freeze-until-reanchor: WITHHOLD a decoder-concealed post-loss frame (the gray/ + // garbage VideoToolbox returns Ok for a reference-missing delta) — don't submit it, + // so the CAMetalLayer keeps its last good drawable on glass. The gate lifts (returns + // present) on a proven clean re-anchor (IDR / RFI anchor / 2nd recovery mark) or the + // bounded backstop. decoderKeyframe=false: VT doesn't flag IDRs, the wire FLAG_SOF does. + guard gate.onDecoded(flags: frame.flags) else { return } ring.submit(frame) // FRAME ARRIVAL is the render trigger (never the display link — see the header). renderSignal.signal() }, - // Async decode failure (a bad P-frame referencing a lost/corrupt IDR): the pump resets to - // re-gate on the next IDR, and we ask the host to send one now (infinite GOP — it wouldn't + // Async decode failure (a bad P-frame referencing a lost/corrupt IDR): fold it into the + // gate's no-output streak (which arms the freeze after a short run, matching the desktop), + // and when that trips ask the host for a fresh IDR now (infinite GOP — it wouldn't // otherwise come soon). Throttled in KeyframeRecovery. - onDecodeError: { _ in recovery.request() }) + onDecodeError: { _ in if gate.onNoOutput() { recovery.request() } }) } /// Start pulling AUs into the decoder. MAIN THREAD. `onFrame` fires per AU at receipt (the @@ -334,6 +346,7 @@ public final class Stage2Pipeline { ) { offsetNs = connection.clockOffsetNs recovery.bind(connection) // arm host-keyframe recovery for this session + gate.reseed(framesDropped: connection.framesDropped()) // baseline the freeze to this session token = StopFlag() // fresh token per start — a stop is permanent (like StreamPump) // Configure the decoder's chroma + the layer's initial colorimetry before the first frame. The @@ -348,6 +361,7 @@ public final class Stage2Pipeline { let recovery = recovery let presenter = presenter let pumpStopped = pumpStopped + let reanchorGate = gate let thread = Thread { defer { pumpStopped.signal() } // let stop() join the pump (bounded) before decoder.reset() var format: CMVideoFormatDescription? @@ -379,6 +393,9 @@ public final class Stage2Pipeline { awaitingIDR = true } if awaitingIDR { recovery.request() } + // Freeze backstop: a drop-count climb arms the gate (in case the frame-index gap + // below was itself lost), and an overdue freeze re-asks for the re-anchor. + if reanchorGate.poll(framesDropped: dropped) { recovery.request() } // Drain HDR mastering metadata (0xCE) and hand it to the PRESENTER (→ CAEDRMetadata). // Polled UNCONDITIONALLY (not gated on connection.isHDR, the fixed Welcome flag): the // host sends 0xCE only for HDR, INCLUDING a mid-session SDR→HDR transition (a game @@ -391,8 +408,10 @@ public final class Stage2Pipeline { // Loss recovery (RFI): a forward frame-index gap fires a throttled reference- // frame-invalidation request so an RFI-capable host (AMD LTR / NVENC) recovers // with a cheap clean P-frame instead of a full IDR. The framesDropped-driven - // recovery below stays the backstop for when the recovery frame itself is lost. - connection.noteFrameIndex(au.frameIndex) + // recovery above stays the backstop for when the recovery frame itself is lost. + // The same gap is the earliest, most precise signal to ARM the display freeze — + // the following concealed frames are withheld until a clean re-anchor. + if connection.noteFrameIndexGap(au.frameIndex) { reanchorGate.arm() } onFrame?(au) if let f = connection.videoCodec.formatDescription(fromKeyframe: au.data) { format = f // refreshed on every IDR (mode changes included) diff --git a/clients/apple/Sources/PunktfunkKit/Video/StreamPump.swift b/clients/apple/Sources/PunktfunkKit/Video/StreamPump.swift index 70b9f03b..bb4df5aa 100644 --- a/clients/apple/Sources/PunktfunkKit/Video/StreamPump.swift +++ b/clients/apple/Sources/PunktfunkKit/Video/StreamPump.swift @@ -28,6 +28,11 @@ final class StreamPump { // Coalesced host keyframe requests (100 ms throttle — see KeyframeRecovery). let recovery = KeyframeRecovery() recovery.bind(connection) + // Post-loss freeze-until-reanchor (shared core policy via the C ABI). Stage-1 has no per-frame + // decode callback, so the gate is folded at ENQUEUE (from the AU's wire flags): a withheld + // frame is still enqueued but flagged DoNotDisplay so the layer's decoder keeps the reference + // chain fed while the last GOOD picture stays on glass — until a clean re-anchor lifts it. + let gate = ReanchorGate(framesDropped: connection.framesDropped()) // The layer is non-Sendable but its enqueue/flush are documented thread-safe, and after // this point only the pump thread drives it — assert that so the @Sendable Thread closure // may capture it. @@ -77,13 +82,17 @@ final class StreamPump { awaitingIDR = true } if awaitingIDR { recovery.request() } + // Freeze backstop: a drop-count climb arms the gate (should the frame-index gap + // below be lost too), and an overdue freeze re-asks for the re-anchor. + if gate.poll(framesDropped: dropped) { recovery.request() } guard let au = try connection.nextAU(timeoutMs: 100) else { return true } // Loss recovery (RFI): a forward frame-index gap fires a throttled reference- // frame-invalidation request so an RFI-capable host (AMD LTR / NVENC) recovers // with a cheap clean P-frame instead of a full IDR. The framesDropped-driven // recovery above stays the backstop for when the recovery frame itself is lost. - connection.noteFrameIndex(au.frameIndex) + // The same gap is the earliest, most precise signal to ARM the display freeze. + if connection.noteFrameIndexGap(au.frameIndex) { gate.arm() } onFrame?(au) let idrFormat = connection.videoCodec.formatDescription(fromKeyframe: au.data) if let f = idrFormat { @@ -107,6 +116,7 @@ final class StreamPump { // delta into a failed layer can't recover it. if !wasFailed { pumpLog.warning("video: display layer .failed — flushing + re-anchoring") } layer.flush() + gate.arm() // a wedged decoder is a loss — freeze until the re-anchor if idrFormat == nil { format = nil awaitingIDR = true @@ -117,6 +127,13 @@ final class StreamPump { let sample = connection.videoCodec.sampleBuffer(au: au, format: f), !token.isStopped // don't enqueue a stale frame after a restart else { return true } + // Freeze-until-reanchor: while holding, WITHHOLD this concealed post-loss frame by + // flagging it DoNotDisplay — the layer still decodes it (keeping the reference + // chain fed) but shows the last GOOD picture until a clean re-anchor lifts the + // gate. Folded from the AU's wire flags (stage-1 has no decode callback). + if !gate.onDecoded(flags: au.flags) { + StreamPump.setDoNotDisplay(sample) + } layer.enqueue(sample) return true } catch { @@ -133,6 +150,21 @@ final class StreamPump { thread.start() } + /// Flag a sample decode-but-don't-display (`kCMSampleAttachmentKey_DoNotDisplay`). Used to + /// withhold decoder-concealed post-loss frames while the re-anchor gate holds: the layer keeps + /// its reference chain fed without flipping the frozen picture. No-op if the attachments array + /// can't be materialized (then the frame just displays — the freeze degrades to the old behavior). + private static func setDoNotDisplay(_ sample: CMSampleBuffer) { + guard let attachments = CMSampleBufferGetSampleAttachmentsArray( + sample, createIfNecessary: true), CFArrayGetCount(attachments) > 0 + else { return } + let dict = unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self) + CFDictionarySetValue( + dict, + Unmanaged.passUnretained(kCMSampleAttachmentKey_DoNotDisplay).toOpaque(), + Unmanaged.passUnretained(kCFBooleanTrue).toOpaque()) + } + /// Stop pumping (≤ one poll timeout). Does not close the connection. func stop() { token.stop() diff --git a/clients/apple/Sources/PunktfunkKit/Video/VideoDecoder.swift b/clients/apple/Sources/PunktfunkKit/Video/VideoDecoder.swift index 17b65c7b..6115f9bf 100644 --- a/clients/apple/Sources/PunktfunkKit/Video/VideoDecoder.swift +++ b/clients/apple/Sources/PunktfunkKit/Video/VideoDecoder.swift @@ -27,19 +27,40 @@ public struct ReadyFrame: @unchecked Sendable { /// True when the stream is HDR (BT.2020 PQ): the buffer is 10-bit P010 and the presenter must /// configure EDR + BT.2020 PQ output. Derived from the decoded buffer's pixel format. public let isHDR: Bool + /// The AU's wire `user_flags` (`AccessUnit.flags`), threaded through the decode via the frame + /// context so the re-anchor gate can classify this decoded frame (IDR / RFI anchor / recovery + /// mark) at present time — the async decode callback has no other access to it. 0 when unknown. + public let flags: UInt32 +} + +/// Per-frame context threaded through the VideoToolbox frame refcon: the AU's receipt instant (for +/// the decode-stage meter) and its wire `user_flags` (for the re-anchor gate). Retained across the +/// async decode and reclaimed exactly once — by the output callback for every frame VideoToolbox +/// accepts, or by `decode`'s error branch for a frame `DecodeFrame` rejected outright (the callback +/// then never fires). A tiny per-frame allocation, the price of smuggling two values (a 64-bit +/// instant plus the flags) through the single `void*` a bit-pattern scalar can't hold. +private final class FrameContext { + let receivedNs: Int64 + let flags: UInt32 + init(receivedNs: Int64, flags: UInt32) { + self.receivedNs = receivedNs + self.flags = flags + } } /// The C output callback can't capture context, so VideoToolbox hands it the refcon we set at -/// session creation — a pointer back to the owning `VideoDecoder`. The per-frame refcon carries -/// the AU's `receivedNs` as a pointer bit pattern (a scalar smuggled through the C void*, never -/// dereferenced) so the decode stage can be computed against decode-completion. +/// session creation — a pointer back to the owning `VideoDecoder`. The per-frame refcon is the +/// retained `FrameContext` set at submit; reclaim it here (balancing `passRetained`) and unpack the +/// AU's receipt instant (for the decode stage) and wire flags (for the re-anchor gate). private let decoderOutputCallback: VTDecompressionOutputCallback = { refcon, frameRefcon, status, _, imageBuffer, pts, _ in guard let refcon else { return } - let receivedNs = frameRefcon.map { Int64(Int(bitPattern: $0)) } ?? 0 + let ctx = frameRefcon.map { Unmanaged.fromOpaque($0).takeRetainedValue() } Unmanaged.fromOpaque(refcon) .takeUnretainedValue() - .handleDecoded(status: status, imageBuffer: imageBuffer, pts: pts, receivedNs: receivedNs) + .handleDecoded( + status: status, imageBuffer: imageBuffer, pts: pts, + receivedNs: ctx?.receivedNs ?? 0, flags: ctx?.flags ?? 0) } /// Owns a `VTDecompressionSession` rebuilt whenever the format description changes (every IDR / @@ -117,16 +138,21 @@ public final class VideoDecoder: @unchecked Sendable { let sample = codec.sampleBuffer(au: au, format: newFormat) else { lock.unlock(); return false } var infoOut = VTDecodeInfoFlags() + // The AU's receipt instant + wire flags ride through as a retained context; the output + // callback reclaims it. Retain immediately before submit so no early return can leak it. + let ctx = FrameContext(receivedNs: au.receivedNs, flags: au.flags) + let refcon = Unmanaged.passRetained(ctx).toOpaque() let status = VTDecompressionSessionDecodeFrame( session, sampleBuffer: sample, flags: [._EnableAsynchronousDecompression], - // The AU's receipt instant rides through as a bit pattern (nil for 0 — the output - // callback maps that back to 0); the callback needs it to stamp the decode stage. - frameRefcon: UnsafeMutableRawPointer(bitPattern: Int(au.receivedNs)), + frameRefcon: refcon, infoFlagsOut: &infoOut) lock.unlock() if status != noErr { + // DecodeFrame rejected the frame outright — the output callback will NOT fire, so + // reclaim the context here (balancing passRetained) to avoid leaking it. + Unmanaged.fromOpaque(refcon).release() onDecodeError(status) return false } @@ -231,9 +257,10 @@ public final class VideoDecoder: @unchecked Sendable { } /// VT thread. Stamp decode-completion and enqueue, or report the error. `receivedNs` is the - /// AU's receipt instant threaded through the frame refcon (0 = unknown). + /// AU's receipt instant and `flags` its wire `user_flags`, both threaded through the frame refcon + /// (0 = unknown). fileprivate func handleDecoded( - status: OSStatus, imageBuffer: CVImageBuffer?, pts: CMTime, receivedNs: Int64 + status: OSStatus, imageBuffer: CVImageBuffer?, pts: CMTime, receivedNs: Int64, flags: UInt32 ) { guard status == noErr, let imageBuffer else { onDecodeError(status) @@ -259,6 +286,6 @@ public final class VideoDecoder: @unchecked Sendable { onDecoded( ReadyFrame( ptsNs: ptsNs, receivedNs: receivedNs, decodedNs: decodedNs, - pixelBuffer: imageBuffer, isHDR: isHDR)) + pixelBuffer: imageBuffer, isHDR: isHDR, flags: flags)) } } diff --git a/crates/pf-client-core/src/session.rs b/crates/pf-client-core/src/session.rs index 6863afd9..5720b181 100644 --- a/crates/pf-client-core/src/session.rs +++ b/crates/pf-client-core/src/session.rs @@ -10,6 +10,7 @@ use crate::audio; use crate::video::{DecodedFrame, DecodedImage, Decoder}; use punktfunk_core::client::NativeClient; use punktfunk_core::config::{CompositorPref, GamepadPref, Mode}; +use punktfunk_core::reanchor::{index_gap, GateVerdict, ReanchorGate}; use punktfunk_core::PunktfunkError; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -99,86 +100,6 @@ pub struct Stats { pub decoder: &'static str, } -/// Consecutive no-output AUs that force a keyframe request. ~50 ms at 60 Hz — long -/// enough not to fire on a one-frame decoder hiccup, short enough that a lost initial -/// IDR (or a mid-GOP join) unfreezes almost immediately instead of never. -const NO_OUTPUT_KEYFRAME_STREAK: u32 = 3; - -/// Longest the pump holds the last good frame waiting for a post-loss re-anchor keyframe before it -/// gives up and resumes display. After a reference loss the hardware decoder does not error — it -/// conceals the reference-missing deltas (on RADV, the DPB-and-output-COINCIDE path renders them as -/// a gray plate with the new frame's motion painted over it) and returns Ok, so displaying them is -/// the "gray frames mid-stream" artifact. We instead freeze on the last good picture until a fresh -/// IDR re-anchors decode — the behaviour NVIDIA already shows (its DISTINCT output image + different -/// concealment reads as a brief freeze, not gray). This cap only bounds the freeze when recovery -/// genuinely stalls (host ignores the request, or an RFI recovery that never emits a keyframe), so a -/// glitch can never become a permanent freeze. A recovery IDR round-trips well under this on any -/// live link. -const REANCHOR_FREEZE_MAX: Duration = Duration::from_millis(500); - -/// How many host intra-refresh recovery marks ([`USER_FLAG_RECOVERY_POINT`]) must arrive since the -/// latest frame gap before the pump lifts its freeze on an IDR-free stream. TWO, not one: with a -/// continuous rolling wave the host marks phase-fixed wave boundaries, so the FIRST boundary after a -/// loss is only partially healed — stripes swept BEFORE the loss still reference the lost frame — and -/// lifting there would flash a partially-stale picture. The SECOND boundary guarantees a full wave -/// swept entirely after the loss, so the picture is clean. This stays correct under repeated loss -/// because every new gap resets the count. The cost is up to ~2 wave periods of holding the last good -/// frame — the deliberate "hold longer, never show garbage" trade. -/// -/// [`USER_FLAG_RECOVERY_POINT`]: punktfunk_core::packet::USER_FLAG_RECOVERY_POINT -const REANCHOR_MARKS_TO_LIFT: u32 = 2; - -/// Backstop patience while a host intra-refresh heal is visibly in progress. Each recovery mark -/// pushes the freeze deadline out by this much, so a live mark stream (the host actively healing via -/// its wave) keeps the client patiently holding the last good frame instead of tripping the IDR -/// floor mid-heal. Must exceed the inter-mark interval (one wave period, ~0.5 s) with margin; if the -/// marks STOP (heal stalled, or the host isn't running intra-refresh) the deadline lapses and the -/// normal recovery-IDR floor fires, so a real stall still recovers. -const RECOVERY_MARK_PATIENCE: Duration = Duration::from_millis(1500); - -/// Frames skipped when `got` arrives while `expected` was the next index, or `None` if `got` is -/// contiguous (`== expected`) or a straggler we have already passed. Frame indices are u32 counters -/// that wrap, so the "ahead" test is a wrapping subtraction split at the half-space: a small -/// positive delta is a forward gap (missing frames whose dependents will decode against absent -/// references); a delta in the top half is an index behind us. -fn index_gap(expected: u32, got: u32) -> Option { - let ahead = got.wrapping_sub(expected); - (ahead != 0 && ahead < u32::MAX / 2).then_some(ahead) -} - -/// Fold one decoded frame into the re-anchor state and decide whether it lifts the post-loss freeze. -/// -/// `is_keyframe` — a real IDR (always a clean re-anchor). `has_anchor` — this AU carried -/// [`USER_FLAG_RECOVERY_ANCHOR`](punktfunk_core::packet::USER_FLAG_RECOVERY_ANCHOR), the host's -/// definitive single-frame re-anchor from an LTR-RFI recovery (a clean P-frame coded against a -/// known-good reference), so it lifts on the FIRST occurrence exactly like an IDR — no two-mark wait. -/// `has_mark` — this AU carried [`USER_FLAG_RECOVERY_POINT`](punktfunk_core::packet::USER_FLAG_RECOVERY_POINT), -/// a host-signalled intra-refresh wave boundary (only *half* a re-anchor). `marks` — recovery marks -/// seen since the latest gap. -/// -/// Returns `(lift, new_marks)`: `lift` clears the freeze; `new_marks` is the running count (reset to 0 -/// on a lift). The two-mark rule ([`REANCHOR_MARKS_TO_LIFT`]) lives here so it is unit-tested -/// independent of the pump's channel/decoder plumbing — the first wave boundary after a loss is only -/// partially healed, so a single mark must NOT lift. An anchor (or IDR) is a *whole* re-anchor and -/// lifts immediately. -fn reanchor_after_frame( - is_keyframe: bool, - has_anchor: bool, - has_mark: bool, - marks: u32, -) -> (bool, u32) { - let marks = if has_mark { - marks.saturating_add(1) - } else { - marks - }; - if is_keyframe || has_anchor || marks >= REANCHOR_MARKS_TO_LIFT { - (true, 0) - } else { - (false, marks) - } -} - /// Frames the pump keeps waiting for their 0xCF host timing (pts → capture→received µs). /// ~2 s at 120 Hz — a timing arrives within a frame or two of its AU, and against an old /// host (no 0xCF at all) this just caps the dead-weight ring. @@ -382,27 +303,17 @@ fn pump( // What actually decoded the last frame — a VAAPI failure demotes mid-session, so // this is read off each frame's image variant rather than fixed at startup. let mut dec_path: &'static str = ""; - // Loss recovery: watch the host→client unrecoverable-drop count and ask for an IDR when it climbs. - let mut last_dropped = connector.frames_dropped(); // The stats window keeps its own drop cursor — the OSD shows the per-window delta. - let mut window_dropped = last_dropped; + let mut window_dropped = connector.frames_dropped(); let mut last_kf_req: Option = None; - // Consecutive received AUs that produced NO decoded frame (decode error, or the - // decoder swallowed a reference-missing delta and returned nothing). Distinct from - // `frames_dropped`, which counts reassembler drops: when the initial IDR is lost (or - // we join mid-GOP) the reassembler delivers complete-but-undecodable deltas — it - // never drops, so the drop-count trigger below stays silent and the stream freezes - // on the last good frame. A short streak forces a fresh IDR to re-anchor. - let mut no_output_streak = 0u32; - // Freeze-until-reanchor: armed the moment we request a recovery keyframe (loss, decode error, or - // a no-output streak), it withholds the decoder's concealed frames from the presenter — which - // then redraws the last good picture — until a fresh keyframe re-anchors decode. See - // [`REANCHOR_FREEZE_MAX`] for why this exists and its backstop deadline. - let mut awaiting_reanchor = false; - let mut reanchor_deadline: Option = None; - // Host intra-refresh recovery marks seen since the latest gap (see [`REANCHOR_MARKS_TO_LIFT`]). - // Reset to 0 whenever the freeze is (re-)armed, so a fresh loss always waits out two fresh marks. - let mut recovery_marks: u32 = 0; + // Freeze-until-reanchor: the shared post-loss gate ([`punktfunk_core::reanchor::ReanchorGate`]). + // Armed on any loss signal (frame-index gap, dropped-count climb, decoder wedge/demotion), it + // withholds the decoder's concealed frames from the presenter — which then redraws the last good + // picture — until a proven clean re-anchor (IDR / RFI anchor / second recovery mark) lifts it. It + // also owns the no-output streak and the overdue-freeze backstop; the client keeps its own + // `last_kf_req` request throttle and routes the gate's keyframe intents through it. Seeded with the + // current drop count so the first `poll` doesn't read the baseline as a loss. + let mut gate = ReanchorGate::new(connector.frames_dropped()); // The frame_index we expect next (the host numbers frames consecutively). A jump means a frame // went missing — the earliest, most reliable signal that the decoder is about to conceal, ~120 ms // ahead of `frames_dropped` (the reassembler only declares a straggler lost once it ages out of @@ -447,9 +358,7 @@ fn pump( Some(exp) => { if let Some(gap) = index_gap(exp, frame.frame_index) { let now = Instant::now(); - awaiting_reanchor = true; - recovery_marks = 0; - reanchor_deadline = Some(now + REANCHOR_FREEZE_MAX); + gate.arm(now); next_expected_index = Some(frame.frame_index.wrapping_add(1)); // The gap carries the PRECISE lost range — [first missing, newest // received - 1] — so this is the one recovery signal that can drive true @@ -488,38 +397,14 @@ fn pump( } match decoder.decode(&frame.data) { Ok(Some(image)) => { - // A decoded frame — the anchor holds. - no_output_streak = 0; - // Host-signalled intra-refresh recovery mark: on an IDR-free intra-refresh - // stream this wave-boundary flag is the only clean point the client can honor - // (the decoder never flags the re-anchor — the coded frame stays `P`). A live - // mark stream also means the host is actively healing, so push the backstop out - // rather than trip a mid-heal IDR (see `RECOVERY_MARK_PATIENCE`). - let has_mark = - frame.flags & punktfunk_core::packet::USER_FLAG_RECOVERY_POINT != 0; - // The host's definitive single-frame re-anchor: an LTR-RFI recovery frame (a - // clean P-frame off a known-good reference), the AMD twin of an IDR re-anchor - // but without the spike. It lifts on the FIRST occurrence. - let has_anchor = - frame.flags & punktfunk_core::packet::USER_FLAG_RECOVERY_ANCHOR != 0; - if has_mark && awaiting_reanchor { - reanchor_deadline = Some(Instant::now() + RECOVERY_MARK_PATIENCE); - } - // A fresh clean re-anchor lifts the freeze and shows this frame: a real intra - // keyframe (IDR, always clean), an LTR-RFI recovery anchor (also whole), OR the - // second recovery mark since the gap (the first wave boundary is only - // half-healed — see `reanchor_after_frame`). - let (lift, marks) = reanchor_after_frame( - image.is_keyframe(), - has_anchor, - has_mark, - recovery_marks, - ); - recovery_marks = marks; - if lift { - awaiting_reanchor = false; - reanchor_deadline = None; - } + // Fold this decoded frame through the shared freeze gate: it reads the AU's + // re-anchor wire flags (FLAG_SOF IDR marker / RECOVERY_ANCHOR / RECOVERY_POINT), + // takes `image.is_keyframe()` as the ffmpeg keyframe belt, applies the two-mark + // rule + the mark-patience backstop, clears the no-output streak, and returns + // whether to present this frame or withhold it as a post-loss concealment. + let present = gate + .on_decoded(frame.flags, image.is_keyframe(), Instant::now()) + == GateVerdict::Present; total_frames += 1; dec_path = match &image { DecodedImage::Cpu(_) => "software", @@ -574,19 +459,19 @@ fn pump( DecodedImage::VkFrame(v) => Some((v.timeline_sem, v.decode_done_value)), _ => None, }; - if awaiting_reanchor { - // Post-loss concealment: withhold this frame (it references a lost/gray - // reference) so the presenter keeps redrawing the last good picture - // rather than flashing the decoder's gray plate. Dropped here — the - // hw-decode stat below still samples via `hw_fence` (raw handle + value, - // valid past the guard). Cleared by the next keyframe or the backstop. - tracing::trace!("holding last frame — awaiting post-loss re-anchor"); - } else { + if present { let _ = frame_tx.force_send(DecodedFrame { pts_ns: frame.pts_ns, decoded_ns, image, }); + } else { + // Post-loss concealment: withhold this frame (it references a lost/gray + // reference) so the presenter keeps redrawing the last good picture rather + // than flashing the decoder's gray plate. Dropped here — the hw-decode stat + // below still samples via `hw_fence` (raw handle + value, valid past the + // guard). The gate lifts the freeze on the next clean re-anchor / backstop. + tracing::trace!("holding last frame — awaiting post-loss re-anchor"); } // `decode` stage: received→decode COMPLETE, single clock. match hw_fence { @@ -602,36 +487,35 @@ fn pump( } } } - Ok(None) => no_output_streak += 1, + // The decoder produced nothing — under zero-reorder LOW_DELAY (one-in/one-out) that + // means it's wedged on missing references with no reassembler drop to trigger + // recovery. The gate counts the streak and, once it trips, arms the freeze and tells + // us to (throttled) request a fresh IDR to re-anchor. Both the empty-output and the + // survivable-decode-error arms feed it; a decoded frame resets the streak in + // `on_decoded`. + Ok(None) => { + let now = Instant::now(); + if gate.on_no_output(now) + && last_kf_req + .is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) + { + last_kf_req = Some(now); + let _ = connector.request_keyframe(); + tracing::debug!("requested keyframe (decoder produced no output)"); + } + } // Survivable (loss until the next IDR/RFI recovery) — keep feeding. Err(e) => { - no_output_streak += 1; tracing::debug!(error = %e, "decode error (recovering)"); - } - } - // The decoder has produced nothing for a short run — under zero-reorder - // LOW_DELAY (one-in/one-out) that means it's wedged on missing references - // with no reassembler drop to trigger recovery below. Ask for a fresh IDR - // (throttled), then re-arm the streak so we wait out the request→IDR round - // trip before asking again instead of flooding. - if no_output_streak >= NO_OUTPUT_KEYFRAME_STREAK { - let now = Instant::now(); - // Wedged on missing references: hold the last good frame until re-anchor - // (armed even when the IDR request itself is throttled — the stream is broken - // regardless of whether we ask again this iteration). - awaiting_reanchor = true; - recovery_marks = 0; - reanchor_deadline = Some(now + REANCHOR_FREEZE_MAX); - if last_kf_req - .is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) - { - last_kf_req = Some(now); - let _ = connector.request_keyframe(); - tracing::debug!( - streak = no_output_streak, - "requested keyframe (decoder produced no output)" - ); - no_output_streak = 0; + let now = Instant::now(); + if gate.on_no_output(now) + && last_kf_req + .is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) + { + last_kf_req = Some(now); + let _ = connector.request_keyframe(); + tracing::debug!("requested keyframe (decode error recovery)"); + } } } // The presenter's verdict: hardware frames can't be displayed (GL converter @@ -649,9 +533,7 @@ fn pump( // through the same throttle as loss recovery below. if decoder.take_keyframe_request() { let now = Instant::now(); - awaiting_reanchor = true; - recovery_marks = 0; - reanchor_deadline = Some(now + REANCHOR_FREEZE_MAX); + gate.arm(now); if last_kf_req .is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) { @@ -679,41 +561,23 @@ fn pump( } } - // Loss recovery: under infinite GOP the only recovery keyframe is one we request. The - // reassembler drops unrecoverable AUs (frames_dropped); the decoder then conceals the - // reference-missing delta frames that follow and returns Ok, so keying off a decode error - // rarely fires. Request an IDR when the drop count climbs, throttled — the decode stays - // wedged for several frames until the IDR lands, so requesting every frame would flood. + // Loss recovery + overdue backstop, folded through the shared gate. A climb in the + // reassembler's unrecoverable-drop count (`frames_dropped`) means the AUs after the lost one + // reference a picture we never decoded — the decoder conceals them (gray on RADV) and returns + // Ok, so a decode-error trigger rarely fires; the gate arms the freeze on the climb instead. An + // overdue freeze (held a full REANCHOR_FREEZE_MAX with no clean re-anchor — a lost recovery IDR, + // or a benign reorder that produced no `frames_dropped`) re-asks while it keeps holding: NEVER + // resume to gray — a genuinely dead stream is the QUIC idle-timeout watchdog's job. Both route + // the gate's keyframe intent through the shared 100 ms throttle; under infinite GOP the only + // recovery keyframe is one we request. let dropped = connector.frames_dropped(); - if dropped > last_dropped { - last_dropped = dropped; - let now = Instant::now(); - // A dropped AU means the frames after it reference a picture we never decoded — the - // decoder will conceal them (gray on RADV). Freeze on the last good frame until a fresh - // IDR re-anchors, so the concealment never reaches the screen. - awaiting_reanchor = true; - recovery_marks = 0; - reanchor_deadline = Some(now + REANCHOR_FREEZE_MAX); - if last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) { - last_kf_req = Some(now); - let _ = connector.request_keyframe(); - tracing::debug!(dropped, "requested keyframe (loss recovery)"); - } - } - // Re-anchor overdue: the freeze has held the whole window with no keyframe — a lost recovery - // IDR, or a benign reorder that produced no `frames_dropped` and so requested none. Do NOT - // resume to gray (the one thing worse than a freeze): keep holding the last good frame and - // (re-)request a keyframe, throttled + host-coalesced, so a CLEAN re-anchor is what un-freezes - // us. A genuinely dead stream — host gone, link collapsed — is caught by the QUIC idle-timeout - // watchdog (returns to the menu), never by painting the decoder's concealment. - if awaiting_reanchor && reanchor_deadline.is_some_and(|d| Instant::now() >= d) { - let now = Instant::now(); - reanchor_deadline = Some(now + REANCHOR_FREEZE_MAX); - if last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) { - last_kf_req = Some(now); - let _ = connector.request_keyframe(); - tracing::debug!("re-anchor overdue — still holding, re-requesting keyframe"); - } + let now = Instant::now(); + if gate.poll(dropped, now) + && last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) + { + last_kf_req = Some(now); + let _ = connector.request_keyframe(); + tracing::debug!(dropped, "requested keyframe (loss recovery / overdue re-anchor)"); } if window_start.elapsed() >= Duration::from_secs(1) { @@ -836,111 +700,3 @@ fn spawn_audio( .map_err(|e| tracing::warn!(error = %e, "audio thread failed to start — audio disabled")) .ok() } - -#[cfg(test)] -mod tests { - use super::{index_gap, reanchor_after_frame, REANCHOR_MARKS_TO_LIFT}; - - // Simulate the pump's re-anchor state across a sequence of decoded frames: each `(is_keyframe, - // has_mark)` pair is folded through `reanchor_after_frame`, returning the frame index (0-based) - // at which the freeze first lifts, or `None` if it never does. `gap_before` reset points model a - // fresh loss re-arming the freeze (the pump zeroes the count at every gap/arm site). - fn lift_at(frames: &[(bool, bool)]) -> Option { - let mut marks = 0u32; - for (i, &(is_kf, has_mark)) in frames.iter().enumerate() { - // The intra-refresh-mark model never carries an LTR-RFI anchor (that path is exercised - // by `an_rfi_anchor_lifts_immediately`), so `has_anchor` is always false here. - let (lift, m) = reanchor_after_frame(is_kf, false, has_mark, marks); - marks = m; - if lift { - return Some(i); - } - } - None - } - - #[test] - fn a_single_recovery_mark_does_not_lift() { - // The first wave boundary after a loss is only half-healed — one mark must hold the freeze. - assert_eq!(REANCHOR_MARKS_TO_LIFT, 2); - assert_eq!(lift_at(&[(false, true)]), None); - assert_eq!( - lift_at(&[(false, false), (false, true), (false, false)]), - None - ); - } - - #[test] - fn the_second_recovery_mark_lifts() { - // Two marks = a full wave swept after the loss → clean re-anchor. - assert_eq!(lift_at(&[(false, true), (false, true)]), Some(1)); - assert_eq!( - lift_at(&[(false, false), (false, true), (false, false), (false, true)]), - Some(3) - ); - } - - #[test] - fn a_real_keyframe_lifts_immediately() { - // An IDR is always a clean anchor — no marks needed. - assert_eq!(lift_at(&[(true, false)]), Some(0)); - assert_eq!(lift_at(&[(false, true), (true, false)]), Some(1)); - } - - #[test] - fn a_fresh_gap_resets_the_mark_count() { - // The pump zeroes `recovery_marks` at each arm site, so one mark before a new gap plus one - // after must NOT lift — the model resets the running count to imitate that. - let mut marks = 0u32; - let (_, m) = reanchor_after_frame(false, false, true, marks); // mark #1 (pre-gap) - marks = m; - assert_eq!(marks, 1); - marks = 0; // a new gap re-arms the freeze → count reset - let (lift, m) = reanchor_after_frame(false, false, true, marks); // first mark of the new wave - assert!(!lift, "a single post-gap mark must not lift"); - assert_eq!(m, 1); - } - - #[test] - fn an_rfi_anchor_lifts_immediately() { - // An LTR-RFI recovery anchor is a WHOLE re-anchor (a clean P-frame off a known-good - // reference), so — like an IDR — it lifts on the FIRST occurrence, no two-mark wait. - let (lift, marks) = reanchor_after_frame(false, true, false, 0); - assert!(lift, "an RFI anchor must lift the freeze immediately"); - assert_eq!(marks, 0, "a lift resets the running mark count"); - // Even with zero prior marks and no keyframe, the anchor alone is sufficient. - let (lift, _) = reanchor_after_frame(false, true, true, 1); - assert!(lift, "an anchor lifts regardless of the pending mark count"); - } - - #[test] - fn contiguous_indices_are_not_a_gap() { - assert_eq!(index_gap(5, 5), None); - assert_eq!(index_gap(0, 0), None); - } - - #[test] - fn a_forward_jump_reports_the_skip_count() { - assert_eq!(index_gap(5, 6), Some(1)); // one frame missing - assert_eq!(index_gap(5, 9), Some(4)); - } - - #[test] - fn a_straggler_behind_us_is_not_a_gap() { - // The reassembler emitted a newer frame first; the late one must not re-arm. - assert_eq!(index_gap(9, 5), None); - assert_eq!(index_gap(1, 0), None); - } - - #[test] - fn the_index_counter_wraps_cleanly() { - // last frame = u32::MAX, so the next expected wraps to 0. - // Contiguous across the wrap. - assert_eq!(index_gap(0, 0), None); - // waiting on u32::MAX, frame 0 arrived → MAX was skipped. - assert_eq!(index_gap(u32::MAX, 0), Some(1)); - assert_eq!(index_gap(u32::MAX, 2), Some(3)); - // an old frame arriving just after the wrap is still a straggler. - assert_eq!(index_gap(0, u32::MAX), None); - } -} diff --git a/crates/punktfunk-core/src/abi.rs b/crates/punktfunk-core/src/abi.rs index 82bf497f..6dca2b29 100644 --- a/crates/punktfunk-core/src/abi.rs +++ b/crates/punktfunk-core/src/abi.rs @@ -13,6 +13,7 @@ use crate::config::{Config, FecConfig, FecScheme, ProtocolPhase, Role}; use crate::error::PunktfunkStatus; use crate::input::InputEvent; +use crate::reanchor::{GateVerdict, ReanchorGate}; use crate::session::Session; use crate::stats::Stats; use crate::transport::{loopback_pair, Transport, UdpTransport}; @@ -2620,3 +2621,142 @@ pub unsafe extern "C" fn punktfunk_connection_close(c: *mut PunktfunkConnection) drop(unsafe { Box::from_raw(c) }); } } + +// ---- Post-loss re-anchor freeze gate ---- +// +// The shared [`ReanchorGate`](crate::reanchor::ReanchorGate) exposed for the Swift client (Rust +// embedders — Android/Windows/Linux — use the struct directly). After an unrecoverable reference +// loss the decoder silently conceals the missing-reference deltas (gray/garbage picture, no error); +// the client freezes on the last good frame and lifts only on a proven clean re-anchor. The gate +// takes time internally (`Instant::now`) so no timestamps cross the boundary. Drive it per session: +// `arm` on a loss (frame-index gap from `punktfunk_connection_note_frame_index`, a decoder +// wedge/demotion), `on_decoded` per decoded frame to gate presentation, `on_no_output` per AU that +// produced nothing, and `poll` each iteration for the dropped-count climb + overdue backstop. Route +// the returned keyframe intents through the client's existing request throttle. + +/// Create a re-anchor gate seeded with the session's current `frames_dropped` (so the first +/// [`punktfunk_reanchor_gate_poll`] doesn't read the baseline as a loss). Free with +/// [`punktfunk_reanchor_gate_free`]. Never returns NULL. +#[no_mangle] +pub extern "C" fn punktfunk_reanchor_gate_new(frames_dropped: u64) -> *mut ReanchorGate { + Box::into_raw(Box::new(ReanchorGate::new(frames_dropped))) +} + +/// Free a gate created by [`punktfunk_reanchor_gate_new`]. NULL is a no-op. +/// +/// # Safety +/// `g` was returned by [`punktfunk_reanchor_gate_new`] and is not used after this call. +#[no_mangle] +pub unsafe extern "C" fn punktfunk_reanchor_gate_free(g: *mut ReanchorGate) { + if !g.is_null() { + drop(unsafe { Box::from_raw(g) }); + } +} + +/// Arm the freeze: a loss was detected (a frame-index gap, or a decoder wedge/demotion). Zeroes the +/// recovery-mark count and (re-)sets the backstop deadline. NULL is a no-op. +/// +/// # Safety +/// `g` is a valid gate handle. +#[no_mangle] +pub unsafe extern "C" fn punktfunk_reanchor_gate_arm(g: *mut ReanchorGate) { + if let Some(g) = unsafe { g.as_mut() } { + g.arm(std::time::Instant::now()); + } +} + +/// Fold one decoded frame and write to `out_present` whether to display it (`true`) or withhold it as +/// a post-loss concealment (`false`). `flags` is the AU's `user_flags` word ([`PunktfunkFrame::flags`]): +/// the gate reads `FLAG_SOF` (the host's IDR marker), `USER_FLAG_RECOVERY_ANCHOR` and +/// `USER_FLAG_RECOVERY_POINT`. Pass `decoder_keyframe = false` where the platform decoder doesn't flag +/// IDRs (VideoToolbox/MediaCodec) — the wire `FLAG_SOF` covers it. +/// +/// # Safety +/// `g` is a valid gate handle; `out_present` is writable or NULL. +#[no_mangle] +pub unsafe extern "C" fn punktfunk_reanchor_gate_on_decoded( + g: *mut ReanchorGate, + flags: u32, + decoder_keyframe: bool, + out_present: *mut bool, +) -> PunktfunkStatus { + guard(|| { + let g = match unsafe { g.as_mut() } { + Some(g) => g, + None => return PunktfunkStatus::NullPointer, + }; + let present = g.on_decoded(flags, decoder_keyframe, std::time::Instant::now()) == GateVerdict::Present; + if !out_present.is_null() { + unsafe { *out_present = present }; + } + PunktfunkStatus::Ok + }) +} + +/// A received AU produced no decoded frame. Writes to `out_request_kf` whether the no-output streak has +/// tripped and the client should (throttled) request a keyframe — the gate arms the freeze at the same +/// time. +/// +/// # Safety +/// `g` is a valid gate handle; `out_request_kf` is writable or NULL. +#[no_mangle] +pub unsafe extern "C" fn punktfunk_reanchor_gate_on_no_output( + g: *mut ReanchorGate, + out_request_kf: *mut bool, +) -> PunktfunkStatus { + guard(|| { + let g = match unsafe { g.as_mut() } { + Some(g) => g, + None => return PunktfunkStatus::NullPointer, + }; + let request = g.on_no_output(std::time::Instant::now()); + if !out_request_kf.is_null() { + unsafe { *out_request_kf = request }; + } + PunktfunkStatus::Ok + }) +} + +/// Periodic fold of the session's `frames_dropped` counter plus the overdue backstop. Writes to +/// `out_request_kf` whether the client should (throttled) request a keyframe (a drop-count climb armed +/// a fresh freeze, or the freeze is overdue and re-asks while it keeps holding). +/// +/// # Safety +/// `g` is a valid gate handle; `out_request_kf` is writable or NULL. +#[no_mangle] +pub unsafe extern "C" fn punktfunk_reanchor_gate_poll( + g: *mut ReanchorGate, + frames_dropped: u64, + out_request_kf: *mut bool, +) -> PunktfunkStatus { + guard(|| { + let g = match unsafe { g.as_mut() } { + Some(g) => g, + None => return PunktfunkStatus::NullPointer, + }; + let request = g.poll(frames_dropped, std::time::Instant::now()); + if !out_request_kf.is_null() { + unsafe { *out_request_kf = request }; + } + PunktfunkStatus::Ok + }) +} + +/// Whether the gate is currently withholding concealed frames (frozen on the last good picture). +/// Writes `false` on a NULL gate. +/// +/// # Safety +/// `g` is a valid gate handle; `out_holding` is writable or NULL. +#[no_mangle] +pub unsafe extern "C" fn punktfunk_reanchor_gate_is_holding( + g: *const ReanchorGate, + out_holding: *mut bool, +) -> PunktfunkStatus { + guard(|| { + let holding = unsafe { g.as_ref() }.is_some_and(ReanchorGate::is_holding); + if !out_holding.is_null() { + unsafe { *out_holding = holding }; + } + PunktfunkStatus::Ok + }) +} diff --git a/crates/punktfunk-core/src/lib.rs b/crates/punktfunk-core/src/lib.rs index b3fe949e..781034a5 100644 --- a/crates/punktfunk-core/src/lib.rs +++ b/crates/punktfunk-core/src/lib.rs @@ -38,6 +38,7 @@ pub mod input; pub mod packet; #[cfg(feature = "quic")] pub mod quic; +pub mod reanchor; pub mod session; pub mod stats; pub mod transport; @@ -61,7 +62,10 @@ pub use stats::Stats; /// TTL of a v2 envelope; `punktfunk_connection_next_rumble` is unchanged and drops it). Additive — /// the wire is backward-compatible (the envelope is a length-tolerant tail on 0xCA), so /// [`WIRE_VERSION`] is unchanged. -pub const ABI_VERSION: u32 = 5; +/// v6: added the `punktfunk_reanchor_gate_*` surface (post-loss freeze-until-reanchor gate for the +/// Swift client; Rust embedders use [`reanchor::ReanchorGate`] directly). Additive, client-local — +/// no wire change, so [`WIRE_VERSION`] is unchanged. +pub const ABI_VERSION: u32 = 6; /// The punktfunk/1 **wire** version — what `Hello`/`Welcome` carry and hosts equality-check. /// Deliberately its own constant: [`ABI_VERSION`] tracks the embeddable **C surface** diff --git a/crates/punktfunk-core/src/reanchor.rs b/crates/punktfunk-core/src/reanchor.rs new file mode 100644 index 00000000..d768775f --- /dev/null +++ b/crates/punktfunk-core/src/reanchor.rs @@ -0,0 +1,456 @@ +//! Post-loss display freeze — the shared "freeze-until-reanchor" gate. +//! +//! After an unrecoverable reference loss the hardware decoder does **not** error: it *conceals* the +//! reference-missing delta frames (on RADV, the DPB-and-output-COINCIDE path paints a gray plate with +//! the new frame's motion on top) and returns Ok. Displaying that is the "gray frames mid-stream" +//! artifact. Instead every client freezes on the last good picture — withholds the concealed frames +//! from its presenter, which keeps redrawing the held frame — and lifts the freeze ONLY on a proven +//! clean re-anchor: a real IDR, an LTR-RFI recovery anchor ([`USER_FLAG_RECOVERY_ANCHOR`]), or the +//! second intra-refresh recovery mark ([`USER_FLAG_RECOVERY_POINT`]) since the loss. +//! +//! This module owns that decision so every embedder shares ONE implementation instead of re-deriving +//! it (the Linux/Deck pump in `pf-client-core`, the Windows in-process pump, the Android decode loops, +//! and — over the C ABI — the Apple client). The state machine is time-driven but takes `now` as a +//! parameter so it is unit-testable without a clock; the C ABI wrappers supply `Instant::now()`. +//! +//! [`USER_FLAG_RECOVERY_POINT`]: crate::packet::USER_FLAG_RECOVERY_POINT +//! [`USER_FLAG_RECOVERY_ANCHOR`]: crate::packet::USER_FLAG_RECOVERY_ANCHOR + +use crate::packet::{FLAG_SOF, USER_FLAG_RECOVERY_ANCHOR, USER_FLAG_RECOVERY_POINT}; +use std::time::{Duration, Instant}; + +/// Consecutive no-output AUs that force a keyframe request. ~50 ms at 60 Hz — long enough not to fire +/// on a one-frame decoder hiccup, short enough that a lost initial IDR (or a mid-GOP join) unfreezes +/// almost immediately instead of never. +pub const NO_OUTPUT_KEYFRAME_STREAK: u32 = 3; + +/// Longest the gate holds the last good frame waiting for a post-loss re-anchor keyframe before it +/// re-asks. After a reference loss the hardware decoder does not error — it conceals the +/// reference-missing deltas (on RADV, the DPB-and-output-COINCIDE path renders them as a gray plate +/// with the new frame's motion painted over it) and returns Ok, so displaying them is the "gray frames +/// mid-stream" artifact. We instead freeze on the last good picture until a fresh IDR re-anchors decode +/// — the behaviour NVIDIA already shows (its DISTINCT output image + different concealment reads as a +/// brief freeze, not gray). This cap only bounds the freeze when recovery genuinely stalls (host +/// ignores the request, or an RFI recovery that never emits a keyframe): the freeze is NEVER lifted to +/// the concealed picture — the deadline re-asks for a keyframe and keeps holding, so a glitch can never +/// become a permanent freeze while a clean re-anchor is what un-freezes. A recovery IDR round-trips well +/// under this on any live link. +pub const REANCHOR_FREEZE_MAX: Duration = Duration::from_millis(500); + +/// How many host intra-refresh recovery marks ([`USER_FLAG_RECOVERY_POINT`]) must arrive since the +/// latest loss before the gate lifts its freeze on an IDR-free stream. TWO, not one: with a continuous +/// rolling wave the host marks phase-fixed wave boundaries, so the FIRST boundary after a loss is only +/// partially healed — stripes swept BEFORE the loss still reference the lost frame — and lifting there +/// would flash a partially-stale picture. The SECOND boundary guarantees a full wave swept entirely +/// after the loss, so the picture is clean. This stays correct under repeated loss because every fresh +/// arm resets the count. The cost is up to ~2 wave periods of holding the last good frame — the +/// deliberate "hold longer, never show garbage" trade. +/// +/// [`USER_FLAG_RECOVERY_POINT`]: crate::packet::USER_FLAG_RECOVERY_POINT +pub const REANCHOR_MARKS_TO_LIFT: u32 = 2; + +/// Backstop patience while a host intra-refresh heal is visibly in progress. Each recovery mark pushes +/// the freeze deadline out by this much, so a live mark stream (the host actively healing via its wave) +/// keeps the gate patiently holding the last good frame instead of tripping the IDR floor mid-heal. +/// Must exceed the inter-mark interval (one wave period, ~0.5 s) with margin; if the marks STOP (heal +/// stalled, or the host isn't running intra-refresh) the deadline lapses and the normal recovery-IDR +/// floor fires, so a real stall still recovers. +pub const RECOVERY_MARK_PATIENCE: Duration = Duration::from_millis(1500); + +/// Frames skipped when `got` arrives while `expected` was the next index, or `None` if `got` is +/// contiguous (`== expected`) or a straggler we have already passed. Frame indices are u32 counters +/// that wrap, so the "ahead" test is a wrapping subtraction split at the half-space: a small positive +/// delta is a forward gap (missing frames whose dependents will decode against absent references); a +/// delta in the top half is an index behind us. +pub fn index_gap(expected: u32, got: u32) -> Option { + let ahead = got.wrapping_sub(expected); + (ahead != 0 && ahead < u32::MAX / 2).then_some(ahead) +} + +/// Fold one decoded frame into the re-anchor state and decide whether it lifts the post-loss freeze. +/// +/// `is_keyframe` — a real IDR (always a clean re-anchor). `has_anchor` — this AU carried +/// [`USER_FLAG_RECOVERY_ANCHOR`](crate::packet::USER_FLAG_RECOVERY_ANCHOR), the host's definitive +/// single-frame re-anchor from an LTR-RFI recovery (a clean P-frame coded against a known-good +/// reference), so it lifts on the FIRST occurrence exactly like an IDR — no two-mark wait. `has_mark` — +/// this AU carried [`USER_FLAG_RECOVERY_POINT`](crate::packet::USER_FLAG_RECOVERY_POINT), a +/// host-signalled intra-refresh wave boundary (only *half* a re-anchor). `marks` — recovery marks seen +/// since the latest loss. +/// +/// Returns `(lift, new_marks)`: `lift` clears the freeze; `new_marks` is the running count (reset to 0 +/// on a lift). The two-mark rule ([`REANCHOR_MARKS_TO_LIFT`]) lives here so it is unit-tested +/// independent of the pump's channel/decoder plumbing — the first wave boundary after a loss is only +/// partially healed, so a single mark must NOT lift. An anchor (or IDR) is a *whole* re-anchor and +/// lifts immediately. +fn reanchor_after_frame(is_keyframe: bool, has_anchor: bool, has_mark: bool, marks: u32) -> (bool, u32) { + let marks = if has_mark { + marks.saturating_add(1) + } else { + marks + }; + if is_keyframe || has_anchor || marks >= REANCHOR_MARKS_TO_LIFT { + (true, 0) + } else { + (false, marks) + } +} + +/// Whether a decoded frame should be shown or withheld while the gate is (or isn't) frozen. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GateVerdict { + /// Present this frame — the gate is not frozen, or this frame is the clean re-anchor that lifts it. + Present, + /// Withhold this frame — it is a post-loss concealment; the presenter keeps the last good picture. + Hold, +} + +/// The shared post-loss freeze state machine. A client feeds it three kinds of event — an *arm* (a +/// loss was detected: a frame-index gap, a dropped-count climb, or a decoder wedge/demotion), each +/// *decoded frame* ([`on_decoded`](Self::on_decoded), which decides present-vs-hold and interprets the +/// re-anchor wire flags), and each *no-output* AU ([`on_no_output`](Self::on_no_output)) — plus a +/// periodic [`poll`](Self::poll) that folds the dropped counter and fires the overdue backstop. +/// +/// The gate emits *intents* only: [`on_no_output`](Self::on_no_output) and [`poll`](Self::poll) return +/// `true` when the client should ask the host for a keyframe. The client routes that through its own +/// ~100 ms request throttle (and the precise RFI-vs-keyframe range decision stays in the loss-range +/// tracker behind [`crate::client::NativeClient::note_frame_index`]) — the gate never touches the wire. +#[derive(Debug, Clone)] +pub struct ReanchorGate { + /// Frozen on the last good frame, withholding the decoder's concealed output until a clean + /// re-anchor. Armed by any loss signal; cleared only by [`on_decoded`](Self::on_decoded) lifting. + awaiting: bool, + /// Host intra-refresh recovery marks seen since the latest arm (see [`REANCHOR_MARKS_TO_LIFT`]). + /// Reset to 0 whenever the freeze is (re-)armed, so a fresh loss always waits out two fresh marks. + marks: u32, + /// When the freeze becomes overdue and [`poll`](Self::poll) re-asks for a keyframe (holding, never + /// resuming to the concealed picture). `None` when not frozen. + deadline: Option, + /// Consecutive received AUs that produced no decoded frame — a decoder wedged on missing references + /// with no reassembler drop to trigger recovery. A short streak forces a fresh IDR. + no_output_streak: u32, + /// The last `frames_dropped` value [`poll`](Self::poll) observed; a climb means the reassembler + /// declared an AU unrecoverable and the following deltas will conceal, so arm. + last_dropped: u64, +} + +impl ReanchorGate { + /// Seed the gate with the session's current `frames_dropped` so the first [`poll`](Self::poll) + /// doesn't read the baseline as a loss. + pub fn new(frames_dropped: u64) -> Self { + ReanchorGate { + awaiting: false, + marks: 0, + deadline: None, + no_output_streak: 0, + last_dropped: frames_dropped, + } + } + + /// Arm the freeze: a loss was detected (a frame-index gap, a dropped-count climb, or a decoder + /// wedge/demotion). Zeroes the mark count so a fresh loss waits out two fresh recovery marks, and + /// (re-)sets the backstop deadline. Idempotent while already frozen (re-arming just re-zeroes the + /// marks and pushes the deadline — the correct behaviour when a second loss lands mid-freeze). + pub fn arm(&mut self, now: Instant) { + self.awaiting = true; + self.marks = 0; + self.deadline = Some(now + REANCHOR_FREEZE_MAX); + } + + /// Fold one decoded frame and decide whether to present or withhold it. + /// + /// `wire_flags` is the AU's `user_flags` word ([`crate::session::Frame::flags`] / + /// `PunktfunkFrame.flags`); the gate reads [`FLAG_SOF`](crate::packet::FLAG_SOF) (the host sets it + /// only on IDR AUs — the codec-agnostic keyframe signal the platform decoders don't expose), + /// [`USER_FLAG_RECOVERY_ANCHOR`] and [`USER_FLAG_RECOVERY_POINT`]. `decoder_keyframe` is an optional + /// belt from decoders that flag IDRs themselves (libavcodec's `AV_FRAME_FLAG_KEY` on Linux/Windows); + /// pass `false` where the decoder doesn't (Android MediaCodec, Apple VideoToolbox) and rely on the + /// wire `FLAG_SOF`. + /// + /// A decoded frame always clears the no-output streak. When frozen, a live mark stream pushes the + /// backstop out ([`RECOVERY_MARK_PATIENCE`]) so a healing wave isn't pre-empted by a mid-heal IDR. + /// + /// [`USER_FLAG_RECOVERY_ANCHOR`]: crate::packet::USER_FLAG_RECOVERY_ANCHOR + /// [`USER_FLAG_RECOVERY_POINT`]: crate::packet::USER_FLAG_RECOVERY_POINT + pub fn on_decoded(&mut self, wire_flags: u32, decoder_keyframe: bool, now: Instant) -> GateVerdict { + self.no_output_streak = 0; + let is_keyframe = decoder_keyframe || (wire_flags & FLAG_SOF as u32 != 0); + let has_anchor = wire_flags & USER_FLAG_RECOVERY_ANCHOR != 0; + let has_mark = wire_flags & USER_FLAG_RECOVERY_POINT != 0; + if has_mark && self.awaiting { + self.deadline = Some(now + RECOVERY_MARK_PATIENCE); + } + let (lift, marks) = reanchor_after_frame(is_keyframe, has_anchor, has_mark, self.marks); + self.marks = marks; + if lift { + self.awaiting = false; + self.deadline = None; + } + if self.awaiting { + GateVerdict::Hold + } else { + GateVerdict::Present + } + } + + /// A received AU produced no decoded frame (decode error, or the decoder swallowed a + /// reference-missing delta). Returns `true` when the streak has tripped and the client should + /// (throttled) request a keyframe — arming the freeze at the same time, since the stream is broken + /// regardless of whether the throttle lets the request through this iteration. + pub fn on_no_output(&mut self, now: Instant) -> bool { + self.no_output_streak += 1; + if self.no_output_streak >= NO_OUTPUT_KEYFRAME_STREAK { + self.arm(now); + self.no_output_streak = 0; + true + } else { + false + } + } + + /// Periodic fold of the session's `frames_dropped` counter plus the overdue backstop. Returns + /// `true` when the client should (throttled) request a keyframe: either the drop count climbed (a + /// fresh unrecoverable loss — arm the freeze) or the freeze has held a full [`REANCHOR_FREEZE_MAX`] + /// window with no re-anchor (re-ask and keep holding — NEVER resume to the concealed picture; a + /// genuinely dead stream is the QUIC idle-timeout watchdog's job, not the gate's). + pub fn poll(&mut self, frames_dropped: u64, now: Instant) -> bool { + let mut want_keyframe = false; + if frames_dropped > self.last_dropped { + self.last_dropped = frames_dropped; + self.arm(now); + want_keyframe = true; + } + if self.awaiting && self.deadline.is_some_and(|d| now >= d) { + self.deadline = Some(now + REANCHOR_FREEZE_MAX); + want_keyframe = true; + } + want_keyframe + } + + /// Whether the gate is currently withholding concealed frames (frozen on the last good picture). + pub fn is_holding(&self) -> bool { + self.awaiting + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Simulate the gate's re-anchor state across a sequence of decoded frames: each `(is_keyframe, + // has_mark)` pair is folded through `reanchor_after_frame`, returning the frame index (0-based) at + // which the freeze first lifts, or `None` if it never does. A reset to 0 models a fresh loss + // re-arming the freeze (the gate zeroes the count at every arm site). + fn lift_at(frames: &[(bool, bool)]) -> Option { + let mut marks = 0u32; + for (i, &(is_kf, has_mark)) in frames.iter().enumerate() { + // The intra-refresh-mark model never carries an LTR-RFI anchor (that path is exercised by + // `an_rfi_anchor_lifts_immediately`), so `has_anchor` is always false here. + let (lift, m) = reanchor_after_frame(is_kf, false, has_mark, marks); + marks = m; + if lift { + return Some(i); + } + } + None + } + + #[test] + fn a_single_recovery_mark_does_not_lift() { + // The first wave boundary after a loss is only half-healed — one mark must hold the freeze. + assert_eq!(REANCHOR_MARKS_TO_LIFT, 2); + assert_eq!(lift_at(&[(false, true)]), None); + assert_eq!(lift_at(&[(false, false), (false, true), (false, false)]), None); + } + + #[test] + fn the_second_recovery_mark_lifts() { + // Two marks = a full wave swept after the loss → clean re-anchor. + assert_eq!(lift_at(&[(false, true), (false, true)]), Some(1)); + assert_eq!( + lift_at(&[(false, false), (false, true), (false, false), (false, true)]), + Some(3) + ); + } + + #[test] + fn a_real_keyframe_lifts_immediately() { + // An IDR is always a clean anchor — no marks needed. + assert_eq!(lift_at(&[(true, false)]), Some(0)); + assert_eq!(lift_at(&[(false, true), (true, false)]), Some(1)); + } + + #[test] + fn a_fresh_gap_resets_the_mark_count() { + // The gate zeroes `marks` at each arm site, so one mark before a new gap plus one after must + // NOT lift — the model resets the running count to imitate that. + let mut marks = 0u32; + let (_, m) = reanchor_after_frame(false, false, true, marks); // mark #1 (pre-gap) + marks = m; + assert_eq!(marks, 1); + marks = 0; // a new gap re-arms the freeze → count reset + let (lift, m) = reanchor_after_frame(false, false, true, marks); // first mark of the new wave + assert!(!lift, "a single post-gap mark must not lift"); + assert_eq!(m, 1); + } + + #[test] + fn an_rfi_anchor_lifts_immediately() { + // An LTR-RFI recovery anchor is a WHOLE re-anchor (a clean P-frame off a known-good reference), + // so — like an IDR — it lifts on the FIRST occurrence, no two-mark wait. + let (lift, marks) = reanchor_after_frame(false, true, false, 0); + assert!(lift, "an RFI anchor must lift the freeze immediately"); + assert_eq!(marks, 0, "a lift resets the running mark count"); + // Even with zero prior marks and no keyframe, the anchor alone is sufficient. + let (lift, _) = reanchor_after_frame(false, true, true, 1); + assert!(lift, "an anchor lifts regardless of the pending mark count"); + } + + #[test] + fn contiguous_indices_are_not_a_gap() { + assert_eq!(index_gap(5, 5), None); + assert_eq!(index_gap(0, 0), None); + } + + #[test] + fn a_forward_jump_reports_the_skip_count() { + assert_eq!(index_gap(5, 6), Some(1)); // one frame missing + assert_eq!(index_gap(5, 9), Some(4)); + } + + #[test] + fn a_straggler_behind_us_is_not_a_gap() { + // The reassembler emitted a newer frame first; the late one must not re-arm. + assert_eq!(index_gap(9, 5), None); + assert_eq!(index_gap(1, 0), None); + } + + #[test] + fn the_index_counter_wraps_cleanly() { + // last frame = u32::MAX, so the next expected wraps to 0. + assert_eq!(index_gap(0, 0), None); + // waiting on u32::MAX, frame 0 arrived → MAX was skipped. + assert_eq!(index_gap(u32::MAX, 0), Some(1)); + assert_eq!(index_gap(u32::MAX, 2), Some(3)); + // an old frame arriving just after the wrap is still a straggler. + assert_eq!(index_gap(0, u32::MAX), None); + } + + // ---- gate-level sequence tests (the whole behavioural contract) ---- + + const SOF: u32 = FLAG_SOF as u32; // IDR wire flag + const ANCHOR: u32 = USER_FLAG_RECOVERY_ANCHOR; + const POINT: u32 = USER_FLAG_RECOVERY_POINT; + + fn t0() -> Instant { + Instant::now() + } + + #[test] + fn a_clean_link_never_holds() { + // Disarmed gate presents every frame, keyframe or not, and never asks for anything. + let mut g = ReanchorGate::new(0); + let now = t0(); + assert_eq!(g.on_decoded(0, false, now), GateVerdict::Present); + assert_eq!(g.on_decoded(SOF, true, now), GateVerdict::Present); + assert!(!g.is_holding()); + assert!(!g.poll(0, now)); + } + + #[test] + fn a_gap_holds_until_the_wire_keyframe_lifts() { + // Android/Apple path: no decoder keyframe flag, lift comes from the wire FLAG_SOF alone. + let mut g = ReanchorGate::new(0); + let now = t0(); + g.arm(now); // frame-index gap + assert!(g.is_holding()); + assert_eq!(g.on_decoded(0, false, now), GateVerdict::Hold); // concealed delta withheld + assert_eq!(g.on_decoded(0, false, now), GateVerdict::Hold); + assert_eq!(g.on_decoded(SOF, false, now), GateVerdict::Present); // IDR re-anchors + assert!(!g.is_holding()); + assert_eq!(g.on_decoded(0, false, now), GateVerdict::Present); // stays presenting + } + + #[test] + fn a_gap_lifts_on_the_first_rfi_anchor() { + let mut g = ReanchorGate::new(0); + let now = t0(); + g.arm(now); + assert_eq!(g.on_decoded(0, false, now), GateVerdict::Hold); + assert_eq!(g.on_decoded(ANCHOR, false, now), GateVerdict::Present); + assert!(!g.is_holding()); + } + + #[test] + fn a_gap_lifts_on_the_second_recovery_mark() { + let mut g = ReanchorGate::new(0); + let now = t0(); + g.arm(now); + assert_eq!(g.on_decoded(POINT, false, now), GateVerdict::Hold); // first boundary: half-healed + assert_eq!(g.on_decoded(0, false, now), GateVerdict::Hold); + assert_eq!(g.on_decoded(POINT, false, now), GateVerdict::Present); // second: clean + } + + #[test] + fn a_second_gap_mid_freeze_resets_the_marks() { + let mut g = ReanchorGate::new(0); + let now = t0(); + g.arm(now); + assert_eq!(g.on_decoded(POINT, false, now), GateVerdict::Hold); // mark #1 + g.arm(now); // a fresh loss re-arms → mark count zeroed + assert_eq!(g.on_decoded(POINT, false, now), GateVerdict::Hold); // this is mark #1 of the new wave + assert_eq!(g.on_decoded(POINT, false, now), GateVerdict::Present); // #2 lifts + } + + #[test] + fn the_dropped_climb_arms_and_asks() { + let mut g = ReanchorGate::new(5); + let now = t0(); + assert!(!g.poll(5, now), "no climb → no ask"); // baseline + assert!(g.poll(6, now), "a climb asks for a keyframe"); + assert!(g.is_holding(), "and arms the freeze"); + assert!(!g.poll(6, now), "same value → no repeat ask from the drop path"); + } + + #[test] + fn the_no_output_streak_trips_at_three() { + let mut g = ReanchorGate::new(0); + let now = t0(); + assert!(!g.on_no_output(now)); + assert!(!g.on_no_output(now)); + assert!(g.on_no_output(now), "third no-output trips the streak"); + assert!(g.is_holding()); + // A decoded frame resets the streak. + g.on_decoded(SOF, false, now); // lifts + resets streak + assert!(!g.on_no_output(now)); + assert!(!g.on_no_output(now)); + assert!(g.on_no_output(now)); + } + + #[test] + fn an_overdue_freeze_re_asks_but_keeps_holding() { + let mut g = ReanchorGate::new(0); + let start = t0(); + g.arm(start); + // Before the deadline: holding, no re-ask. + assert!(!g.poll(0, start)); + assert!(g.is_holding()); + // Past REANCHOR_FREEZE_MAX with no re-anchor: re-ask, still holding. + let later = start + REANCHOR_FREEZE_MAX + Duration::from_millis(1); + assert!(g.poll(0, later), "overdue freeze re-asks for a keyframe"); + assert!(g.is_holding(), "but never resumes to the concealed picture"); + } + + #[test] + fn a_live_mark_stream_pushes_the_deadline_out() { + // A healing wave (marks arriving) must not be pre-empted by the overdue IDR floor. + let mut g = ReanchorGate::new(0); + let start = t0(); + g.arm(start); + // A mark past the original freeze deadline pushes it out by RECOVERY_MARK_PATIENCE. + let t = start + REANCHOR_FREEZE_MAX + Duration::from_millis(10); + assert_eq!(g.on_decoded(POINT, false, t), GateVerdict::Hold); // mark #1, deadline pushed + // At a time that WOULD have been overdue on the original deadline, poll does not re-ask. + assert!(!g.poll(0, t + Duration::from_millis(1))); + assert!(g.is_holding()); + } +} diff --git a/include/punktfunk_core.h b/include/punktfunk_core.h index 47625b49..f7bc8d36 100644 --- a/include/punktfunk_core.h +++ b/include/punktfunk_core.h @@ -25,7 +25,10 @@ // TTL of a v2 envelope; `punktfunk_connection_next_rumble` is unchanged and drops it). Additive — // the wire is backward-compatible (the envelope is a length-tolerant tail on 0xCA), so // [`WIRE_VERSION`] is unchanged. -#define ABI_VERSION 5 +// v6: added the `punktfunk_reanchor_gate_*` surface (post-loss freeze-until-reanchor gate for the +// Swift client; Rust embedders use [`reanchor::ReanchorGate`] directly). Additive, client-local — +// no wire change, so [`WIRE_VERSION`] is unchanged. +#define ABI_VERSION 6 // The punktfunk/1 **wire** version — what `Hello`/`Welcome` carry and hosts equality-check. // Deliberately its own constant: [`ABI_VERSION`] tracks the embeddable **C surface** @@ -586,6 +589,23 @@ #define ColorInfo_MC_BT2020_NCL 9 #endif +// Consecutive no-output AUs that force a keyframe request. ~50 ms at 60 Hz — long enough not to fire +// on a one-frame decoder hiccup, short enough that a lost initial IDR (or a mid-GOP join) unfreezes +// almost immediately instead of never. +#define NO_OUTPUT_KEYFRAME_STREAK 3 + +// How many host intra-refresh recovery marks ([`USER_FLAG_RECOVERY_POINT`]) must arrive since the +// latest loss before the gate lifts its freeze on an IDR-free stream. TWO, not one: with a continuous +// rolling wave the host marks phase-fixed wave boundaries, so the FIRST boundary after a loss is only +// partially healed — stripes swept BEFORE the loss still reference the lost frame — and lifting there +// would flash a partially-stale picture. The SECOND boundary guarantees a full wave swept entirely +// after the loss, so the picture is clean. This stays correct under repeated loss because every fresh +// arm resets the count. The cost is up to ~2 wave periods of holding the last good frame — the +// deliberate "hold longer, never show garbage" trade. +// +// [`USER_FLAG_RECOVERY_POINT`]: crate::packet::USER_FLAG_RECOVERY_POINT +#define REANCHOR_MARKS_TO_LIFT 2 + // Stable C ABI status codes. `Ok` is 0; all errors are negative so callers can // test `rc < 0`. Do not renumber existing variants — only append. enum PunktfunkStatus @@ -714,6 +734,18 @@ typedef struct PunktfunkConnection PunktfunkConnection; // Opaque session handle. Pointer-only from C. typedef struct PunktfunkSession PunktfunkSession; +// The shared post-loss freeze state machine. A client feeds it three kinds of event — an *arm* (a +// loss was detected: a frame-index gap, a dropped-count climb, or a decoder wedge/demotion), each +// *decoded frame* ([`on_decoded`](Self::on_decoded), which decides present-vs-hold and interprets the +// re-anchor wire flags), and each *no-output* AU ([`on_no_output`](Self::on_no_output)) — plus a +// periodic [`poll`](Self::poll) that folds the dropped counter and fires the overdue backstop. +// +// The gate emits *intents* only: [`on_no_output`](Self::on_no_output) and [`poll`](Self::poll) return +// `true` when the client should ask the host for a keyframe. The client routes that through its own +// ~100 ms request throttle (and the precise RFI-vs-keyframe range decision stays in the loss-range +// tracker behind [`crate::client::NativeClient::note_frame_index`]) — the gate never touches the wire. +typedef struct ReanchorGate ReanchorGate; + // Forward-compatible session configuration. The caller MUST set `struct_size` to // `sizeof(PunktfunkConfig)`; the core uses it to detect ABI skew. typedef struct { @@ -1737,6 +1769,63 @@ void punktfunk_connection_disconnect_quit(PunktfunkConnection *c); void punktfunk_connection_close(PunktfunkConnection *c); #endif +// Create a re-anchor gate seeded with the session's current `frames_dropped` (so the first +// [`punktfunk_reanchor_gate_poll`] doesn't read the baseline as a loss). Free with +// [`punktfunk_reanchor_gate_free`]. Never returns NULL. +ReanchorGate *punktfunk_reanchor_gate_new(uint64_t frames_dropped); + +// Free a gate created by [`punktfunk_reanchor_gate_new`]. NULL is a no-op. +// +// # Safety +// `g` was returned by [`punktfunk_reanchor_gate_new`] and is not used after this call. +void punktfunk_reanchor_gate_free(ReanchorGate *g); + +// Arm the freeze: a loss was detected (a frame-index gap, or a decoder wedge/demotion). Zeroes the +// recovery-mark count and (re-)sets the backstop deadline. NULL is a no-op. +// +// # Safety +// `g` is a valid gate handle. +void punktfunk_reanchor_gate_arm(ReanchorGate *g); + +// Fold one decoded frame and write to `out_present` whether to display it (`true`) or withhold it as +// a post-loss concealment (`false`). `flags` is the AU's `user_flags` word ([`PunktfunkFrame::flags`]): +// the gate reads `FLAG_SOF` (the host's IDR marker), `USER_FLAG_RECOVERY_ANCHOR` and +// `USER_FLAG_RECOVERY_POINT`. Pass `decoder_keyframe = false` where the platform decoder doesn't flag +// IDRs (VideoToolbox/MediaCodec) — the wire `FLAG_SOF` covers it. +// +// # Safety +// `g` is a valid gate handle; `out_present` is writable or NULL. +PunktfunkStatus punktfunk_reanchor_gate_on_decoded(ReanchorGate *g, + uint32_t flags, + bool decoder_keyframe, + bool *out_present); + +// A received AU produced no decoded frame. Writes to `out_request_kf` whether the no-output streak has +// tripped and the client should (throttled) request a keyframe — the gate arms the freeze at the same +// time. +// +// # Safety +// `g` is a valid gate handle; `out_request_kf` is writable or NULL. +PunktfunkStatus punktfunk_reanchor_gate_on_no_output(ReanchorGate *g, + bool *out_request_kf); + +// Periodic fold of the session's `frames_dropped` counter plus the overdue backstop. Writes to +// `out_request_kf` whether the client should (throttled) request a keyframe (a drop-count climb armed +// a fresh freeze, or the freeze is overdue and re-asks while it keeps holding). +// +// # Safety +// `g` is a valid gate handle; `out_request_kf` is writable or NULL. +PunktfunkStatus punktfunk_reanchor_gate_poll(ReanchorGate *g, + uint64_t frames_dropped, + bool *out_request_kf); + +// Whether the gate is currently withholding concealed frames (frozen on the last good picture). +// Writes `false` on a NULL gate. +// +// # Safety +// `g` is a valid gate handle; `out_holding` is writable or NULL. +PunktfunkStatus punktfunk_reanchor_gate_is_holding(const ReanchorGate *g, bool *out_holding); + #ifdef __cplusplus } // extern "C" #endif // __cplusplus