diff --git a/clients/session/README.md b/clients/session/README.md index e9104632..8d9534f6 100644 --- a/clients/session/README.md +++ b/clients/session/README.md @@ -67,3 +67,22 @@ mailbox|fifo|immediate|fifo_relaxed` (default MAILBOX, FIFO where the surface of MAILBOX — AMD on Windows), `PUNKTFUNK_VK_DEVICE=` (multi-GPU), and `PUNKTFUNK_HW_FAULT=import` (fault every VAAPI dmabuf import — proves the three-strike demotion to software on healthy hardware). + +`PUNKTFUNK_AU_FAULT=drop|truncate|flip[:period]` deliberately corrupts decoder input on the +native Vulkan lane (default period 60 — one AU a second at 60 fps; inert everywhere else, and +inert entirely if the value doesn't parse). `drop` swallows the AU, so the next one references a +picture that was never decoded — the bitstream planner catches it immediately. `truncate` delivers +a picture whose slice data stops mid-frame and `flip` alters one byte deep in the payload: both +parse perfectly, so only the driver's per-frame decode-status query can see them, and neither is +visible at all on a driver without `queryResultStatusSupport` or on any FFmpeg lane. Watch the +result on the Detailed stats line's `integrity:` term (`damaged` = concealment the planner caught, +`refused` = AUs the decoder rejected outright, `driver-failed` = the hardware's own verdict, `run` += consecutive frames with no picture, `worst run` = the longest such stretch of the session — the +once-a-second `run` sample misses the bad moment almost every time — and `no driver status` = this +device cannot answer the driver question at all). A session that lands on any other lane says so +in the log rather than faulting silently. + +Note that `PUNKTFUNK_AU_DUMP` records the AU as it arrived from the HOST, while the fault injector +runs later, at the native decoder's own entry. On a faulted run the dump is therefore the clean +bitstream — reconstruct the damaged bytes from the spec if you need them (the injector is pure and +deterministic). diff --git a/crates/pf-client-core/src/session.rs b/crates/pf-client-core/src/session.rs index 9723bcfa..e8be2f8a 100644 --- a/crates/pf-client-core/src/session.rs +++ b/crates/pf-client-core/src/session.rs @@ -181,6 +181,46 @@ pub struct Stats { /// `chroma_444` false, the host declined — the OSD says so instead of leaving the /// switch's effect unobservable. pub asked_444: bool, + /// The decode lane can answer integrity questions AT ALL (M4). True only on the + /// native rung; every FFmpeg rung leaves it false, because libavcodec's Vulkan + /// decoder creates no status queries (`nb_queries = 0`), never sets + /// `AV_FRAME_FLAG_CORRUPT`, and reports trouble only as log lines. + /// + /// Everything below is meaningless without it, and a surface that renders the + /// four counters as zeros on a lane that cannot see damage is repeating the + /// exact mistake this program exists to end: "clean" and "unmeasured" are not + /// the same claim. + pub decode_integrity: bool, + /// AUs whose plan needed CONCEALMENT this window — a lost reference, a + /// `frame_num` gap, a short NALU walk. Each one cost a frame (released unshown) + /// and a re-anchor request. + pub decode_damaged: u32, + /// Frames the DRIVER reported corrupt this window through their per-op + /// `RESULT_STATUS` query — the Xbox Ally X class, and the count no FFmpeg rung + /// can produce. Always 0 where `decode_status_queries` is false: there is no + /// verdict to read, not nothing to report. (`video::DecodeHealth::note` + /// enforces that, so the two fields can never contradict each other here.) + pub decode_failed: u32, + /// AUs the decoder REFUSED outright this window — a plan error, a + /// Vulkan/session failure. Distinct from `decode_damaged`, and the difference + /// is the whole diagnosis: concealment means the decoder coped with a damaged + /// stream, refusal means it could not run and the screen is frozen. A rung + /// refusing every AU used to report as a perfectly clean session. + pub decode_refused: u32, + /// Consecutive AUs with no showable picture as of this window's end (0 = the + /// stream is decoding clean right now). The field that separates a lossy link + /// from a stream that never came back — see `video::DecodeHealth::run`. + pub concealed_run: u32, + /// The LONGEST such run of the session so far — session-cumulative, not + /// windowed, and deliberately so: `concealed_run` is an instant sampled once a + /// second, which misses the bad moment almost every time. A window whose + /// `concealed_run` is 0 and whose `worst_concealed_run` is 40 is a session that + /// froze hard and recovered, and no other field on this struct says that. + pub worst_concealed_run: u32, + /// The device answers per-op decode-status queries (`queryResultStatusSupport`). + /// FALSE on RADV, where recording one HANGS the VCN ring, and there the integrity + /// report covers the parser's half only. + pub decode_status_queries: bool, } /// Frames the pump keeps waiting for their 0xCF host timing (pts → capture→received µs). @@ -650,7 +690,30 @@ fn pump( let mut next_expected_index: Option = None; // Fixture capture for the native-decode program: every AU exactly as it reaches // `decode_frame`, plus a boundary/flags index — see `au_dump.rs` for the format. + // + // NOTE for fault runs: this captures what the HOST sent. `PUNKTFUNK_AU_FAULT`'s + // injector lives one level down, at the native backend's decode entry, so on a + // faulted run the fixture is the CLEAN bitstream and replaying it will not + // reproduce the damage (reconstruct that from the spec — the injector is pure + // and deterministic). Deliberate: the dump's job is to preserve the host's + // output, and moving the injector above it would corrupt every backend's input + // rather than only the lane whose detectors it exists to fire. let mut au_dump = crate::au_dump::AuDump::from_env(connector.codec); + // The decode-order watermark at the latest arm of the freeze gate (M4 review): + // a frame whose `decode_order` is at or below this was DECODED before the loss, + // whatever order it was delivered in, so its recovery point SEI describes a wave + // that completed before the loss and must not lift the freeze the loss raised. + // `gate.arms()` is the trigger to re-stamp — it moves at every arm site, + // including the two inside the gate, and not on the overdue backstop (which + // re-asks without re-arming, and where discarding an in-flight heal would be + // exactly wrong). Inert on every lane without its own parser: `decode_order` is + // 0 there and `local_recovery` is NONE anyway. + let mut gate_arms = gate.arms(); + let mut arm_decode_order: u64 = 0; + // Decode-integrity window cursor (M4), the same per-window diffing as + // `window_dropped`: the decoder's counters are session-cumulative, the OSD shows + // the delta. `None` on every lane that cannot answer — see `Stats::decode_integrity`. + let mut window_health = decoder.decode_health(); let end: Option = loop { if stop.load(Ordering::SeqCst) { @@ -769,11 +832,56 @@ fn pump( au_dump = None; } } + // Re-stamp the arm watermark BEFORE this AU decodes and advances the + // decoder's ordinal, so it names the newest picture that existed when the + // freeze was armed. One site covers every arm: the frame-gap arm above + // happened moments ago in this same iteration, and the four sites below + // (`on_no_output` ×2, the decoder-recovery arm, `poll`'s dropped climb) all + // run AFTER the decode, so the next iteration reaches here with the ordinal + // still exactly as they left it. + if gate.arms() != gate_arms { + gate_arms = gate.arms(); + arm_decode_order = decoder.decode_order(); + } match decoder.decode_frame(&frame.data, frame.flags, frame.complete) { Ok(Some(image)) => { - // Fold this decoded frame through the shared freeze gate: it reads the AU's - // re-anchor wire flags (FLAG_SOF IDR marker / RECOVERY_ANCHOR / RECOVERY_POINT), - // takes `image.is_keyframe()` as the ffmpeg keyframe belt, applies the two-mark + // The decoder's OWN re-anchor observation FIRST (M4): a recovery point SEI + // is the only clean point an intra-refresh session has when the host does not + // mark the wire — its wave emits no IDR and libavcodec flags none, and only + // one of the three encoder backends that run a wave sets + // USER_FLAG_RECOVERY_POINT — so without this such a session freezes for the + // full REANCHOR_FREEZE_MAX and then forces the very IDR the wave exists to + // avoid. The gate pairs the mark against its own arm (only a wave that + // STARTED after the loss proves anything about it) and lifts on the first + // trusted one. Before `on_decoded`, so the frame that healed the picture is + // itself presented rather than held one more round. Inert on every other + // lane: `local_recovery` reports NONE and the wire path is untouched. + // + // The gate pairs by TIME; this pairs by DECODE ORDER, and both are + // needed. A decoder that flushes its DPB after a failed AU hands back + // every picture it still held — pictures decoded BEFORE the loss, + // carrying the marks of the wave they were decoded in — and they + // arrive after the arm, so the gate cannot tell. Their ordinal can. + let local = match image.decode_order() { + Some(order) if order <= arm_decode_order => { + tracing::trace!( + order, + arm_decode_order, + "discarding the local recovery of a frame decoded before \ + the loss" + ); + punktfunk_core::reanchor::LocalRecovery::NONE + } + _ => image.local_recovery(), + }; + if gate.on_local_recovery(local) { + tracing::debug!( + "re-anchored on the stream's own recovery point SEI — no IDR needed" + ); + } + // Then the shared freeze gate: it reads the AU's re-anchor wire flags + // (FLAG_SOF IDR marker / RECOVERY_ANCHOR / RECOVERY_POINT), takes + // `image.is_keyframe()` as the ffmpeg keyframe belt, applies the two-mark // rule + the mark-patience backstop, clears the no-output streak, and returns // whether to present this frame or withhold it as a post-loss concealment. let present = @@ -838,6 +946,20 @@ fn pump( // shows becomes that sample — honest, at zero pipeline cost on // every other frame. Software keeps the synchronous stamp on // every frame (its decode really is done by now). + // + // M4 re-examined this against the native rung's non-blocking + // reads (`poll_status`, `get_semaphore_counter_value`) and left + // it exactly as it is. Polling can only ever answer "complete + // by NOW", and the only place this thread polls is once per AU + // — so every sample would be quantized up by as much as a whole + // frame interval (8.3 ms at 120 Hz, against decodes that + // measure ~0.1-2 ms). That is not a cheaper measurement, it is + // a wrong one, and it would replace a true figure with a + // plausible-looking upper bound nothing downstream could tell + // apart. Sampling faster needs either a spin (the CPU burn this + // comment already warns about) or a second thread on a decoder + // that is `Send` but deliberately not `Sync`. Correctness beats + // the metric: one honest sample per window stands. let hw_fence = match &image { DecodedImage::VkFrame(v) => Some((v.timeline_sem, v.decode_done_value)), // The native rung's frame carries the same pair: the @@ -931,9 +1053,31 @@ fn pump( // GOP has no periodic keyframe, so a rebuilt/erroring decoder would stay // gray/frozen until an unrelated packet drop happened to request one. Route it // through the same throttle as loss recovery below. + // + // The native rung's DAMAGE path arrives here too (M4): an AU whose plan needed + // concealment answers `Ok(None)` and raises this flag rather than erroring, so + // the ask happens at exactly this moment and through exactly this throttle + // while the decoder keeps its rung — stream damage is not a decoder fault (see + // `video_vk_native`'s recovery policy). That also bounds the whole thing: one + // ask per 100 ms per session however fast the damage arrives, and once the gate + // is armed further damage refreshes an existing freeze rather than compounding + // into more requests. + // + // ARM ONLY WHEN NOT ALREADY HOLDING. This flag fires per DAMAGED AU, not per + // loss, and every `arm` zeroes the gate's recovery-mark count and its + // local-SEI credit. Re-arming on each one therefore made both re-anchor paths + // — the wire's two-mark rule and M4's local SEI — impossible to complete + // during exactly the sustained damage they were written for, leaving recovery + // resting entirely on the throttled keyframe ask. A genuinely NEW loss still + // re-arms with its marks zeroed: it arrives as a frame-index gap or a + // `frames_dropped` climb, both of which arm unconditionally. The keyframe ask + // below is untouched — it still fires per damaged AU, through the same 100 ms + // throttle. if decoder.take_keyframe_request() { let now = Instant::now(); - gate.arm(now); + if !gate.is_holding() { + gate.arm(now); + } if last_kf_req .is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) { @@ -1064,6 +1208,28 @@ fn pump( .saturating_sub(window_mic.dropped_full + window_mic.dropped_stale) as u32; window_mic = mic_now; + // Decode integrity (M4): session-cumulative counters, diffed per window + // like `frames_dropped`. `None` on a lane that cannot see damage at all — + // and that stays distinguishable from "saw none" all the way to the OSD. + let health_now = decoder.decode_health(); + let (decode_damaged, decode_failed, decode_refused) = match (health_now, window_health) + { + (Some(now), Some(prev)) => ( + now.damaged.saturating_sub(prev.damaged) as u32, + now.failed.saturating_sub(prev.failed) as u32, + now.refused.saturating_sub(prev.refused) as u32, + ), + // A lane that could not answer at the last window and can now. + // Unreachable today — the cursor is seeded from the decoder before + // the first AU and the ladder only ever demotes AWAY from the + // native rung, never back onto it — so this exists to keep the + // match total with a defensible answer (the cumulative figure) + // instead of an `unwrap` that would be a panic if that ever + // changed. + (Some(now), None) => (now.damaged as u32, now.failed as u32, now.refused as u32), + (None, _) => (0, 0, 0), + }; + window_health = health_now; tracing::debug!( fps = frames_n, hostnet_p50_us = hn_p50, @@ -1077,6 +1243,12 @@ fn pump( lost, mic_sent, mic_dropped, + decode_damaged, + decode_failed, + decode_refused, + concealed_run = health_now.map(|h| h.run).unwrap_or(0), + worst_concealed_run = health_now.map(|h| h.worst_run).unwrap_or(0), + decode_status_queries = health_now.map(|h| h.status_queries).unwrap_or(false), total_frames, "stream window" ); @@ -1106,6 +1278,13 @@ fn pump( auto_rate, chroma_444, asked_444, + decode_integrity: health_now.is_some(), + decode_damaged, + decode_failed, + decode_refused, + concealed_run: health_now.map(|h| h.run).unwrap_or(0), + worst_concealed_run: health_now.map(|h| h.worst_run).unwrap_or(0), + decode_status_queries: health_now.is_some_and(|h| h.status_queries), })); window_start = Instant::now(); frames_n = 0; diff --git a/crates/pf-client-core/src/video.rs b/crates/pf-client-core/src/video.rs index efe65d61..82141788 100644 --- a/crates/pf-client-core/src/video.rs +++ b/crates/pf-client-core/src/video.rs @@ -102,6 +102,112 @@ pub enum DecodedImage { NativeVk(NativeVkFrame), } +/// What the decode lane knows about this session's INTEGRITY — M4's telemetry +/// surface, and the answer to the question that started the whole native-decode +/// program: "was that stream actually clean, or could nothing here have told us?" +/// +/// Only the native rung fills it in ([`Decoder::decode_health`] answers `None` +/// everywhere else), because only the native rung has the two detectors: a +/// bitstream planner that reports lost references, and a per-op `RESULT_STATUS` +/// query that reports what the DRIVER thought of the decode. FFmpeg's Vulkan +/// decoder creates no queries at all (`nb_queries = 0`), never sets +/// `AV_FRAME_FLAG_CORRUPT`, and reports trouble only as log lines — which is why +/// the Xbox Ally X corruption was undetectable rather than merely undetected. +/// +/// Counters are session-cumulative and monotonic; the stats window diffs them the +/// way it already diffs `frames_dropped`. Nothing here allocates, and nothing here +/// is computed per frame beyond an add — the whole struct is read once per stats +/// window. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct DecodeHealth { + /// AUs whose plan needed CONCEALMENT: a reference the DPB no longer held, a + /// `frame_num` gap, a NALU walk that stopped early. The picture would have + /// been decoded from a substitute, so its output was released unshown. + pub damaged: u64, + /// Frames the DRIVER reported corrupt through their `RESULT_STATUS_ONLY` + /// query. Distinct from [`Self::damaged`] on purpose: damaged means the + /// bitstream arrived incomplete, failed means the hardware could not decode + /// what did arrive. They have different causes and different fixes, and + /// collapsing them is how "the stream is fine, it's your GPU" arguments start. + /// + /// **Structurally 0 where [`Self::status_queries`] is false**, and + /// [`Self::note`] enforces that rather than trusting its callers: on such a + /// device `poll_status` still answers `Failed` for a lost device or an + /// unreadable timeline, and reporting THAT as a driver verdict would point a + /// support engineer at a verdict the hardware cannot produce ("driver-failed 1 + /// · no driver status" on one line). Those frames still cost a picture, so + /// they still extend [`Self::run`] — they are just not attributed to a driver + /// that never spoke. + pub failed: u64, + /// AUs the decoder REFUSED outright: a plan error (a parse failure, an AU + /// outside the punktfunk envelope, a slice against a parameter set never + /// seen), or a Vulkan/session failure. The decoder produced no picture and + /// said so with an error. + /// + /// Counted apart from [`Self::damaged`] because the two mean opposite things + /// about the RUNG: concealment says the decoder coped with a damaged stream, + /// refusal says the decoder could not run at all. A rung refusing every AU is + /// the shape of a host renegotiating outside the envelope — a frozen screen — + /// and without this counter its stats surface reads exactly like a clean + /// session, which is the founding failure mode of this whole program. + pub refused: u64, + /// Consecutive AUs that produced no showable picture, ending at the latest one + /// — 0 the moment a clean AU decodes. + /// + /// This is the field a support engineer reads first, because it separates the + /// two failure shapes a raw count cannot: `damaged 40 · run 0` is a lossy link + /// that keeps recovering, `damaged 40 · run 40` is a stream that went down and + /// never came back. Both look identical as a total. + pub run: u32, + /// The longest [`Self::run`] of the session — the worst moment, which a + /// once-per-second sample of `run` will usually miss entirely. + pub worst_run: u32, + /// This device answers per-op decode-status queries + /// (`queryResultStatusSupport`). When FALSE — RADV, where recording a query + /// anyway HANGS the VCN ring — [`Self::failed`] can only ever read 0, because + /// there is no verdict to read: the status degrades to timeline completion, + /// exactly what FFmpeg knows on every driver. A report that omits this cannot + /// tell "clean" from "unmeasured", which is the precise shape of the failure + /// this program exists to end. + pub status_queries: bool, +} + +impl DecodeHealth { + /// Fold one AU's verdict. `damaged` = its plan needed concealment; `refused` = + /// the decoder rejected the AU outright (an `Err` out of `decode`); `failed` = + /// how many PRIOR frames just read a `Failed` decode status. + /// + /// All three extend the run: a support engineer asking "did it ever recover?" + /// means the picture, and a refused AU or a driver-failed frame is as absent + /// from the screen as a concealed one. + /// + /// The one asymmetry is deliberate and is the whole point of + /// [`Self::status_queries`]: where the device answers no status queries, a + /// `Failed` read is NOT a driver verdict — it is the degraded timeline path + /// (the session generation is gone, the device is lost, the semaphore could + /// not be read) — so it extends the run without ever being counted as + /// [`Self::failed`]. Enforced here, at the one place every counter is written, + /// rather than at each call site, because "clean" and "unmeasured" staying + /// distinguishable is the invariant this struct exists for. + pub(crate) fn note(&mut self, damaged: bool, refused: bool, failed: u32) { + if self.status_queries { + self.failed = self.failed.saturating_add(u64::from(failed)); + } + if damaged { + self.damaged = self.damaged.saturating_add(1); + } + if refused { + self.refused = self.refused.saturating_add(1); + } + if damaged || refused || failed > 0 { + self.run = self.run.saturating_add(1); + self.worst_run = self.worst_run.max(self.run); + } else { + self.run = 0; + } + } +} + /// A raw `VkFormat` code point, carried across the ash-free boundary. /// /// A newtype rather than a bare `i32` because the two hardware frame types @@ -319,6 +425,28 @@ pub struct NativeVkFrame { /// leading pictures may be undecodable. pub keyframe: bool, pub poc: i32, + /// What this frame's AU said about intra-refresh RECOVERY, read out of the + /// bitstream's own recovery point SEI (pf-vkdecode's `RecoveryWatch`). + /// + /// [`Self::keyframe`] cannot answer for an intra-refresh session — the wave + /// never emits an IDR — so without this the pump has no clean point to lift a + /// post-loss freeze on and holds the last good picture until its 500 ms + /// backstop forces the very IDR the wave exists to avoid. The wire's + /// `USER_FLAG_RECOVERY_POINT` says the same thing when the host sets it, which + /// only one of the three wave-running encoder backends does (Linux + /// libav-NVENC); this is the same fact taken from the stream instead of from + /// the host, and it cannot be lost separately from the picture. Fed to + /// [`ReanchorGate::on_local_recovery`](punktfunk_core::reanchor::ReanchorGate::on_local_recovery). + pub recovery: punktfunk_core::reanchor::LocalRecovery, + /// This picture's position in DECODE order (pf-vkdecode's strictly increasing + /// per-session ordinal). Delivery order is not decode order: after a failed AU + /// the H.265 decoder flushes its DPB, handing back every buffered picture at + /// once — pictures decoded BEFORE the loss, carrying the recovery marks of the + /// wave they were decoded in. Arriving after the pump armed its freeze, those + /// marks would lift it on a heal that completed before the loss. The pump + /// stamps this ordinal at every arm and ignores [`Self::recovery`] from + /// anything older. + pub decode_order: u64, /// Sends the release token on drop — see [`NativeReleaseGuard`]. pub guard: NativeReleaseGuard, } @@ -363,6 +491,32 @@ impl DecodedImage { } } + /// What the decoder's OWN bitstream parser saw about an intra-refresh heal on + /// this frame's AU — the recovery point SEI, which no platform decoder exposes. + /// + /// Only the native rung can answer: libavcodec parses the SEI internally and + /// surfaces nothing of it (its `AV_FRAME_FLAG_KEY` is IDR-only), MediaCodec and + /// VideoToolbox likewise. Everyone else reports + /// [`LocalRecovery::NONE`](punktfunk_core::reanchor::LocalRecovery::NONE) and + /// the pump's re-anchor behaviour on those lanes is byte-for-byte what it was. + pub fn local_recovery(&self) -> punktfunk_core::reanchor::LocalRecovery { + match self { + DecodedImage::NativeVk(f) => f.recovery, + _ => punktfunk_core::reanchor::LocalRecovery::NONE, + } + } + + /// This frame's position in DECODE order, where the lane knows one — see + /// [`NativeVkFrame::decode_order`]. `None` everywhere else, which is what the + /// pump reads as "this lane reports no local recovery either, so there is + /// nothing to date-stamp". + pub fn decode_order(&self) -> Option { + match self { + DecodedImage::NativeVk(f) => Some(f.decode_order), + _ => None, + } + } + /// The decoded image's pixel dimensions. The presenter's resize indicator uses these /// as the mid-stream-resize END signal: a frame arriving at the target size means the /// new-mode picture is on glass (the ack alone lands before the host's rebuild does). @@ -554,6 +708,34 @@ const VAAPI_DEMOTE_AFTER: u32 = 3; /// software before the first requested IDR could even arrive. const HW_DEMOTE_MIN_STREAK: std::time::Duration = std::time::Duration::from_millis(1000); +/// May a successful `decode` answer CLEAR the demotion error streak? +/// +/// The streak is the hardware rungs' only escape hatch, and clearing it is a +/// claim: *this decoder is working*. A delivered frame proves that outright. So +/// does a clean `Ok(None)` — the decoder ran and had nothing to object to (it +/// buffered, or skipped an H.265 RASL picture after an open-GOP join). +/// +/// What proves nothing is the third `Ok(None)`: the native rung's CONCEALMENT +/// answer, where the plan needed a substitute for something lost and the picture +/// was released unshown. That is deliberately not an `Err` — stream damage is not +/// a decoder fault, and three of them in a second must not demote the rung on +/// exactly the lossy links it exists to diagnose — but "not an error" was silently +/// read as "a success", and clearing on it is the dangerous half of that: +/// +/// * a driver failing every OTHER AU on a lossy link has its `Err`s zeroed by the +/// concealment between them and never reaches [`VAAPI_DEMOTE_AFTER`]; +/// * and a rung answering concealment FOREVER — a host framing regression putting +/// two pictures in one AU makes every AU conceal, and unlike a reference gap it +/// does not self-heal at an IDR — holds a frozen last-good frame with no escape +/// at all, where before this milestone the same stream demoted to a rung that +/// ignores AU boundaries and showed a picture. +/// +/// Leaving the streak untouched costs nothing on a healthy link: one damaged AU +/// between good frames is cleared by the next good frame. +fn clears_demotion_streak(delivered: bool, concealed: bool) -> bool { + delivered || !concealed +} + /// `VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR` — the raw flag bit within /// [`VulkanDecodeDevice::decode_video_caps`] (this crate stays ash-free). const VIDEO_CODEC_OP_DECODE_H264: u32 = 0x0000_0001; @@ -763,6 +945,39 @@ fn quiet_ffmpeg_log() { ffmpeg::util::log::set_level(level); } +/// Say what `PUNKTFUNK_AU_FAULT` will do to THIS session, once, at decoder +/// construction — including the two cases where the answer is "nothing". +/// +/// The knob only bites on the native rung (its injector sits at that backend's +/// decode entry), so a lab run that armed it and landed anywhere else — an FFmpeg +/// rung, a shape the native rung refused, a session that demoted — must be told +/// so. Silence there is indistinguishable from "the fault was injected and +/// nothing detected it", which is precisely the conclusion a fault run exists to +/// make trustworthy. Unset is the normal state and says nothing at all. +fn report_au_fault_env(native_rung: bool) { + let Ok(spec) = std::env::var("PUNKTFUNK_AU_FAULT") else { + return; + }; + if spec.is_empty() { + return; + } + match pf_vkdecode::AuFault::from_spec(&spec) { + // The native backend logs the arming itself (mode + period), with the + // decoder it is about to corrupt in hand — no need to say it twice. + Some(_) if native_rung => {} + Some(_) => tracing::warn!( + value = %spec, + "PUNKTFUNK_AU_FAULT is armed, but this session is NOT on the native \ + Vulkan rung — no AU will be corrupted and no detector will fire" + ), + None => tracing::warn!( + value = %spec, + "PUNKTFUNK_AU_FAULT not understood (want drop|truncate|flip[:period]) \ + — ignored" + ), + } +} + impl Decoder { /// `codec_id` is the codec the host resolved in the Welcome (never assume HEVC). /// `pref` is the Settings "Video decoder" value (`auto`/`vulkan`/`vaapi`/`d3d11va`/ @@ -807,7 +1022,15 @@ impl Decoder { vk.and_then(|v| v.adapter_luid), vk.is_some_and(|v| v.d3d11_hdr10), ); - let done = |backend| { + let done = |backend: Backend| { + // Whatever rung this session landed on, say what `PUNKTFUNK_AU_FAULT` + // is going to do about it — see [`report_au_fault_env`]. Here, at the + // one exit every backend leaves through, rather than in the native + // backend's constructor: a lab run whose session never REACHES that + // constructor (an FFmpeg rung, a refused shape, a demotion) would + // otherwise sit silently un-faulted and read as a fault run that + // detected nothing. + report_au_fault_env(matches!(backend, Backend::NativeVulkan(_))); Ok(Decoder { backend, codec_id, @@ -1086,6 +1309,33 @@ impl Decoder { } } + /// This session's decode-integrity counters, or `None` on a backend that has + /// no way to answer (every FFmpeg rung and PyroWave — see [`DecodeHealth`]). + /// + /// `None` and `Some(DecodeHealth::default())` are deliberately different + /// answers, and the stats surface must keep them different: the first is "this + /// decoder cannot see corruption", the second is "this decoder looked and saw + /// none". Reporting the first as the second is exactly the mistake that let a + /// field corruption run undetected for a release. + pub fn decode_health(&self) -> Option { + match &self.backend { + Backend::NativeVulkan(d) => Some(d.health()), + _ => None, + } + } + + /// The DECODE-order ordinal of the newest picture this lane has planned — the + /// watermark a caller stamps when it arms a post-loss freeze, so it can tell a + /// frame decoded before the loss from one decoded after it (see + /// [`NativeVkFrame::decode_order`]). 0 on every lane that has no bitstream + /// parser of its own, which is also every lane that reports no local recovery. + pub fn decode_order(&self) -> u64 { + match &self.backend { + Backend::NativeVulkan(d) => d.decode_order(), + _ => 0, + } + } + /// Drain the "please ask the host for an IDR" flag — the pump calls this each iteration /// (throttled) so a demoted/erroring decoder can resynchronize under the infinite GOP. /// Open a PyroWave decoder for a `CODEC_PYROWAVE` session (plan §4.5): pyrowave @@ -1101,6 +1351,8 @@ impl Decoder { color: ColorDesc, hdr16: bool, ) -> Result { + // Never the native rung — see [`report_au_fault_env`]. + report_au_fault_env(false); Ok(Decoder { backend: Backend::PyroWave(Box::new(crate::video_pyrowave::PyroWaveDecoder::new( vk, @@ -1177,6 +1429,11 @@ impl Decoder { user_flags: u32, complete: bool, ) -> Result> { + // Did THIS AU come back as a concealment — an `Ok(None)` the native rung + // produced because the picture was damaged, not because the decoder was + // buffering? Only the native rung can answer, and the answer decides + // whether the `Ok` below is allowed to clear the demotion streak. + let mut concealed = false; let result = match &mut self.backend { Backend::Vulkan(v) => { debug_assert!(complete, "partial AUs are pyrowave-only"); @@ -1184,7 +1441,31 @@ impl Decoder { } Backend::NativeVulkan(n) => { debug_assert!(complete, "partial AUs are pyrowave-only"); - n.decode(au).map(|f| f.map(DecodedImage::NativeVk)) + let r = n.decode(au).map(|f| f.map(DecodedImage::NativeVk)); + // STREAM damage is not a decoder fault, and must not ride the + // demotion streak. + // + // This distinction only exists on the native rung, because it is + // the only one that can SEE damage — and that is precisely what + // makes it dangerous. An FFmpeg rung conceals a lost reference + // silently and keeps its job; if the native rung turned the same + // event into an error, three of them over a second would demote + // the program's own headline decoder exactly on the lossy links it + // was built to diagnose. So concealment comes back as `Ok(None)` + // plus this flag: the pump still asks for a re-anchor at the same + // moment and through the same throttle it always did, and the + // hardware rung survives the loss that caused it. + // + // A driver `RESULT_STATUS` verdict of Failed is NOT routed here — + // it stays an `Err` below. That one really is a statement about + // the decoder ("I could not decode what I was given"), and a + // driver making it repeatedly is the exact case demotion exists + // for; it is also the Xbox Ally X shape. + if n.take_recovery_request() { + self.want_keyframe = true; + concealed = true; + } + r } #[cfg(target_os = "linux")] Backend::Vaapi(v) => v.decode(au).map(|f| f.map(DecodedImage::Dmabuf)), @@ -1204,8 +1485,12 @@ impl Decoder { }; match result { Ok(f) => { - self.vaapi_fails = 0; - self.first_fail = None; + // Only an answer that PROVES the rung works may clear the streak — + // see [`clears_demotion_streak`] for the whole argument. + if clears_demotion_streak(f.is_some(), concealed) { + self.vaapi_fails = 0; + self.first_fail = None; + } self.delivered |= f.is_some(); Ok(f) } @@ -1567,6 +1852,52 @@ mod tests { } } + /// The demotion streak's escape hatch, stated as the invariant it is: an `Ok` + /// clears the streak only when it PROVES the rung works. + /// + /// Concealment (`Ok(None)` with a recovery request) proves nothing — it is the + /// STREAM that was damaged — and before M4's review it cleared the streak + /// anyway, because the `Ok(_)` arm matched `Ok(None)` too. Two shapes followed + /// from that, and this test pins both away: + /// + /// * a driver failing every other AU on a lossy link: `Err` / concealment / + /// `Err` / concealment … the concealment zeroed the count and + /// [`VAAPI_DEMOTE_AFTER`] was never reached; + /// * and a rung that conceals forever and ships nothing: a frozen picture with + /// no path down the ladder at all. + #[test] + fn only_an_answer_that_proves_the_rung_works_clears_the_demotion_streak() { + // A shipped frame is proof, concealed or not (the AU carried damage AND a + // picture — the decoder is plainly alive). + assert!(clears_demotion_streak(true, false)); + assert!(clears_demotion_streak(true, true)); + // A CLEAN no-output AU is proof too: the decoder ran and objected to + // nothing (it buffered, or skipped an H.265 RASL picture after an open-GOP + // join). Treating that as suspicious would demote healthy sessions. + assert!(clears_demotion_streak(false, false)); + // Concealment with no picture is the one that proves nothing. + assert!(!clears_demotion_streak(false, true)); + + // The streak arithmetic that follows, spelled out on the milder and + // likelier shape: a broken driver alternating with concealment must still + // reach the demotion threshold. + let mut fails = 0u32; + for concealed_ok in [false, true, false, true, false] { + if concealed_ok { + if clears_demotion_streak(false, true) { + fails = 0; + } + } else { + fails += 1; // an Err from the driver's own verdict + } + } + assert!( + fails >= VAAPI_DEMOTE_AFTER, + "three driver errors interleaved with concealment must still reach the \ + demotion threshold — they got to {fails}" + ); + } + /// Auto's hardware order (both OSes): Vulkan-first on NVIDIA (on Linux: no usable /// VAAPI) and ALL AMD (Vulkan decode outperforms VAAPI on RADV — on-glass verdict; /// VanGogh additionally chroma-fringes over VAAPI); Intel/unknown take the proven diff --git a/crates/pf-client-core/src/video_vk_native.rs b/crates/pf-client-core/src/video_vk_native.rs index 6efbb753..1be7d591 100644 --- a/crates/pf-client-core/src/video_vk_native.rs +++ b/crates/pf-client-core/src/video_vk_native.rs @@ -62,6 +62,54 @@ //! field case. It surfaces as an `Err` from the CURRENT `decode_frame` call so the //! existing streak/reanchor machinery fires exactly as it does for FFmpeg errors. //! +//! **The recovery policy** (M4) — what a damaged stream ASKS for, and why it cannot +//! storm. There are two kinds of damage and they are answered differently: +//! +//! - **Concealment** (the plan needed a substitute for something lost: an integrity +//! warning). The AU's output is released UNSHOWN, [`DecodeHealth`] records it, and +//! `decode` answers `Ok(None)` with [`NativeVulkanDecoder::take_recovery_request`] +//! raised. `video::Decoder` turns that into its ordinary `want_keyframe`, which the +//! pump drains, arms the freeze on, and asks through the ONE ~100 ms recovery +//! throttle every other ask already shares (`session.rs`'s `last_kf_req`: frame-gap +//! RFI, dropped-climb, no-output streak, overdue backstop, decoder recovery). It is +//! deliberately NOT an `Err`: an error ticks the demotion streak, and three of them +//! in a second would demote the native rung on exactly the lossy links it exists to +//! diagnose — an FFmpeg rung conceals the same event silently and keeps its job. +//! - **A driver `Failed` verdict** (and its query-less twin, a decode status that +//! could not be established at all — [`StatusVerdicts`]). That is a statement about +//! the DECODER, not the stream, so it stays an `Err`: same volume as an FFmpeg +//! reference-miss error, streak-eligible, and a driver making it repeatedly is +//! precisely what demotion is for. +//! - **A REFUSED AU** — the decoder answering `Err` outright (a plan error, a +//! Vulkan/session failure). Also an error, also streak-eligible, and counted +//! separately from concealment in [`DecodeHealth::refused`]: "the stream is +//! damaged and I coped" and "I could not run" are opposite statements about the +//! rung, and only the second one means the session is looking at a frozen screen. +//! +//! Because concealment is not an error, it must not clear the demotion streak +//! either — `video::Decoder::decode_frame` leaves the streak untouched on a +//! concealed `Ok(None)` and resets it only on a shipped frame or a clean AU. +//! Otherwise a driver failing every other AU on a lossy link has its errors zeroed +//! by the concealment between them, and a rung that conceals FOREVER (a host +//! framing regression: every AU damaged, no frame ever shipped) has no escape +//! hatch at all. +//! +//! Neither can storm, for two independent reasons. The ask is throttled to one per +//! 100 ms per session whatever the damage rate; and once the freeze is armed the gate +//! lifts only on a proven re-anchor, so a run of damaged AUs refreshes an existing +//! freeze rather than compounding into more requests. A stream that never recovers +//! therefore costs one keyframe ask per 100 ms, not one per AU. +//! +//! **Recovery-point SEI** (M4): pf-vkdecode's `RecoveryWatch` folds the parsed SEI +//! into a per-picture mark that rides the frame ([`NativeVkFrame::recovery`]) into +//! the shared gate's `on_local_recovery`. It is the only way a client can see an +//! intra-refresh session heal on the two backends that run a wave WITHOUT setting the +//! wire mark (Windows AMF and QSV — only Linux libav-NVENC sets it): the wave emits +//! no IDR and libavcodec flags none, so without this such a session freezes for the +//! full 500 ms backstop and then forces the very IDR the wave exists to avoid. +//! Additional, never a replacement: the wire path is untouched and the FFmpeg rungs +//! keep exactly the behaviour they had. +//! //! **Teardown:** dropping this backend (demotion, session end) waits — bounded — for //! every shipped frame's token before dropping the decoder, because the decoder's Drop //! destroys the pool images and its own drain only covers DECODE work, not the @@ -69,7 +117,7 @@ //! displace the frames; a presenter wedged past [`TEARDOWN_BUDGET`] forfeits (warned). use crate::video::{ - ColorDesc, NativeReleaseGuard, NativeReleaseToken, NativeVkFrame, NativeVkLayout, + ColorDesc, DecodeHealth, NativeReleaseGuard, NativeReleaseToken, NativeVkFrame, NativeVkLayout, VulkanDecodeDevice, }; use anyhow::{anyhow, bail, Result}; @@ -243,6 +291,57 @@ impl Codec { Codec::H265(d) => d.wait_decoded(frame, timeout_ns), } } + + /// Does this device answer per-op decode-status queries at all? A device + /// fact, not a codec one — forwarded per arm only because the decoders own + /// the `DecodeDevice`. + fn status_queries(&self) -> bool { + match self { + Codec::H264(d) => d.status_queries(), + Codec::H265(d) => d.status_queries(), + } + } + + /// The newest planned picture's DECODE-order ordinal — the watermark the + /// pump stamps when it arms a freeze (see [`NativeVkFrame::decode_order`]). + fn decode_order(&self) -> u64 { + match self { + Codec::H264(d) => d.decode_order(), + Codec::H265(d) => d.decode_order(), + } + } +} + +/// What one pass of [`NativeVulkanDecoder::settle_statuses`] learned about the +/// decode status of previously shipped frames. +/// +/// Two numbers, not one, because the SAME `DecodeStatus::Failed` means two +/// different things depending on the device. Where the decode family answers +/// `RESULT_STATUS` queries it is the driver's own verdict on its own decode — the +/// Xbox Ally X signal, and the count `DecodeHealth::failed` reports. Where it does +/// NOT (RADV, whose VCN ring hangs if a query is recorded anyway), `poll_status` +/// degrades to reading the decode timeline, and a `Failed` there means the session +/// generation is gone, the device was lost, or the semaphore could not be read — +/// none of which the driver ever said anything about. Reporting those as driver +/// verdicts renders `integrity: driver-failed 1 · no driver status`, which is +/// self-contradictory and points a support engineer at hardware that never spoke. +/// +/// Both cost the picture, so both release their frame unshown, both surface as an +/// error, and both extend the concealed run. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct StatusVerdicts { + /// Frames the DRIVER reported corrupt. Only ever non-zero on a device that + /// answers status queries. + driver_failed: u32, + /// Frames whose status could not be established on a device with no query + /// support — the degraded timeline path. + unreadable: u32, +} + +impl StatusVerdicts { + fn total(&self) -> u32 { + self.driver_failed + self.unreadable + } } /// The plan warnings one AU produced, still in their codec's own enum. @@ -270,24 +369,6 @@ enum PlanWarnings { H265(Vec), } -/// Does this H.264 warning mean the PICTURE is damaged? `Mmco5Rebase` does not: the -/// AU carried an MMCO 5 and pf-bitstream planned it in full (the plan holds the -/// pre-rebase 8.2.1 values, later AUs reference the rebased ones). -fn h264_is_integrity(w: &pf_vkdecode::PlanWarning) -> bool { - use pf_vkdecode::PlanWarning as W; - matches!( - w, - W::FrameNumGap { .. } | W::MissingReference { .. } | W::TruncatedAu { .. } - ) -} - -/// The H.265 twin — the same set pf-bitstream's `h265` conformance harness calls -/// integrity, `NonZeroReorder` deliberately excluded (see [`PlanWarnings`]). -fn h265_is_integrity(w: &pf_vkdecode::H265PlanWarning) -> bool { - use pf_vkdecode::H265PlanWarning as W; - matches!(w, W::MissingReference { .. } | W::TruncatedAu { .. }) -} - impl PlanWarnings { fn is_empty(&self) -> bool { match self { @@ -299,14 +380,26 @@ impl PlanWarnings { /// Just the warnings that mean the picture is damaged — the concealment set. /// Allocates, but only off the clean path: [`Self::is_empty`] is true for every /// AU of a healthy stream. + /// + /// The predicate itself lives in pf-vkdecode + /// ([`pf_vkdecode::is_integrity_warning`]) rather than here, so the + /// fault-injection harness asserts detection against the SAME list this + /// conceals on. Two copies would let a test prove a detection production does + /// not actually perform. fn integrity(&self) -> PlanWarnings { match self { - PlanWarnings::H264(w) => { - PlanWarnings::H264(w.iter().filter(|x| h264_is_integrity(x)).cloned().collect()) - } - PlanWarnings::H265(w) => { - PlanWarnings::H265(w.iter().filter(|x| h265_is_integrity(x)).cloned().collect()) - } + PlanWarnings::H264(w) => PlanWarnings::H264( + w.iter() + .filter(|x| pf_vkdecode::is_integrity_warning(x)) + .cloned() + .collect(), + ), + PlanWarnings::H265(w) => PlanWarnings::H265( + w.iter() + .filter(|x| pf_vkdecode::is_integrity_warning_h265(x)) + .cloned() + .collect(), + ), } } @@ -321,17 +414,25 @@ impl PlanWarnings { /// `warnings=[FrameNumGap { .. }]`, exactly what the hardware-verified H.264 /// path emitted before dispatch existed (a `Vec` renders /// `["FrameNumGap { .. }"]`, and a wrapper enum would prefix the arm). - fn warn_concealment(&self) { + /// + /// `concealed` is how many of the rendered warnings are INTEGRITY warnings — + /// the count the frame was actually dropped for. Both numbers are carried + /// because they can differ: the list is every warning of the AU (a spec-legal + /// companion is context worth having), while the count is the damage. On H.264 + /// the two coincide for every warning a punktfunk host can produce. + fn warn_concealment(&self, concealed: usize) { // Spelled out per arm rather than shared through a `const`: this is the // H.264 path's PRODUCTION log line, and a literal is what keeps it a static // tracing message rather than a formatted one. match self { PlanWarnings::H264(w) => tracing::warn!( + concealed, warnings = ?w, "native decode planned with concealment — dropping the frame, \ requesting re-anchor" ), PlanWarnings::H265(w) => tracing::warn!( + concealed, warnings = ?w, "native decode planned with concealment — dropping the frame, \ requesting re-anchor" @@ -439,6 +540,21 @@ fn project_frame(frame: &DecodedVkFrame, guard: NativeReleaseGuard) -> NativeVkF }, keyframe: frame.is_idr, poc: frame.poc, + // The recovery point SEI's verdict for THIS picture, folded by + // pf-vkdecode's `RecoveryWatch` at plan time and translated here into the + // shared gate's vocabulary. The two structs are deliberately separate + // types with the same shape: pf-vkdecode must not depend on punktfunk-core + // to describe a bitstream fact, and punktfunk-core must not depend on + // pf-vkdecode to accept one. + recovery: punktfunk_core::reanchor::LocalRecovery { + sei_here: frame.recovery.sei_here, + is_recovery_point: frame.recovery.is_recovery_point, + }, + // Which side of a loss this picture was DECODED on. Carried beside the + // recovery mark because the mark is worthless without it: a post-failure + // DPB flush delivers pre-loss pictures after the loss, and their marks + // describe a wave that completed before it. + decode_order: frame.decode_order, guard, } } @@ -482,6 +598,21 @@ pub(crate) struct NativeVulkanDecoder { deliverable: std::collections::VecDeque, outstanding: Vec, next_seq: u64, + /// The session's integrity counters (M4). Plain adds on the decode path, read + /// once per stats window — no allocation, no per-frame work. + health: DecodeHealth, + /// Stream damage happened and the host should be asked for a re-anchor. + /// Drained by `video::Decoder::decode_frame`, which routes it into the same + /// `want_keyframe` every other recovery ask uses (module doc's policy). + want_recovery: bool, + /// Corrupt the AU on its way into the decoder — `PUNKTFUNK_AU_FAULT`, + /// `None` unless armed. Lives at THIS boundary rather than in + /// `video::Decoder::decode_frame` on purpose: this is the lane whose detectors + /// the injector exists to fire, and putting the knob here means a faulted AU + /// is byte-identical to what the decoder would have been handed by a lossy + /// network — no other backend's behaviour can be perturbed by a typo'd + /// variable. + fault: Option, } // SAFETY: the decoder is used strictly serially through `&mut self` from whichever @@ -602,6 +733,43 @@ impl NativeVulkanDecoder { } }; let (release_tx, release_rx) = mpsc::channel(); + let status_queries = dec.status_queries(); + if !status_queries { + // Said once, loudly, at construction rather than only in the stats + // line: on this device a clean integrity report means "nothing was + // detectable", not "nothing was wrong" — and a support engineer + // reading a log after the fact has no stats window to consult. + tracing::warn!( + "native decode: this device's decode queue family does not support \ + RESULT_STATUS queries — driver-reported corruption is not \ + observable on this session (decode status degrades to timeline \ + completion, FFmpeg parity)" + ); + } + // `PUNKTFUNK_AU_FAULT=[:]` — the deliberate-corruption knob + // (pf_vkdecode::fault). Unset is the only normal state; a spec that does + // not parse leaves the injector disarmed and says so rather than half + // arming. + let fault = std::env::var("PUNKTFUNK_AU_FAULT").ok().and_then(|spec| { + match pf_vkdecode::AuFault::from_spec(&spec) { + Some(f) => { + tracing::warn!( + mode = ?f.mode(), + period = f.period(), + "PUNKTFUNK_AU_FAULT: deliberately corrupting decoder input" + ); + Some(f) + } + None => { + tracing::warn!( + value = %spec, + "PUNKTFUNK_AU_FAULT not understood (want drop|truncate|flip[:period]) \ + — ignored" + ); + None + } + } + }); Ok(NativeVulkanDecoder { dec, release_tx: Some(release_tx), @@ -609,21 +777,57 @@ impl NativeVulkanDecoder { deliverable: std::collections::VecDeque::new(), outstanding: Vec::new(), next_seq: 0, + health: DecodeHealth { + status_queries, + ..DecodeHealth::default() + }, + want_recovery: false, + fault, }) } - /// Feed one complete access unit. `Ok(None)` = no display-ready picture (the pump's - /// no-output/reanchor machinery reads that exactly as it does for FFmpeg). `Err` = - /// decode trouble — a decoder error, a plan that needed concealment, or a - /// driver-reported corrupt PREVIOUS frame — routed through the caller's shared - /// streak/demotion machinery. + /// This session's integrity counters — see [`DecodeHealth`]. + pub(crate) fn health(&self) -> DecodeHealth { + self.health + } + + /// The newest planned picture's DECODE-order ordinal — see + /// [`NativeVkFrame::decode_order`]. + pub(crate) fn decode_order(&self) -> u64 { + self.dec.decode_order() + } + + /// Drain the "the stream was damaged; please ask the host to re-anchor" flag. + /// Deliberately separate from an `Err` return — see the module doc's recovery + /// policy: concealment is a fact about the STREAM and must not tick the + /// decoder-demotion streak. + pub(crate) fn take_recovery_request(&mut self) -> bool { + std::mem::take(&mut self.want_recovery) + } + + /// Feed one complete access unit. /// - /// The one thing `Ok(None)` deliberately does NOT mean is trouble. An H.265 RASL - /// picture skipped after an open-GOP join arrives here as exactly that — the - /// decoder never turns `h265::PlanError::RaslSkipped` into a `VkDecodeError`, and - /// it clears the warning ledger on its way out, so the concealment branch below - /// cannot fire on it either. Nothing is released unshown, no re-anchor is asked - /// for, and the next AU decodes normally (module doc; pf-bitstream `h265`). + /// `Ok(Some)` = a display-ready picture. `Ok(None)` = no picture this AU, which + /// covers three unrelated things and the caller treats all three the same + /// (its no-output/re-anchor machinery, exactly as for FFmpeg): the decoder + /// buffered without output, an H.265 RASL picture was skipped after an open-GOP + /// join, or the AU's plan needed CONCEALMENT and its output was released + /// unshown. `Err` = the DECODER is in trouble — a Vulkan/session error, or a + /// driver `RESULT_STATUS` verdict of Failed on a prior frame — which the + /// caller's streak/demotion machinery is entitled to act on. + /// + /// That split is the M4 recovery policy and it is deliberate (module doc): + /// concealment says the STREAM lost data, not that this decoder is failing, so + /// it raises [`Self::take_recovery_request`] instead of an error. The ask + /// reaches the host at the same moment and through the same 100 ms throttle it + /// always did; what it no longer does is spend a life on the demotion streak + /// and cost the session its hardware rung on a lossy link. + /// + /// A skipped RASL picture is not trouble at all: the decoder never turns + /// `h265::PlanError::RaslSkipped` into a `VkDecodeError` and clears the warning + /// ledger on its way out, so the concealment branch cannot fire on it either. + /// Nothing is released unshown and no re-anchor is asked for (module doc; + /// pf-bitstream `h265`). /// /// Ordering: the CURRENT AU decodes FIRST — the planner's reference state must /// advance even when a PRIOR frame's status turns out Failed, or the recovery @@ -633,7 +837,71 @@ impl NativeVulkanDecoder { pub(crate) fn decode(&mut self, au: &[u8]) -> Result> { self.drain_releases(); - let delivered = self.dec.decode(au).map_err(|e| anyhow!("decode: {e}"))?; + // Fault injection, at the last possible moment before the decoder: a + // faulted AU is byte-for-byte what a lossy network would have delivered, + // so every detector below sees the real thing rather than a special case. + // Inert (and free — no branch cost worth naming, no copy) unless armed. + let faulted; + let au = match self.fault.as_mut().map(|f| f.apply(au)) { + None | Some(pf_vkdecode::FaultAction::Pass) => au, + Some(pf_vkdecode::FaultAction::Drop) => { + tracing::warn!(len = au.len(), "PUNKTFUNK_AU_FAULT: dropping this AU"); + // Never fed, so nothing decodes and nothing is display-ready — + // the same observable state a lost AU produces. The NEXT AU is + // where detection happens. + // + // Prior frames' verdicts still have to SETTLE here, though, or a + // fault run defers every one of them by an AU and the query slots + // sit unread meanwhile. What is deliberately NOT folded is a + // clean verdict: the health ledger holds one entry per AU the + // decoder was FED, and an AU that never reached it is no evidence + // that anything is healthy — folding `note(false, false, 0)` here + // would reset the concealed run on the very AU that was lost. + let verdicts = self.settle_statuses(); + if verdicts.total() > 0 { + self.health.note(false, false, verdicts.total()); + return Err(self.status_error(verdicts)); + } + return Ok(None); + } + Some(pf_vkdecode::FaultAction::Corrupt(bytes)) => { + tracing::warn!( + len = au.len(), + corrupted_len = bytes.len(), + "PUNKTFUNK_AU_FAULT: corrupting this AU" + ); + faulted = bytes; + &faulted[..] + } + }; + + // A REFUSAL is the loudest thing this lane can say, and it has to reach + // the health ledger before it reaches the caller. Folded here rather than + // after the `?` because there is no after: a `PlanError` + // (`Parse`/`OutsideEnvelope`/`AwaitingIdr`/`NoActiveParamSet`) or a + // Vulkan/session failure returns straight out, and until M4's review this + // path incremented nothing at all — so a rung refusing EVERY AU (a host + // renegotiating outside the envelope: a frozen screen) reported + // `damaged 0 · failed 0 · run 0` and printed no integrity line whatsoever. + // A clean bill of health on a decoder that decoded nothing is the exact + // failure this program exists to end. + // + // Prior frames' status verdicts settle first, for the same reason the + // clean path settles before it folds: the refusal costs this AU, and the + // frames already shipped still owe their verdicts. + let delivered = match self.dec.decode(au) { + Ok(delivered) => delivered, + Err(e) => { + let verdicts = self.settle_statuses(); + self.health.note(false, true, verdicts.total()); + tracing::warn!( + error = %e, + driver_failed = verdicts.driver_failed, + "native decode refused the access unit" + ); + return Err(anyhow!("decode: {e}")); + } + }; let warnings = self.dec.take_warnings(); // Everything this AU made display-ready, oldest first (`take_ready` drained // so burst outputs are never stranded inside the decoder). @@ -645,38 +913,37 @@ impl NativeVulkanDecoder { fresh.push(frame); } - let corrupt = self.settle_statuses(); + let verdicts = self.settle_statuses(); // ONLY integrity warnings are concealment (see [`PlanWarnings`]): a // spec-legal envelope signal — h265's `NonZeroReorder` on every SPS // activation, h264's `Mmco5Rebase` — is an AU the planner planned in FULL, // and dropping its frame would hitch the picture at every renegotiation. let integrity = warnings.integrity(); - if !integrity.is_empty() || corrupt > 0 { - // Concealment planned into THIS AU, or driver-reported corruption on a - // PRIOR frame (the Ally X class, invisible to FFmpeg's query-less - // decoder): this call's output is released unshown and the call errors, - // arming the reanchor gate — same path, same volume as an FFmpeg - // reference-miss error (never quieter). + let concealed = !integrity.is_empty(); + // One fold per AU, whatever the verdict: a clean AU is what ENDS a run, + // and a counter that only ever counts damage cannot tell a lossy link + // apart from a stream that never came back. + self.health.note(concealed, false, verdicts.total()); + if concealed || verdicts.total() > 0 { + // Concealment planned into THIS AU, or a bad status verdict on a + // PRIOR frame (driver-reported corruption — the Ally X class, + // invisible to FFmpeg's query-less decoder — or a status that could + // not be established at all): this call's output is released unshown + // either way, because the picture is not fit to present. for frame in fresh { if let Err(e) = self.dec.release_frame(&frame, false) { tracing::debug!(error = %e, "releasing an unshown frame failed"); } } - if corrupt > 0 { - return Err(anyhow!( - "driver reported decode corruption on {corrupt} prior frame(s) \ - (RESULT_STATUS_ONLY query) — re-anchor needed" - )); + if verdicts.total() > 0 { + // A verdict about the DECODER rather than the stream: an error, + // streak-eligible, same volume as an FFmpeg reference-miss error + // (never quieter). + return Err(self.status_error(verdicts)); } - // The log carries EVERY warning of the AU (the spec-legal ones are - // context); the count is the concealment count, which is what the - // frame was dropped for. On H.264 the two coincide for every warning - // a punktfunk host can produce. - warnings.warn_concealment(); - bail!( - "AU planned with concealment ({} warning(s))", - integrity.len() - ); + warnings.warn_concealment(integrity.len()); + self.want_recovery = true; + return Ok(None); } if !warnings.is_empty() { warnings.warn_planned_in_full(); @@ -745,15 +1012,39 @@ impl NativeVulkanDecoder { } } + /// The error a bad status verdict surfaces as, worded for the device it came + /// from: a driver that reported corruption is named as such, a device that + /// cannot report one is not blamed for a verdict it never gave. + fn status_error(&self, verdicts: StatusVerdicts) -> anyhow::Error { + if verdicts.driver_failed > 0 { + anyhow!( + "driver reported decode corruption on {} prior frame(s) \ + (RESULT_STATUS_ONLY query) — re-anchor needed", + verdicts.driver_failed + ) + } else { + anyhow!( + "decode status unreadable on {} prior frame(s) (this device answers \ + no RESULT_STATUS queries — the verdict degraded to the decode \ + timeline) — re-anchor needed", + verdicts.unreadable + ) + } + } + /// Poll the status query of every unresolved shipped frame (non-blocking) and - /// release the ones that are both status-settled and token-returned. Returns how - /// many frames NEWLY read `Failed` — driver-reported corruption. + /// release the ones that are both status-settled and token-returned. Returns the + /// frames that NEWLY read `Failed`, split by whether this device can produce a + /// driver verdict at all — see [`StatusVerdicts`]. /// /// Polling an unreleased frame is always sound: its slot is pinned until /// `release_frame`, so the query slot it names cannot have been recycled under it /// (the false-`Failed` a recycled slot would read). - fn settle_statuses(&mut self) -> u32 { - let mut corrupt = 0u32; + fn settle_statuses(&mut self) -> StatusVerdicts { + let mut verdicts = StatusVerdicts::default(); + // A device fact, read once: it decides which KIND of verdict a `Failed` + // read below is (`StatusVerdicts`), never whether the frame is dropped. + let status_queries = self.dec.status_queries(); let Self { dec, outstanding, .. } = self; @@ -781,12 +1072,28 @@ impl NativeVulkanDecoder { DecodeStatus::Ok => s.resolved = true, DecodeStatus::Failed => { s.resolved = true; - corrupt += 1; - tracing::warn!( - poc = s.frame.poc, - slot = s.frame.query_slot, - "decode status query: Failed (driver-reported corruption)" - ); + if status_queries { + verdicts.driver_failed += 1; + tracing::warn!( + poc = s.frame.poc, + slot = s.frame.query_slot, + "decode status query: Failed (driver-reported corruption)" + ); + } else { + // No query pool on this device, so nothing here is the + // driver's opinion of the decode: `poll_status` degraded + // to reading the decode timeline and could not establish + // completion (a lost device, an unreadable semaphore). + // The picture is dropped exactly the same way — it is the + // ATTRIBUTION that must not be invented (`StatusVerdicts`). + verdicts.unreadable += 1; + tracing::warn!( + poc = s.frame.poc, + "decode status unreadable — this device answers no \ + RESULT_STATUS queries, so this is a timeline failure, \ + not a driver verdict" + ); + } } DecodeStatus::Pending => { if s.released { @@ -820,7 +1127,7 @@ impl NativeVulkanDecoder { } false }); - corrupt + verdicts } } @@ -952,6 +1259,18 @@ mod tests { value: 7, poc: 5, is_idr: true, + // Both facts SET and distinct from the defaults, for the same reason + // every other field here is: a projection that dropped the recovery + // mark would silently reinstate the 500 ms freeze on every + // intra-refresh session, and against `false` that passes. + recovery: pf_vkdecode::RecoveryMark { + sei_here: true, + is_recovery_point: true, + }, + // Distinct from every other number here for the same reason: a + // projection that dropped the decode ordinal would make every frame + // look pre-loss (0) and silently disable the local-recovery path. + decode_order: 17, query_slot: 2, submission: 11, picture: 6, @@ -1049,6 +1368,8 @@ mod tests { color, keyframe, poc, + recovery, + decode_order, guard: _, } = p; assert_eq!(image, 0x1001); @@ -1085,6 +1406,20 @@ mod tests { "is_idr rides through as the pump's re-anchor signal" ); assert_eq!(poc, 5); + assert_eq!( + recovery, + punktfunk_core::reanchor::LocalRecovery { + sei_here: true, + is_recovery_point: true, + }, + "the recovery point SEI's verdict reaches the gate — it is the ONLY \ + clean point an intra-refresh session has" + ); + assert_eq!( + decode_order, 17, + "the decode ordinal rides along — without it the pump cannot tell a \ + frame decoded before a loss from one decoded after it" + ); // Coincide mode: the picture IS a DPB slot, so the presenter must put the // layer back in DPB layout after sampling. @@ -1211,6 +1546,8 @@ mod tests { }, keyframe: true, poc: 0, + recovery: punktfunk_core::reanchor::LocalRecovery::NONE, + decode_order: 1, guard: NativeReleaseGuard::new( tx, NativeReleaseToken { @@ -1319,6 +1656,150 @@ mod tests { assert_eq!(mixed.integrity().len(), 1); } + /// The counter a support engineer reads first. A total alone cannot tell a + /// lossy link that keeps recovering apart from a stream that went down and + /// stayed down — `damaged 40 · run 0` and `damaged 40 · run 40` are the same + /// number and completely different problems. So the run must climb only while + /// damage is CONSECUTIVE, and the worst run must survive the recovery that + /// clears it (a once-per-second sample of `run` misses the bad moment almost + /// every time). + #[test] + fn the_concealed_run_separates_a_lossy_link_from_a_stream_that_never_came_back() { + let mut h = DecodeHealth::default(); + // A lossy link: single damaged AUs with clean stretches between. + for _ in 0..3 { + h.note(true, false, 0); + h.note(false, false, 0); + h.note(false, false, 0); + } + assert_eq!(h.damaged, 3); + assert_eq!(h.run, 0, "the last AU was clean"); + assert_eq!(h.worst_run, 1, "…and no two damaged AUs were adjacent"); + + // A stream that stopped recovering. + let mut h = DecodeHealth::default(); + for _ in 0..7 { + h.note(true, false, 0); + } + assert_eq!((h.damaged, h.run, h.worst_run), (7, 7, 7)); + // One clean AU ends the run but never the record. + h.note(false, false, 0); + assert_eq!((h.damaged, h.run, h.worst_run), (7, 0, 7)); + } + + /// A REFUSED AU — the decoder answering `Err` rather than concealing — has to + /// reach the ledger, and has to be told apart from concealment. + /// + /// This is the shape the M4 review found reporting a clean bill of health: a + /// host renegotiating outside the decode envelope makes every `plan_au` fail, + /// the picture freezes, and before this counter existed the stats surface read + /// `damaged 0 · failed 0 · run 0` and printed no integrity line at all. The + /// two counts must stay separate because they say opposite things about the + /// RUNG: concealment means the decoder coped with a damaged stream, refusal + /// means it could not run. + #[test] + fn a_rung_refusing_every_au_cannot_report_a_clean_bill_of_health() { + let mut h = DecodeHealth { + status_queries: true, + ..DecodeHealth::default() + }; + for _ in 0..5 { + h.note(false, true, 0); + } + assert_eq!(h.refused, 5, "every refusal is counted"); + assert_eq!(h.damaged, 0, "and none of them is concealment"); + assert_eq!(h.failed, 0, "nor a driver verdict — the driver never ran"); + assert_eq!( + (h.run, h.worst_run), + (5, 5), + "a refused AU is as absent from the screen as a concealed one" + ); + // A single good AU ends the run; the totals stand. + h.note(false, false, 0); + assert_eq!((h.refused, h.run, h.worst_run), (5, 0, 5)); + } + + /// The three verdicts count apart and share one run — because "the bitstream + /// arrived incomplete", "the decoder refused it" and "the hardware failed the + /// decode" have three different causes and three different fixes, while "did + /// the picture ever come back" has one answer. + #[test] + fn concealment_refusal_and_driver_failure_are_three_separate_counts() { + let mut h = DecodeHealth { + status_queries: true, + ..DecodeHealth::default() + }; + h.note(true, false, 0); + h.note(false, true, 0); + h.note(false, false, 2); + assert_eq!((h.damaged, h.refused, h.failed), (1, 1, 2)); + assert_eq!((h.run, h.worst_run), (3, 3), "one unbroken run of three"); + } + + /// A driver `Failed` verdict counts apart from concealment and extends the same + /// run. Apart, because "the bitstream arrived incomplete" and "the hardware + /// could not decode what arrived" have different causes and different fixes, + /// and collapsing them is how "the stream is fine, it's your GPU" arguments + /// start. Same run, because a frame the driver failed is as absent from the + /// screen as a concealed one — and "did the picture ever come back" is what the + /// run answers. + #[test] + fn driver_failures_count_separately_but_share_the_run() { + let mut h = DecodeHealth { + status_queries: true, + ..DecodeHealth::default() + }; + h.note(false, false, 2); // two prior frames reported corrupt at once + assert_eq!((h.damaged, h.failed, h.run), (0, 2, 1)); + h.note(true, false, 1); // and an AU that ALSO needed concealment + assert_eq!((h.damaged, h.failed, h.run), (1, 3, 2)); + h.note(false, false, 0); + assert_eq!((h.run, h.worst_run), (0, 2)); + } + + /// `status_queries` is set once from the device and never touched by the + /// per-AU fold — a counter update must not be able to turn "this driver cannot + /// report corruption" into "it reported none". + /// + /// And, the invariant the doc contracts on both sides of this boundary state: + /// where the device answers no status queries, `failed` can only ever read 0. + /// It is not a hypothetical. `read_status` returns `Failed` on such a device + /// for a lost device, a retired session generation or an unreadable semaphore, + /// and counting those would render `integrity: driver-failed 1 · no driver + /// status` — one line contradicting itself, pointing a support engineer at a + /// verdict the hardware cannot give. So this feeds `note` a real failure and + /// pins the zero; a test that only ever passed `0` would assert nothing. + #[test] + fn the_status_query_capability_survives_every_fold() { + let mut h = DecodeHealth { + status_queries: false, + ..DecodeHealth::default() + }; + h.note(true, false, 0); + h.note(false, false, 0); + assert!(!h.status_queries); + h.note(false, false, 1); + assert_eq!( + h.failed, 0, + "a device that answers no status queries can produce no driver \ + verdict — `failed` must stay 0 whatever `read_status` returned" + ); + assert_eq!( + h.run, 1, + "…but the frame was still dropped, so the run still counts it: the \ + ATTRIBUTION is what must not be invented, not the damage" + ); + assert_eq!(h.worst_run, 1); + + // The same fold on a device that CAN answer does count it. + let mut h = DecodeHealth { + status_queries: true, + ..DecodeHealth::default() + }; + h.note(false, false, 1); + assert_eq!((h.failed, h.run), (1, 1)); + } + #[test] fn the_queue_lock_is_shared_only_when_the_families_collide() { // Same family ⇒ same VkQueue (both sides use index 0) ⇒ shared lock. diff --git a/crates/pf-presenter/src/run.rs b/crates/pf-presenter/src/run.rs index a587fda2..718151a9 100644 --- a/crates/pf-presenter/src/run.rs +++ b/crates/pf-presenter/src/run.rs @@ -2565,6 +2565,57 @@ fn stats_text( text.push_str(&format!(" · dropped {}", s.mic_dropped)); } } + // Decode integrity (M4) — the native lane's answer to "was that stream actually + // clean?". Appended LAST and only when it has something to say, which keeps it + // additive for the stdout `stats:` line's parsers (a machine interface: every + // existing segment stays where it was) and keeps a healthy session's OSD exactly + // as quiet as it is today. + // + // "Something to say" deliberately includes a device with no `RESULT_STATUS` + // support (RADV), even with zero damage: there the counters cover the parser's + // half only, and a silent integrity line would read as a clean bill of health on + // the one configuration that cannot give one. Saying "no driver status" once a + // second is the whole lesson of `nb_queries = 0` — an unmeasured session must + // never look like a measured one. A lane that cannot report integrity at all (any + // FFmpeg rung) prints nothing rather than zeros, for the same reason. + if detailed && s.decode_integrity { + let mut parts: Vec = Vec::new(); + if s.decode_damaged > 0 { + parts.push(format!("damaged {}", s.decode_damaged)); + } + if s.decode_refused > 0 { + // The decoder could not run at all — a different diagnosis from + // `damaged`, and the one that means the screen is frozen rather than + // occasionally glitching. Without it a rung refusing every AU printed + // no integrity line whatsoever. + parts.push(format!("refused {}", s.decode_refused)); + } + if s.decode_failed > 0 { + parts.push(format!("driver-failed {}", s.decode_failed)); + } + if s.concealed_run > 0 { + // The figure that says "and it has not recovered" — a run still climbing + // at the end of the window is a different problem from the same count of + // isolated damaged AUs. + parts.push(format!("run {}", s.concealed_run)); + } + if s.worst_concealed_run > s.concealed_run { + // Only when it says something the instantaneous run does not: this is + // sampled once a second and the worst moment lasts a handful of frames, + // so a window that reads `damaged 40` with no run at all is either forty + // isolated glitches or one 40-AU freeze that recovered — and until this + // figure was surfaced, nothing on the OSD could tell those apart. + // Session-cumulative, unlike everything before it on this line, which is + // why it is labelled rather than folded into `run`. + parts.push(format!("worst run {}", s.worst_concealed_run)); + } + if !s.decode_status_queries { + parts.push("no driver status".into()); + } + if !parts.is_empty() { + text.push_str(&format!("\nintegrity: {}", parts.join(" · "))); + } + } text } @@ -2819,6 +2870,15 @@ mod tests { auto_rate: false, chroma_444: false, asked_444: false, + // An FFmpeg rung: it cannot answer integrity questions at all, so + // every existing tier text below must be unchanged by M4's line. + decode_integrity: false, + decode_damaged: 0, + decode_failed: 0, + decode_refused: 0, + concealed_run: 0, + worst_concealed_run: 0, + decode_status_queries: false, }, PresentedWindow { e2e_p50_ms: 6.4, @@ -2928,6 +2988,138 @@ mod tests { assert!(!normal.contains("present:") && !normal.contains("pace")); } + /// The decode-integrity line (M4) — the whole point of which is that it can tell + /// three states apart that all look identical as "no complaints today": + /// + /// * a lane that CANNOT see corruption (any FFmpeg rung — `nb_queries = 0`, no + /// `AV_FRAME_FLAG_CORRUPT`): silent, never zeros, because printing zeros would + /// assert a cleanliness nothing checked; + /// * a lane that looked and saw nothing: also silent, but it earned it; + /// * a lane that looked with only half its detectors — a device without + /// `queryResultStatusSupport` — which says so EVERY window, damage or not. + /// + /// Plus the shape a support engineer actually needs when there IS damage: how + /// much, whose fault (stream vs driver), and whether it ever recovered. + #[test] + fn the_integrity_line_distinguishes_clean_from_unmeasurable() { + let (base, p) = sample(); + let line = |s: &Stats| { + stats_text( + StatsVerbosity::Detailed, + "m", + s, + &p, + false, + false, + false, + None, + ) + .lines() + .find(|l| l.starts_with("integrity:")) + .map(str::to_string) + }; + + // An FFmpeg rung cannot answer at all — nothing is printed. (The fixture is + // one, so this also pins that every other tier text is untouched by M4.) + assert_eq!(line(&base), None, "a lane with no detectors says nothing"); + + // The native rung on a device with full status support, decoding clean: + // also nothing — a healthy session's OSD stays exactly as quiet as it was. + let clean = Stats { + decode_integrity: true, + decode_status_queries: true, + ..base + }; + assert_eq!(line(&clean), None); + + // The same rung on RADV, where a RESULT_STATUS query would hang the VCN ring: + // clean counters, but only the parser's half was ever measured, and the line + // says so rather than implying a full bill of health. + let unmeasured = Stats { + decode_status_queries: false, + ..clean + }; + assert_eq!( + line(&unmeasured).as_deref(), + Some("integrity: no driver status") + ); + + // Damage, attributed: concealment is the stream's, `driver-failed` is the + // hardware's, and `run` answers "did it come back?". + let damaged = Stats { + decode_damaged: 4, + decode_failed: 2, + concealed_run: 3, + worst_concealed_run: 3, + ..clean + }; + assert_eq!( + line(&damaged).as_deref(), + Some("integrity: damaged 4 · driver-failed 2 · run 3") + ); + + // A lossy window the stream recovered from: the run is 0 and simply drops out. + let recovered = Stats { + decode_damaged: 4, + concealed_run: 0, + ..clean + }; + assert_eq!(line(&recovered).as_deref(), Some("integrity: damaged 4")); + + // …and the reason that window is not the whole story. `concealed_run` is an + // INSTANT sampled once a second; the freeze it missed lasted 40 AUs. Forty + // isolated glitches and one 40-AU freeze that recovered render identically + // without the session's worst run, and they are completely different bugs. + let recovered_hard = Stats { + worst_concealed_run: 40, + ..recovered + }; + assert_eq!( + line(&recovered_hard).as_deref(), + Some("integrity: damaged 4 · worst run 40") + ); + // It stays quiet whenever it adds nothing — a run still climbing at the end + // of the window already IS the worst one. + let still_broken = Stats { + concealed_run: 40, + worst_concealed_run: 40, + ..recovered + }; + assert_eq!( + line(&still_broken).as_deref(), + Some("integrity: damaged 4 · run 40") + ); + + // A rung that REFUSED every AU — a host renegotiating outside the decode + // envelope. The screen is frozen, nothing was concealed, no driver verdict + // exists, and before M4's review this printed no integrity line at all: a + // decoder that decoded nothing, reported as a clean session. + let refusing = Stats { + decode_refused: 60, + concealed_run: 60, + worst_concealed_run: 60, + ..clean + }; + assert_eq!( + line(&refusing).as_deref(), + Some("integrity: refused 60 · run 60") + ); + + // Never below Detailed — the tier ladder is a strict superset chain and this + // is diagnostic detail, not a glanceable number. + for tier in [ + StatsVerbosity::Compact, + StatsVerbosity::Normal, + StatsVerbosity::Off, + ] { + assert!( + !stats_text(tier, "m", &damaged, &p, false, false, false, None) + .contains("integrity:"), + "{tier:?}" + ); + } + } + /// The honest HDR badges: a PQ stream on the software-decode lane is shown WITHOUT /// tone-mapping (that lane has no PQ→sRGB pass), so its badge must not read as the /// hardware lane's `HDR→SDR` tone-map — and an HDR10 swapchain shows plain `HDR` diff --git a/crates/pf-vkdecode/src/decoder.rs b/crates/pf-vkdecode/src/decoder.rs index aeb9f0d9..35ffde48 100644 --- a/crates/pf-vkdecode/src/decoder.rs +++ b/crates/pf-vkdecode/src/decoder.rs @@ -157,6 +157,31 @@ pub struct DecodedVkFrame { pub value: u64, pub poc: i32, pub is_idr: bool, + /// What the recovery point SEI of this picture's AU (and any outstanding one + /// before it) is worth — see [`crate::recovery`]. `RecoveryMark::NONE` on every + /// picture of a stream that carries no recovery point SEI, which is every + /// punktfunk host today that is not running an NVENC intra-refresh wave. + /// + /// It exists because [`Self::is_idr`] cannot answer for an intra-refresh + /// session: the wave never emits an IDR, so a consumer freezing on loss has no + /// decoder-visible clean point and holds the last good picture until its + /// backstop forces the very IDR the wave exists to avoid. The mark is the + /// stream saying, in-band, where it healed. + pub recovery: crate::recovery::RecoveryMark, + /// This picture's position in DECODE order: a strictly increasing per-decoder + /// ordinal stamped when the AU was planned (1 for the first picture of the + /// decoder's life; it survives session rebuilds, because it describes the + /// STREAM, not the Vulkan objects). + /// + /// It exists because delivery order is not decode order, and a consumer + /// pairing [`Self::recovery`] against its own loss needs to know which of the + /// two a frame belongs to. A post-failure DPB flush hands back every picture + /// still buffered — pictures decoded BEFORE the loss — and each carries the + /// recovery marks of the wave it was decoded in. Delivered after the + /// consumer armed its freeze, those marks read as a heal that happened after + /// the loss, and lift a freeze on a wave that completed before it. Comparing + /// this ordinal against the one current at the arm is what tells them apart. + pub decode_order: u64, /// The decode op's slot in the status query pool. pub query_slot: u32, /// The decode op's submission ordinal (validates the query slot has not been @@ -479,6 +504,12 @@ pub(crate) struct PendingPic { pub(crate) colour: ColourDescription, pub(crate) poc: i32, pub(crate) is_idr: bool, + /// Folded at PLAN time (the only place the codec's counting unit is known) and + /// carried here, because a picture's display order is not its decode order — + /// see [`DecodedVkFrame::recovery`]. + pub(crate) recovery: crate::recovery::RecoveryMark, + /// See [`DecodedVkFrame::decode_order`]. + pub(crate) decode_order: u64, } /// A retired generation's picture pool: images the presenter still holds live @@ -539,6 +570,14 @@ pub struct VkH264Decoder { graveyard: Vec, /// The most recent plan's warnings ([`Self::take_warnings`]). last_warnings: Vec, + /// The outstanding recovery point SEI, if any — see [`crate::recovery`]. + /// Survives session rebuilds on purpose: it is a fact about the STREAM's + /// prediction structure, not about this decoder's Vulkan objects. + recovery_watch: crate::recovery::RecoveryWatch, + /// Pictures planned so far — stamped onto each one as + /// [`DecodedVkFrame::decode_order`]. Survives session rebuilds for the same + /// reason the watch does. + decoded: u64, /// Session generation: bumped on every rebuild, stamped into frames. generation: u64, device_lost: bool, @@ -576,6 +615,8 @@ impl VkH264Decoder { ready: VecDeque::new(), graveyard: Vec::new(), last_warnings: Vec::new(), + recovery_watch: crate::recovery::RecoveryWatch::new(), + decoded: 0, generation: 0, device_lost: false, }) @@ -598,6 +639,14 @@ impl VkH264Decoder { } fn decode_inner(&mut self, au: &[u8]) -> Result, VkDecodeError> { + // `take_warnings` promises "cleared by the next decode", and this IS a + // decode: clear BEFORE planning, so an AU that fails to plan at all cannot + // leave the previous AU's warnings behind to be re-read as damage on the + // next one. That the ledger is drained after every successful decode + // (`take_warnings` is a `mem::take`) makes the failed-plan case the only + // one where it could carry over — it is a hole closed by construction, not + // a fix for anything observed in the field. + self.last_warnings.clear(); let plan = self.planner.plan_au(au)?; for warning in &plan.warnings { // The recovery verdict is the integration layer's @@ -605,6 +654,26 @@ impl VkH264Decoder { trace!(?warning, "plan warning"); } self.last_warnings = plan.warnings.clone(); + // One picture per AU under this envelope: stamp its DECODE-order ordinal + // before anything can reorder it (see `DecodedVkFrame::decode_order`). + self.decoded = self.decoded.saturating_add(1); + let decode_order = self.decoded; + // The recovery-point watch, folded ONCE per successfully planned AU and in + // DECODE order — the order the SEI counts in. The mark rides the pending + // picture to display order, which may differ (crate::recovery). + let recovery = self.recovery_watch.note_h264( + plan.picture.frame_num, + plan.picture.is_idr, + plan.picture.recovery_point, + ); + if recovery != crate::recovery::RecoveryMark::NONE { + trace!( + sei = recovery.sei_here, + recovery_point = recovery.is_recovery_point, + frame_num = plan.picture.frame_num, + "recovery point SEI" + ); + } self.ensure_state(&plan)?; let sps_id = plan.sps.seq_parameter_set_id; @@ -810,6 +879,8 @@ impl VkH264Decoder { colour: plan.picture.colour, poc: plan.picture.pic_order_cnt, is_idr: plan.picture.is_idr, + recovery, + decode_order, }, ); @@ -922,6 +993,14 @@ impl VkH264Decoder { self.generation } + /// The DECODE-order ordinal of the most recently planned picture — the + /// watermark a consumer compares [`DecodedVkFrame::decode_order`] against to + /// tell a frame decoded before a loss from one decoded after it. 0 before the + /// first AU plans. + pub fn decode_order(&self) -> u64 { + self.decoded + } + /// One-line state snapshot for failure paths and field logs (not a stable /// format). pub fn debug_snapshot(&self) -> String { @@ -976,6 +1055,24 @@ impl VkH264Decoder { self.read_status(frame, false) } + /// Does this decode queue family answer per-op `RESULT_STATUS` queries at all + /// (`queryResultStatusSupport`)? + /// + /// The single most important thing a support engineer can know about a + /// session's integrity reporting, and the reason this is exposed rather than + /// left internal. Where it is TRUE, [`DecodeStatus::Failed`] is the driver's + /// own verdict on a decode operation — the signal the Xbox Ally X corruption + /// needed and FFmpeg's query-less Vulkan decoder (`nb_queries = 0`) can never + /// produce. Where it is FALSE — RADV, whose VCN ring HANGS if a query is + /// recorded anyway — `Ok` degrades to "the op completed on the timeline", which + /// is exactly as much as FFmpeg knows on every driver: no worse, but a clean + /// integrity report from such a session means "nothing was detectable", not + /// "nothing was wrong". A telemetry surface that cannot say which of those it + /// is repeats the failure this program exists to end. + pub fn status_queries(&self) -> bool { + self.dev.result_status_queries() + } + /// [`Self::poll_status`], but WAITs for the op to complete first — the only /// place a status read blocks (the GPU smoke test's assertion path; the /// integration layer's steady state polls). @@ -1407,6 +1504,8 @@ pub(crate) fn build_frame( value: entry.timeline_value, poc: entry.poc, is_idr: entry.is_idr, + recovery: entry.recovery, + decode_order: entry.decode_order, query_slot: entry.query_slot, submission: entry.submission, picture: entry.image as u32, diff --git a/crates/pf-vkdecode/src/decoder_h265.rs b/crates/pf-vkdecode/src/decoder_h265.rs index 41b01cd4..54558cb5 100644 --- a/crates/pf-vkdecode/src/decoder_h265.rs +++ b/crates/pf-vkdecode/src/decoder_h265.rs @@ -200,6 +200,15 @@ pub struct VkH265Decoder { graveyard: Vec, /// The most recent plan's warnings ([`Self::take_warnings`]). last_warnings: Vec, + /// The outstanding recovery point SEI, if any — see [`crate::recovery`]. + /// Named apart from [`Self::recovery`], which is this decoder's DPB-recovery + /// latch: the two are unrelated (one is a fact about the stream's prediction + /// structure, the other about this decoder's own wedged state). + recovery_watch: crate::recovery::RecoveryWatch, + /// Pictures planned so far — stamped onto each one as + /// [`DecodedVkFrame::decode_order`]. Survives session rebuilds for the same + /// reason the watch does. + decoded: u64, /// Session generation: bumped on every rebuild, stamped into frames. generation: u64, device_lost: bool, @@ -248,6 +257,8 @@ impl VkH265Decoder { ready: VecDeque::new(), graveyard: Vec::new(), last_warnings: Vec::new(), + recovery_watch: crate::recovery::RecoveryWatch::new(), + decoded: 0, generation: 0, device_lost: false, recovery: RecoveryLatch::default(), @@ -322,15 +333,17 @@ impl VkH265Decoder { if self.recovery.take() { self.recover_dpb(); } + // Cleared BEFORE planning for the same reason the RASL arm below clears it: + // "cleared by the next decode" must hold for an AU that fails to plan at + // all, or the previous AU's warnings could be re-read as fresh damage. The + // ledger is drained after every successful decode (`take_warnings` is a + // `mem::take`), so the failed-plan case is the only one that could carry + // over — a hole closed by construction, not a fix for a field symptom. + self.last_warnings.clear(); let plan = match self.planner.plan_au(au) { Ok(plan) => plan, Err(PlanError::RaslSkipped { poc }) => { trace!(poc, "RASL picture after a CRA join — skipped, not failed"); - // take_warnings promises "cleared by the next decode", and this IS - // a decode: a caller polling per AU must not see the PREVIOUS AU's - // warnings a second time — a `MissingReference` re-reported right - // after a CRA join is exactly when the keyframe decision is made. - self.last_warnings.clear(); return Ok(self.ready.pop_front()); } Err(e) => return Err(VkDecodeError::PlanH265(e)), @@ -341,6 +354,26 @@ impl VkH265Decoder { trace!(?warning, "plan warning"); } self.last_warnings = plan.warnings.clone(); + // One picture per AU under this envelope: stamp its DECODE-order ordinal + // before anything can reorder it (see `DecodedVkFrame::decode_order`). + self.decoded = self.decoded.saturating_add(1); + let decode_order = self.decoded; + // The recovery-point watch, folded ONCE per successfully planned AU and in + // DECODE order — the order the SEI's POC delta is measured in. The mark + // rides the pending picture into display order (crate::recovery). + let recovery = self.recovery_watch.note_h265( + plan.picture.pic_order_cnt, + plan.picture.is_irap, + plan.picture.recovery_point, + ); + if recovery != crate::recovery::RecoveryMark::NONE { + trace!( + sei = recovery.sei_here, + recovery_point = recovery.is_recovery_point, + poc = plan.picture.pic_order_cnt, + "recovery point SEI" + ); + } // From here the PLANNER has already advanced past this AU — its DPB holds // the picture whatever happens next — so any failure below leaves the @@ -351,7 +384,7 @@ impl VkH265Decoder { // `UnresolvedReference`, an `ensure_state` refusal — strands the picture // the other way round, planner-resident with no slot at all, and wedges // just as hard. One flush cures both.) - let result = self.decode_planned(&plan, au); + let result = self.decode_planned(&plan, au, recovery, decode_order); if result.is_err() { self.recovery.latch(); } @@ -361,11 +394,16 @@ impl VkH265Decoder { /// The submission half of one decode, from the point the planner has already /// advanced. Split out so [`Self::decode_inner`] can latch recovery on ANY /// failure past that line without threading a flag through every exit. - /// `au` is the same buffer `plan`'s slice ranges index into. + /// `au` is the same buffer `plan`'s slice ranges index into; `recovery` is the + /// recovery-point verdict already folded for this AU and `decode_order` its + /// decode-order ordinal (both advance in decode order, so neither can be + /// derived here — this path is not reached for every planned AU). fn decode_planned( &mut self, plan: &AuPlan, au: &[u8], + recovery: crate::recovery::RecoveryMark, + decode_order: u64, ) -> Result, VkDecodeError> { self.ensure_state(plan)?; @@ -580,6 +618,8 @@ impl VkH265Decoder { colour: plan.picture.colour, poc: plan.picture.pic_order_cnt, is_idr: plan.picture.is_idr, + recovery, + decode_order, }, ); @@ -690,6 +730,16 @@ impl VkH265Decoder { self.generation } + /// The DECODE-order ordinal of the most recently planned picture — the + /// watermark a consumer compares [`DecodedVkFrame::decode_order`] against to + /// tell a frame decoded before a loss from one decoded after it. Especially + /// load-bearing here: [`Self::recover_dpb`] flushes every buffered picture + /// into `ready` at once, so a pre-loss picture routinely reaches the consumer + /// after the loss that flushed it. 0 before the first AU plans. + pub fn decode_order(&self) -> u64 { + self.decoded + } + /// One-line state snapshot for failure paths and field logs (not a stable /// format). pub fn debug_snapshot(&self) -> String { @@ -752,6 +802,14 @@ impl VkH265Decoder { self.read_status(frame, false) } + /// Does this decode queue family answer per-op `RESULT_STATUS` queries at all? + /// See [`crate::VkH264Decoder::status_queries`] — the fact is the DEVICE's, identical + /// for both codecs, and it is what tells a clean integrity report apart from + /// an undetectable one. + pub fn status_queries(&self) -> bool { + self.dev.result_status_queries() + } + /// [`Self::poll_status`], but WAITs for the op to complete first. pub fn wait_status(&mut self, frame: &DecodedVkFrame) -> DecodeStatus { self.read_status(frame, true) diff --git a/crates/pf-vkdecode/src/fault.rs b/crates/pf-vkdecode/src/fault.rs new file mode 100644 index 00000000..6e46eb00 --- /dev/null +++ b/crates/pf-vkdecode/src/fault.rs @@ -0,0 +1,305 @@ +//! Deliberate decoder-input corruption — the fault injector (M4 of the +//! native-decode program). +//! +//! # Why a first-class tool +//! +//! The whole program started from a field corruption that was ARCHITECTURALLY +//! undetectable: FFmpeg's Vulkan decoder creates no status queries +//! (`nb_queries = 0`), never sets `AV_FRAME_FLAG_CORRUPT`, and reports trouble only +//! as `av_log` lines. Nobody could tell a healthy stream from a broken one without +//! looking at the screen. The native decoder now has the signals — plan warnings +//! from pf-bitstream and per-op `RESULT_STATUS` verdicts from the driver — but a +//! detector nobody can fire is exactly as trustworthy as no detector at all. This +//! is the trigger: a deterministic way to break decoder input on purpose, so +//! detection can be PROVEN rather than assumed, on a lab box or in CI. +//! +//! It is inert unless explicitly armed ([`AuFault::from_spec`] returns `None` for +//! an unset/unparsable spec) and it is pure — no I/O, no clock, no randomness — so +//! a reported fault is reproducible from the spec string alone. +//! +//! # The three modes, and which detector each one fires +//! +//! They are not variations on one idea. The native lane has TWO independent +//! detectors — pf-bitstream's planner, which reads syntax, and the driver's per-op +//! `RESULT_STATUS` query, which reads the decode itself — and the modes exist to +//! fire them separately, because a harness that can only trip one of them proves +//! only half the lane: +//! +//! * [`FaultMode::Drop`] — the AU never reaches the decoder. The NEXT AU then +//! references a picture that was never decoded, so the planner reports +//! `FrameNumGap`/`MissingReference`: **parser-visible** damage, caught before a +//! single macroblock is decoded. The everyday network-loss shape, and the one +//! mode whose detection is provable without a GPU. +//! * [`FaultMode::Truncate`] — the AU arrives short. Worth knowing, and initially +//! surprising: this is **NOT** parser-visible. Annex-B carries no NALU length, so +//! a slice cut at a byte boundary is simply a shorter slice — its header parses, +//! the picture plans, every later reference resolves against a DPB entry that +//! exists. (pf-bitstream's `TruncatedAu` warning is a narrower thing: a NALU +//! whose HEADER is malformed with real data still behind it.) What the hardware +//! gets is a slice whose bitstream ends mid-picture, which is a decode error it +//! can report — so this is the mode that fires the DRIVER's detector +//! deterministically. +//! * [`FaultMode::Flip`] — one byte deep inside the slice payload is altered. The +//! bitstream still parses, every reference still resolves, the planner has +//! nothing to say — and the picture decodes WRONG, possibly without the driver +//! minding either (an entropy decoder happily decodes garbage into macroblocks). +//! This is precisely the Xbox Ally X class: corruption that reaches the screen +//! with nothing in the pipeline objecting. It is the mode that shows what +//! `RESULT_STATUS` can and cannot promise. +//! +//! The consequence worth stating plainly, because it is the program's whole thesis: +//! two of the three modes are invisible to every FFmpeg rung by construction +//! (`nb_queries = 0`, no `AV_FRAME_FLAG_CORRUPT`), and invisible to the native lane +//! too on a driver without `queryResultStatusSupport` (RADV). A session that cannot +//! answer the status query is not a clean session; it is an unmeasured one, and the +//! telemetry says so rather than reporting zeros. +//! +//! # Invocation +//! +//! `PUNKTFUNK_AU_FAULT=[:]` on any desktop client — +//! `drop`, `truncate`, `flip`, default period 60 (once a second at 60 fps): +//! +//! ```text +//! PUNKTFUNK_AU_FAULT=drop:120 PUNKTFUNK_DECODER=native-vulkan punktfunk-session --connect host +//! ``` +//! +//! Every `period`-th AU is faulted, counting from the first one the decoder is +//! offered, so the parameter sets and opening IDR of a session ride through +//! untouched at any period above 1. +//! +//! # What the injector is NOT in the same lane as +//! +//! `PUNKTFUNK_AU_DUMP` (the client's `au_dump` fixture capture) writes the AU as +//! it arrives from the wire, and this injector runs LATER — at the native +//! backend's decode entry, the last point before pf-bitstream. So on a faulted +//! run the dumped fixture is the CLEAN bitstream, not the one the decoder saw; +//! replaying it will not reproduce the damage. That ordering is deliberate: the +//! dump is what the HOST sent (the artefact a host-side bug is diagnosed from), +//! and moving the injector above it would corrupt every backend's input rather +//! than only the lane whose detectors it exists to fire. To capture the damaged +//! bytes, reconstruct them from the spec — the injector is pure and deterministic, +//! which is precisely what makes that possible. + +/// What to do to a faulted access unit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FaultMode { + /// Swallow the AU entirely — the decoder never sees it (network loss). + Drop, + /// Deliver a prefix of the AU: a picture whose slice data stops mid-frame. + Truncate, + /// Deliver the whole AU with one payload byte altered (in-picture corruption + /// no parser can see). + Flip, +} + +/// What the caller must do with the access unit it was about to decode. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FaultAction { + /// Untouched — feed the original bytes. The answer for every AU but one in + /// `period`, and for every AU of a session with no fault armed. + Pass, + /// Do not feed this AU at all. + Drop, + /// Feed these bytes instead. Owned because corruption is by definition not a + /// borrow of the input; allocated only on the faulted AU, never on the + /// streaming path. + Corrupt(Vec), +} + +/// The default fault period: one AU per second at 60 fps — frequent enough to see +/// within a few seconds of streaming, rare enough that recovery completes between +/// faults instead of the stream never leaving its post-loss freeze. +pub const DEFAULT_FAULT_PERIOD: u32 = 60; + +/// Where in a faulted AU the damage lands, as a fraction of its length. Deep +/// enough to be past the parameter sets and the first slice header (so `Flip` +/// really is invisible to the parser and `Truncate` really does cut mid-picture +/// rather than refusing the AU at byte 0), and expressed as a fraction so it holds +/// for a 700-byte P-frame and a 4 MB IDR alike. +const FAULT_POINT_NUMERATOR: usize = 3; +const FAULT_POINT_DENOMINATOR: usize = 4; + +/// The armed injector: a mode, a period, and the count of AUs offered so far. +#[derive(Debug, Clone, Copy)] +pub struct AuFault { + mode: FaultMode, + period: u32, + seen: u32, +} + +impl AuFault { + /// Parse a `PUNKTFUNK_AU_FAULT` spec: `[:]`. `None` for anything + /// unrecognized, which is what keeps the injector inert — a typo must leave a + /// user's stream alone rather than half-arm it. + pub fn from_spec(spec: &str) -> Option { + let spec = spec.trim(); + let (mode, period) = match spec.split_once(':') { + Some((m, p)) => (m.trim(), p.trim().parse::().ok()?), + None => (spec, DEFAULT_FAULT_PERIOD), + }; + // A zero period would fault EVERY AU including the opening IDR, which + // never produces a stream to damage in the first place. + if period == 0 { + return None; + } + let mode = match mode { + "drop" => FaultMode::Drop, + "truncate" => FaultMode::Truncate, + "flip" => FaultMode::Flip, + _ => return None, + }; + Some(AuFault { + mode, + period, + seen: 0, + }) + } + + /// Build one directly (tests and callers that resolve the spec themselves). + pub fn new(mode: FaultMode, period: u32) -> AuFault { + AuFault { + mode, + period: period.max(1), + seen: 0, + } + } + + pub fn mode(&self) -> FaultMode { + self.mode + } + + pub fn period(&self) -> u32 { + self.period + } + + /// Offer one access unit. Returns what the caller should feed the decoder. + /// + /// The counter advances on EVERY call, faulted or not, so the cadence is a + /// property of the stream rather than of the damage: `period` AUs of clean + /// stream, one fault, repeat. + pub fn apply(&mut self, au: &[u8]) -> FaultAction { + self.seen = self.seen.wrapping_add(1); + if self.seen % self.period != 0 { + return FaultAction::Pass; + } + // Too short to damage meaningfully — a handful of bytes is a parameter-set + // AU or a fragment, and cutting/flipping inside one tests the parser's + // error handling rather than the decoder's integrity signals. Pass it and + // let the next multiple carry the fault. + if au.len() < 16 { + return FaultAction::Pass; + } + let point = au.len() * FAULT_POINT_NUMERATOR / FAULT_POINT_DENOMINATOR; + match self.mode { + FaultMode::Drop => FaultAction::Drop, + FaultMode::Truncate => FaultAction::Corrupt(au[..point].to_vec()), + FaultMode::Flip => { + let mut bytes = au.to_vec(); + // XOR with a single high-ish bit rather than inverting the byte: + // it moves the sample values the entropy coder decodes without + // being especially likely to manufacture a `00 00 01` start code + // out of the surrounding bytes (which would turn a payload + // corruption into a framing corruption and quietly change which + // detector the mode is testing). + bytes[point] ^= 0x40; + FaultAction::Corrupt(bytes) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The knob is a support tool: it has to be exactly as inert as it looks when + /// unset or mistyped, because the alternative is a user's stream quietly + /// breaking on a typo'd environment variable. + #[test] + fn only_a_well_formed_spec_arms_the_injector() { + assert_eq!( + AuFault::from_spec("drop").map(|f| (f.mode(), f.period())), + Some((FaultMode::Drop, DEFAULT_FAULT_PERIOD)) + ); + assert_eq!( + AuFault::from_spec("truncate:5").map(|f| (f.mode(), f.period())), + Some((FaultMode::Truncate, 5)) + ); + assert_eq!( + AuFault::from_spec(" flip : 30 ").map(|f| (f.mode(), f.period())), + Some((FaultMode::Flip, 30)) + ); + for bad in [ + "", "1", "off", "drop:", "drop:0", "drop:x", "corrupt", "flip:-1", ":30", + ] { + assert!(AuFault::from_spec(bad).is_none(), "{bad:?} must not arm"); + } + } + + /// The cadence: `period - 1` clean AUs, then one faulted, forever. The count + /// starts at the first AU offered, so a period above 1 never touches the + /// session's opening parameter sets and IDR. + #[test] + fn every_nth_au_is_faulted_and_the_rest_pass_through_untouched() { + let au = vec![0xA5u8; 64]; + let mut f = AuFault::new(FaultMode::Drop, 3); + assert_eq!(f.apply(&au), FaultAction::Pass); + assert_eq!(f.apply(&au), FaultAction::Pass); + assert_eq!(f.apply(&au), FaultAction::Drop); + assert_eq!(f.apply(&au), FaultAction::Pass); + assert_eq!(f.apply(&au), FaultAction::Pass); + assert_eq!(f.apply(&au), FaultAction::Drop); + } + + /// Truncation delivers a real prefix — the shape a lost tail shard has, not a + /// zero-length AU (which is a different, uninteresting failure). + #[test] + fn truncation_delivers_a_prefix_of_the_original() { + let au: Vec = (0..100u8).collect(); + let mut f = AuFault::new(FaultMode::Truncate, 1); + let FaultAction::Corrupt(short) = f.apply(&au) else { + panic!("truncate must corrupt"); + }; + assert_eq!(short.len(), 75, "three quarters of the AU survive"); + assert_eq!(short[..], au[..75], "and they are the ORIGINAL bytes"); + } + + /// A flip alters exactly one byte, deep in the payload, deterministically — + /// the corruption a parser cannot see. Determinism is what makes a field + /// report reproducible from the spec string alone. + #[test] + fn a_flip_changes_exactly_one_deep_payload_byte_and_is_reproducible() { + let au: Vec = (0..=255u8).collect(); + let run = || { + let mut f = AuFault::new(FaultMode::Flip, 1); + match f.apply(&au) { + FaultAction::Corrupt(bytes) => bytes, + other => panic!("flip must corrupt, got {other:?}"), + } + }; + let bytes = run(); + assert_eq!( + bytes.len(), + au.len(), + "length is untouched — this is not a cut" + ); + let differing: Vec = (0..au.len()).filter(|&i| bytes[i] != au[i]).collect(); + assert_eq!(differing.len(), 1, "exactly one byte moves"); + let at = differing[0]; + assert_eq!(at, 192, "three quarters in — past the headers"); + assert_eq!(bytes[at], au[at] ^ 0x40); + assert_eq!(run(), bytes, "the same spec produces the same damage"); + } + + /// Tiny AUs (a lone parameter-set NALU, a fragment) are passed through: the + /// modes are about damaging a PICTURE, and cutting a 6-byte AU only tests the + /// parser's own bounds checks. + #[test] + fn an_au_too_short_to_damage_meaningfully_is_left_alone() { + let mut f = AuFault::new(FaultMode::Truncate, 1); + assert_eq!(f.apply(&[0u8; 8]), FaultAction::Pass); + // …and the counter still advanced, so the cadence does not stall waiting + // for a big enough AU. + assert!(matches!(f.apply(&[0u8; 64]), FaultAction::Corrupt(_))); + } +} diff --git a/crates/pf-vkdecode/src/integrity.rs b/crates/pf-vkdecode/src/integrity.rs new file mode 100644 index 00000000..14c57195 --- /dev/null +++ b/crates/pf-vkdecode/src/integrity.rs @@ -0,0 +1,108 @@ +//! Which planner warnings mean the PICTURE is damaged (M4 of the native-decode +//! program). +//! +//! Both planners emit two very different kinds of thing through one warning +//! channel, and the split is what a consumer must branch on: +//! +//! * **Integrity** — a reference the DPB does not hold, a `frame_num` gap, an AU +//! whose NALU walk stopped early. The plan was completed with a SUBSTITUTE in +//! place of something that was lost, so the decoded picture is damaged: its +//! output must be released unshown and the stream must ask for a re-anchor. +//! * **Spec-legal envelope signals** — h265's `NonZeroReorder` (the activated SPS +//! sets `sps_max_num_reorder_pics > 0`) and h264's `Mmco5Rebase`. pf-bitstream +//! documents both as spec-legal and fully planned; they exist as the field +//! signal that a punktfunk-host ASSUMPTION broke, not as damage. `NonZeroReorder` +//! in particular fires on the AU that ACTIVATES an SPS — the opening IDR, and the +//! fresh IDR at every ABR resolution change — so treating it as concealment would +//! cost a released-unshown frame plus a keyframe round trip at every +//! renegotiation, on a stream the planner says it planned correctly. +//! +//! It lives here, in the crate the warnings are re-exported from, rather than in +//! the client that first needed it, because the fault-injection harness +//! ([`crate::fault`]) has to assert against the SAME predicate the client conceals +//! on. Two copies of this list would let a test prove detection that production +//! does not actually perform — the exact shape of the `nb_queries = 0` failure the +//! program exists to end. + +use crate::{H265PlanWarning, PlanWarning}; + +/// Does this H.264 planner warning mean the PICTURE is damaged? +/// +/// `Mmco5Rebase` does not: the AU carried an MMCO 5 and pf-bitstream planned it in +/// full (the plan holds the pre-rebase 8.2.1 values; later AUs reference the +/// rebased ones). +/// +/// Written as an EXHAUSTIVE match with no wildcard, deliberately. A `matches!` (or +/// a `_ => false`) makes "damage" the opt-in and silence the default, so a +/// `PlanWarning` added later — by definition one nobody here has classified — +/// would be reported as clean and its picture shown. Invisible damage is the bug +/// this whole program exists to end; the compiler is the only reviewer guaranteed +/// to be present when that variant is written, so it gets the decision. +pub fn is_integrity_warning(w: &PlanWarning) -> bool { + match w { + PlanWarning::FrameNumGap { .. } + | PlanWarning::MissingReference { .. } + | PlanWarning::TruncatedAu { .. } => true, + PlanWarning::Mmco5Rebase => false, + } +} + +/// The H.265 twin — the same set pf-bitstream's own `h265` conformance harness +/// calls integrity, `NonZeroReorder` deliberately excluded (module docs). +/// +/// Exhaustive for the same reason as [`is_integrity_warning`]: a new H.265 warning +/// must not be able to mean "damaged" and read as clean. +pub fn is_integrity_warning_h265(w: &H265PlanWarning) -> bool { + match w { + H265PlanWarning::MissingReference { .. } | H265PlanWarning::TruncatedAu { .. } => true, + H265PlanWarning::NonZeroReorder { .. } => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The split, stated as the pair of lists it is. This test is the contract: + /// the client drops a frame for everything on the left and shows the picture + /// for everything on the right, and the fault harness asserts detection + /// against the very same predicate. + #[test] + fn damage_is_a_lost_reference_or_a_short_au_and_nothing_else() { + for w in [ + PlanWarning::FrameNumGap { + expected: 4, + got: 7, + }, + PlanWarning::MissingReference { + context: "list0", + detail: "poc 12".into(), + }, + PlanWarning::TruncatedAu { offset: 900 }, + ] { + assert!(is_integrity_warning(&w), "{w:?} is damage"); + } + assert!( + !is_integrity_warning(&PlanWarning::Mmco5Rebase), + "an MMCO 5 was planned in FULL — dropping its frame would hitch a \ + correct stream" + ); + + for w in [ + H265PlanWarning::MissingReference { + context: "StCurrBefore", + detail: "poc 12".into(), + }, + H265PlanWarning::TruncatedAu { offset: 900 }, + ] { + assert!(is_integrity_warning_h265(&w), "{w:?} is damage"); + } + assert!( + !is_integrity_warning_h265(&H265PlanWarning::NonZeroReorder { + max_num_reorder_pics: 1 + }), + "SPS activation is not damage — it fires on the opening IDR and on \ + every ABR renegotiation's IDR" + ); + } +} diff --git a/crates/pf-vkdecode/src/lib.rs b/crates/pf-vkdecode/src/lib.rs index 43aa66e4..7895ad86 100644 --- a/crates/pf-vkdecode/src/lib.rs +++ b/crates/pf-vkdecode/src/lib.rs @@ -65,6 +65,27 @@ //! - [`decoder_h265`]: [`VkH265Decoder`], mirroring [`VkH264Decoder`]'s public //! surface method-for-method. Codec DISPATCH is the client wiring's job. //! +//! M4 (status and telemetry) — three pure modules turning the signals above into +//! something a session, a user and a support engineer can act on: +//! +//! - [`recovery`]: [`RecoveryWatch`], the recovery point SEI folded into a +//! per-picture "the stream healed HERE" mark ([`RecoveryMark`], carried on +//! [`DecodedVkFrame::recovery`]). The only clean point an intra-refresh session +//! has — its wave emits no IDR — so without it a client freezes for its full +//! backstop and then forces the very IDR the wave exists to avoid. +//! - [`integrity`]: [`is_integrity_warning`] / [`is_integrity_warning_h265`], the +//! one list of warnings that mean the PICTURE is damaged. Here rather than in the +//! client so the fault harness asserts against the predicate production conceals +//! on. +//! - [`fault`]: [`AuFault`], deliberate decoder-input corruption +//! (`PUNKTFUNK_AU_FAULT`), inert unless armed. A detector nobody can fire is +//! exactly as trustworthy as no detector at all. +//! +//! Plus [`VkH264Decoder::status_queries`] / [`VkH265Decoder::status_queries`]: does +//! this device answer per-op `RESULT_STATUS` at all? Without that fact a clean +//! integrity report cannot be told apart from an unmeasured one — which is the +//! precise failure the program exists to end. +//! //! Unsafe posture: unlike pf-bitstream (which forbids unsafe outright), this crate //! cannot — the `ash::vk::native` bindgen structs are zero-initialized the way the //! encode side does it (`pf-encode/src/enc/linux/vk_build.rs`), and the GPU half is @@ -78,11 +99,14 @@ pub mod caps_h265; pub mod decoder; pub mod decoder_h265; pub mod device; +pub mod fault; pub mod images; +pub mod integrity; pub mod params; pub mod params_h265; pub mod pic; pub mod pic_h265; +pub mod recovery; pub mod ring; pub mod session; pub mod session_h265; @@ -137,9 +161,15 @@ pub use device::DeviceHandles; pub use device::NoopQueueLock; pub use device::QueueLock; pub use device::QueueSubmitGuard; +pub use fault::AuFault; +pub use fault::FaultAction; +pub use fault::FaultMode; +pub use fault::DEFAULT_FAULT_PERIOD; pub use images::plan_pools; pub use images::PoolPlan; pub use images::HOLD_HEADROOM; +pub use integrity::is_integrity_warning; +pub use integrity::is_integrity_warning_h265; pub use params::pps_to_std; pub use params::sps_to_std; pub use params::OwnedStdPps; @@ -162,6 +192,8 @@ pub use pic_h265::DecodePlanVkH265; pub use pic_h265::PlanToVkH265Error; pub use pic_h265::VkRefH265; pub use pic_h265::H265_RPS_LIST_SIZE; +pub use recovery::RecoveryMark; +pub use recovery::RecoveryWatch; pub use ring::RingLayout; pub use session::ParamsAction; pub use session::SessionConfig; diff --git a/crates/pf-vkdecode/src/recovery.rs b/crates/pf-vkdecode/src/recovery.rs new file mode 100644 index 00000000..5578897e --- /dev/null +++ b/crates/pf-vkdecode/src/recovery.rs @@ -0,0 +1,634 @@ +//! The recovery-point watch: turning the recovery point SEI into a per-picture +//! "the stream is healed HERE" mark (M4 of the native-decode program). +//! +//! # Why this exists +//! +//! A host running an **intra-refresh wave** never emits an IDR: a moving band of +//! intra blocks re-codes the whole picture over ~half a second, and loss self-heals +//! as the band sweeps. That is strictly better for a stream (no 20-40× IDR spike +//! under loss) — but it leaves the CLIENT with no decoder-visible clean point. +//! libavcodec sets `AV_FRAME_FLAG_KEY` only for true IDRs (H.264 flags key when +//! `recovery_frame_cnt == 0`, HEVC only on IRAP), so a client on the FFmpeg rungs +//! sees a healed picture as an unbroken run of ordinary P-frames. Its post-loss +//! freeze therefore holds the last good frame until the shared gate's +//! `REANCHOR_FREEZE_MAX` backstop (punktfunk-core's `reanchor` module) fires and +//! forces the very IDR intra-refresh exists to avoid — half a second of frozen +//! picture on a stream that was already clean. +//! +//! The host CAN say so on the wire (`USER_FLAG_RECOVERY_POINT`), and where it does +//! the shared gate lifts on the second mark. But exactly ONE encoder backend sets +//! it: pf-encode's Linux libav-NVENC, under the `PUNKTFUNK_INTRA_REFRESH` opt-in +//! (`EncoderCaps::intra_refresh_recovery`). The other two backends that actually +//! run a wave — Windows AMF and QSV — leave it `false` pending on-glass GDR +//! validation, so THEIR intra-refresh sessions have no wire clean point at all and +//! ride the backstop on every loss. The flag also cannot help a host that predates +//! it, and it dies with the datagram that carried it. +//! +//! The bitstream, meanwhile, says it directly, and says it in a place loss cannot +//! separate from the picture it describes: an NVENC-family encoder emits a +//! **recovery point SEI** at the start of each wave, naming the picture at which +//! output becomes correct. (Not universal — pf-encode records that AMF emits none, +//! so an AMD Windows wave stays invisible either way; this is an overlapping set +//! with the wire flag's, not a superset.) pf-bitstream parses it for both codecs +//! already ([`pf_bitstream::h264::RecoveryPoint`], +//! [`pf_bitstream::h265::RecoveryPointHevc`]) and carries it on every +//! `AuPlan::picture`. This module is the piece that was missing: the small state +//! machine that remembers an outstanding recovery point and marks the picture that +//! reaches it, so the client can observe its own heal instead of waiting one out. +//! +//! It is an ADDITIONAL, independent signal. Nothing here touches the wire flags; +//! a stream with no recovery point SEI produces no marks and behaves exactly as it +//! did before. +//! +//! # The counting rules, and why they are conservative +//! +//! The two codecs count the distance to the recovery point in different units, so +//! the watch keeps two `note_*` entries rather than pretending one unit fits both: +//! +//! * **H.265** (D.3.8) counts in PICTURE ORDER: `recovery_poc_cnt` is a signed POC +//! delta from the SEI's picture to the recovery point. That is exact arithmetic +//! on a number every plan already carries, so [`RecoveryWatch::note_h265`] simply +//! remembers `poc + recovery_poc_cnt` and marks the first picture at or past it. +//! A NEGATIVE count (a recovery point among leading pictures) marks the SEI's own +//! picture, which is what "at or past" means when the target is behind us. +//! +//! * **H.264** (D.2.8) counts in `frame_num` INCREMENTS, and `frame_num` advances +//! only for reference pictures, so there is no fixed picture-to-increment ratio. +//! [`RecoveryWatch::note_h264`] therefore counts increments the only way a +//! consumer honestly can: a picture whose `frame_num` DIFFERS from the previous +//! one's spends exactly one increment. That under-counts across a `frame_num` GAP +//! (a lost reference frame skips several increments but is charged one), which +//! makes the watch mark the recovery point LATE, never early — and late is the +//! behaviour the client already has today (the backstop). A gap also re-arms the +//! client's freeze and normally brings a fresh SEI with it, so the residue is a +//! heal reported one wave later, not a stale picture presented as clean. +//! +//! Both entries drop an outstanding watch at an IDR/IRAP: a real keyframe is a +//! whole re-anchor by itself, and its POC/`frame_num` reset would make any pending +//! target meaningless. +//! +//! # An SEI is a wave START only when its target ADVANCES +//! +//! [`RecoveryMark::sei_here`] is not "this AU carried a recovery point SEI"; it is +//! "an intra-refresh wave STARTS here", and the difference decides whether a +//! consumer may trust the mark that follows. D.2.8/D.3.8 both permit re-announcing +//! the CURRENT wave's recovery point on every picture with a shrinking count, and +//! that is exactly what x264's `--intra-refresh` emits. Under a "any SEI is a new +//! wave" reading the first picture after a loss carries a fresh-looking SEI whose +//! target is the end of a wave that began BEFORE the loss — so the consumer's +//! arm-pairing lifts on a picture whose already-swept stripes still reference the +//! lost frame, presenting a partially stale picture as clean. That is precisely +//! what the wire path's `REANCHOR_MARKS_TO_LIFT = 2` exists to prevent, and the +//! local path must not be the weaker of the two. +//! +//! So an SEI is honoured as a wave start only when the target it implies lies +//! BEYOND any outstanding one (H.265: `poc + recovery_poc_cnt` strictly greater +//! than the outstanding `Target::Poc`; H.264: `recovery_frame_cnt` strictly greater +//! than the increments still owed, both measured from the same picture). Anything +//! else is a re-announcement: the further target stands and no wave start is +//! reported. The cost is that a genuinely NEW wave which is shorter than the +//! remainder of the one in flight goes unreported — a heal missed, falling back to +//! the consumer's backstop — which is the same direction every other rule here +//! errs in: late, never early. +//! +//! # What the watch deliberately does NOT decide +//! +//! Whether a mark may LIFT a post-loss freeze is the client's decision, not this +//! module's, because it depends on something no bitstream carries: when the loss +//! happened. A recovery point whose SEI arrived BEFORE the loss guarantees nothing +//! — the wave's already-swept stripes still reference the picture that was lost — +//! so the consumer must require an SEI observed at or after the loss. That is why +//! [`RecoveryMark`] reports the SEI and the recovery point as two separate facts: +//! the client pairs them against its own arm. + +use pf_bitstream::h264::RecoveryPoint; +use pf_bitstream::h265::RecoveryPointHevc; + +/// What one planned picture is worth to the recovery-point watch. +/// +/// Two independent booleans rather than one enum because a single picture can be +/// both: an encoder that emits a recovery point SEI with a zero count is saying +/// "this very picture is the clean point", and a consumer pairing SEI-then-mark +/// must see both facts on it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct RecoveryMark { + /// The AU that produced this picture CARRIED a recovery point SEI — an + /// intra-refresh wave starts (or restarts) here. The consumer uses it to + /// decide that a later [`Self::is_recovery_point`] is about a wave that began + /// after ITS loss, which is the only case a mark may be trusted in. + pub sei_here: bool, + /// This picture IS the recovery point an outstanding SEI named: decoding from + /// that SEI's AU onward, this picture's output is correct. + pub is_recovery_point: bool, +} + +impl RecoveryMark { + /// Nothing to report — the overwhelmingly common per-picture answer. + pub const NONE: RecoveryMark = RecoveryMark { + sei_here: false, + is_recovery_point: false, + }; +} + +/// The outstanding recovery point, in the codec's own counting unit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Target { + /// H.264: increments of `frame_num` still owed, plus the `frame_num` the last + /// charged picture carried (an increment is "this picture's differs"). + FrameNumIncrements { owed: u32, last_frame_num: u16 }, + /// H.265: the absolute `PicOrderCntVal` at or past which output is correct. + Poc(i32), +} + +/// One decoder's outstanding recovery point. Pure — no Vulkan, no allocation, two +/// words of state — so the whole rule set below is CPU-testable. +#[derive(Debug, Clone, Copy, Default)] +pub struct RecoveryWatch { + target: Option, +} + +impl RecoveryWatch { + pub fn new() -> RecoveryWatch { + RecoveryWatch { target: None } + } + + /// Is a recovery point still outstanding? (Diagnostics and tests; the decode + /// path reads the per-picture [`RecoveryMark`] instead.) + pub fn is_watching(&self) -> bool { + self.target.is_some() + } + + /// Fold one planned H.264 picture. `frame_num` and `is_idr` come off the + /// plan's `PicturePlan`, `sei` is its `recovery_point` field. + /// + /// Order matters and is deliberate: the outstanding target is CHARGED for this + /// picture first, then a NEW SEI replaces it. So a wave-start AU that also + /// completes the previous wave reports both facts, and the fresh target is not + /// charged for the picture that announced it (D.2.8 counts increments *from* + /// the SEI's picture, exclusive). + /// + /// "New" is load-bearing — see [`Self::starts_a_new_wave_h264`]. + pub fn note_h264( + &mut self, + frame_num: u16, + is_idr: bool, + sei: Option, + ) -> RecoveryMark { + if is_idr { + // A real keyframe is a whole re-anchor and resets `frame_num`; any + // pending count is meaningless past it. The client lifts on the IDR + // itself, so nothing is lost by dropping the watch here. + self.target = None; + } + let mut mark = RecoveryMark::NONE; + // How many increments the OUTSTANDING wave still owes, measured from THIS + // picture (0 = it is reached here); `None` when no wave was outstanding + // when this picture arrived. It is the yardstick a fresh SEI is judged + // against below, so it has to be captured while charging. + let mut owed_here: Option = None; + if let Some(Target::FrameNumIncrements { + owed, + last_frame_num, + }) = self.target + { + // One increment per picture whose `frame_num` differs from the last + // charged one — the honest, conservative reading (module docs). + let owed = if frame_num != last_frame_num { + owed.saturating_sub(1) + } else { + owed + }; + owed_here = Some(owed); + if owed == 0 { + mark.is_recovery_point = true; + self.target = None; + } else { + self.target = Some(Target::FrameNumIncrements { + owed, + last_frame_num: frame_num, + }); + } + } + if let Some(rp) = sei { + if Self::starts_a_new_wave_h264(owed_here, rp.recovery_frame_cnt) { + mark.sei_here = true; + if rp.recovery_frame_cnt == 0 { + // "Start here and this picture is already exact" — the SEI's + // own picture is the recovery point, no waiting. + mark.is_recovery_point = true; + self.target = None; + } else { + self.target = Some(Target::FrameNumIncrements { + owed: rp.recovery_frame_cnt, + last_frame_num: frame_num, + }); + } + } + // A RE-ANNOUNCEMENT changes nothing: the outstanding target the charge + // above left in place is the FURTHER one, and keeping it is what makes + // the mark land late rather than early (module docs). + } + mark + } + + /// Does an H.264 SEI announce a wave that STARTS here, or merely re-announce + /// the one already outstanding? + /// + /// D.2.8 permits — and x264's `--intra-refresh` does — re-emitting the current + /// wave's recovery point on every picture with a DECREASING + /// `recovery_frame_cnt`. Under a "any SEI is a new wave" reading, the first + /// picture after a loss then carries a fresh-looking SEI whose target is the + /// end of a wave that began BEFORE the loss, and the consumer's arm-pairing + /// ([`RecoveryMark::sei_here`]) lifts its freeze on a picture whose + /// already-swept stripes still reference the lost frame: a partially stale + /// picture presented as clean, the one outcome the pairing exists to prevent. + /// + /// So a new wave is one whose target lies BEYOND the outstanding one. Both + /// counts are increments measured from this picture (the outstanding one after + /// this picture's charge), so they compare directly. With no wave outstanding + /// — the ordinary case, and everything after an IDR — every SEI is new. + fn starts_a_new_wave_h264(owed_here: Option, recovery_frame_cnt: u32) -> bool { + match owed_here { + Some(owed) => recovery_frame_cnt > owed, + None => true, + } + } + + /// Fold one planned H.265 picture — `pic_order_cnt` and `is_irap` off the + /// plan's `PicturePlan`, `sei` its `recovery_point`. + /// + /// `is_irap` rather than `is_idr`: a CRA/BLA also restarts the stream's + /// prediction structure, and the POC target that predates it cannot be + /// compared against the POCs that follow. + /// + /// As on the H.264 side, only an SEI whose target ADVANCES past the + /// outstanding one starts a new wave — the arithmetic here is exact, so the + /// test is `poc + delta` strictly greater than the outstanding `Target::Poc` + /// (see [`Self::starts_a_new_wave_h264`] for why). + pub fn note_h265( + &mut self, + pic_order_cnt: i32, + is_irap: bool, + sei: Option, + ) -> RecoveryMark { + if is_irap { + self.target = None; + } + let mut mark = RecoveryMark::NONE; + // The outstanding target as this picture ARRIVED — the yardstick a fresh + // SEI is judged against, captured before the charge below can clear it. + let outstanding = match self.target { + Some(Target::Poc(target)) => Some(target), + _ => None, + }; + if let Some(target) = outstanding { + if pic_order_cnt >= target { + mark.is_recovery_point = true; + self.target = None; + } + } + if let Some(rp) = sei { + // D.3.8: the target is this picture's POC plus the signed delta. A + // delta of 0 (or negative — a recovery point among leading pictures) + // lands at or behind this picture, so it is reached immediately. + let target = pic_order_cnt.saturating_add(rp.recovery_poc_cnt); + if outstanding.is_none_or(|t| target > t) { + mark.sei_here = true; + if pic_order_cnt >= target { + mark.is_recovery_point = true; + self.target = None; + } else { + self.target = Some(Target::Poc(target)); + } + } + // Otherwise it re-announces the wave already outstanding (or an + // earlier point within it): the further target the charge left in + // place stands, and this picture reports no fresh wave start. + } + mark + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn h264_sei(recovery_frame_cnt: u32) -> Option { + Some(RecoveryPoint { + recovery_frame_cnt, + // The realistic value for a rolling wave: an encoder that filters + // across the refresh boundary cannot promise bit-exact output, and + // says so. The watch must not care — see + // `an_approximate_recovery_point_still_marks`. + exact_match: false, + broken_link: false, + }) + } + + fn h265_sei(recovery_poc_cnt: i32) -> Option { + Some(RecoveryPointHevc { + recovery_poc_cnt, + exact_match: false, + broken_link: false, + }) + } + + /// The wave: an SEI announces a recovery point N increments out, and exactly + /// the Nth picture after it is marked — not the ones before, not the ones + /// after. This is the whole point of the module, on the codec whose counting + /// unit is the awkward one. + #[test] + fn an_h264_wave_marks_the_picture_the_sei_counted_to() { + let mut w = RecoveryWatch::new(); + // The wave starts on frame_num 10 and is three increments long. + let start = w.note_h264(10, false, h264_sei(3)); + assert!(start.sei_here, "the wave start is reported"); + assert!( + !start.is_recovery_point, + "a count of 3 is not reached on the picture that announced it" + ); + assert!(w.is_watching()); + assert_eq!(w.note_h264(11, false, None), RecoveryMark::NONE); + assert_eq!(w.note_h264(12, false, None), RecoveryMark::NONE); + let healed = w.note_h264(13, false, None); + assert!(healed.is_recovery_point, "the third increment is the heal"); + assert!(!healed.sei_here); + assert!(!w.is_watching(), "and the watch is spent"); + // Nothing after it is marked — a mark is a moment, not a state. + assert_eq!(w.note_h264(14, false, None), RecoveryMark::NONE); + } + + /// `frame_num` does not advance for a non-reference picture, and D.2.8 counts + /// INCREMENTS — so a repeated `frame_num` must not be charged, or the mark + /// lands early on a picture the wave has not reached. + #[test] + fn a_repeated_frame_num_spends_no_increment() { + let mut w = RecoveryWatch::new(); + w.note_h264(4, false, h264_sei(2)); + // Two pictures at the same frame_num: one increment, not two. + assert_eq!(w.note_h264(5, false, None), RecoveryMark::NONE); + assert_eq!(w.note_h264(5, false, None), RecoveryMark::NONE); + assert!( + w.note_h264(6, false, None).is_recovery_point, + "the second true increment completes the count" + ); + } + + /// A `frame_num` GAP (the lost reference frame this whole program exists for) + /// skips several increments but is charged one, so the mark can only land + /// LATE. Late is today's behaviour (the freeze backstop); early would present + /// a half-swept picture as clean, which is the one outcome that must be + /// impossible. + #[test] + fn a_frame_num_gap_delays_the_mark_it_never_advances_it() { + let mut w = RecoveryWatch::new(); + w.note_h264(100, false, h264_sei(2)); + // frame_num jumps 101 → 104: three real increments, charged as one. + assert_eq!(w.note_h264(104, false, None), RecoveryMark::NONE); + assert!( + w.note_h264(105, false, None).is_recovery_point, + "the count still has to be walked off — never short-circuited" + ); + } + + /// `recovery_frame_cnt == 0` means "start decoding here; this picture is + /// already exact". Both facts land on the one picture. + #[test] + fn a_zero_count_marks_the_pictures_that_carries_the_sei() { + let mut w = RecoveryWatch::new(); + let m = w.note_h264(7, false, h264_sei(0)); + assert!(m.sei_here && m.is_recovery_point); + assert!(!w.is_watching()); + } + + /// A wave that reaches PAST the outstanding one supersedes it: the newest SEI + /// is the encoder's current statement about where the picture becomes correct, + /// and its count is measured from ITS own picture. + #[test] + fn a_new_sei_that_reaches_further_replaces_an_outstanding_watch() { + let mut w = RecoveryWatch::new(); + w.note_h264(1, false, h264_sei(2)); + // A second, LONGER wave two pictures in: 4 increments from here, against + // the 1 the first wave still owed. + let restart = w.note_h264(3, false, h264_sei(4)); + assert!(restart.sei_here, "it reaches further — a fresh wave starts"); + assert!(!restart.is_recovery_point); + for fnum in 4..7 { + assert_eq!(w.note_h264(fnum, false, None), RecoveryMark::NONE); + } + assert!( + w.note_h264(7, false, None).is_recovery_point, + "the SECOND wave's count is what completes, not the first's" + ); + } + + /// The re-announcement rule, on the codec whose counting unit is the awkward + /// one. x264's `--intra-refresh` re-emits the CURRENT wave's recovery point on + /// every picture with a DECREASING `recovery_frame_cnt` (legal under D.2.8). + /// Read as a wave START, every one of those would let a consumer that armed a + /// freeze mid-wave lift on the tail of a wave that began BEFORE its loss — a + /// half-stale picture presented as clean. + #[test] + fn a_decreasing_re_announcement_is_the_same_wave_not_a_new_one() { + let mut w = RecoveryWatch::new(); + // The wave starts on frame_num 10, four increments long, and re-announces + // itself on every picture: 4, 3, 2, 1, 0. + let start = w.note_h264(10, false, h264_sei(4)); + assert!(start.sei_here, "the FIRST announcement is a wave start"); + for (fnum, cnt) in [(11, 3), (12, 2), (13, 1)] { + let m = w.note_h264(fnum, false, h264_sei(cnt)); + assert!( + !m.sei_here, + "frame_num {fnum} re-announces the same wave — not a fresh start" + ); + assert!(!m.is_recovery_point, "and the wave is not there yet"); + } + // The wave completes, and the trailing `recovery_frame_cnt == 0` on that + // very picture must NOT read as a brand-new wave either: a consumer that + // armed mid-wave would otherwise see sei_here + is_recovery_point on one + // picture and lift on the wave it already discounted. + let healed = w.note_h264(14, false, h264_sei(0)); + assert!(healed.is_recovery_point, "the wave really did complete"); + assert!( + !healed.sei_here, + "…but nothing NEW started here — the count only walked to zero" + ); + assert!(!w.is_watching()); + } + + /// The H.265 twin, in exact POC arithmetic: a wave announced at POC 10 for POC + /// 20, re-announced on every picture with the same absolute target. + #[test] + fn an_h265_re_announcement_of_the_same_target_is_not_a_new_wave() { + let mut w = RecoveryWatch::new(); + assert!(w.note_h265(10, false, h265_sei(10)).sei_here); + for poc in 11..20 { + let m = w.note_h265(poc, false, h265_sei(20 - poc)); + assert!(!m.sei_here, "poc {poc} re-announces target 20"); + assert!(!m.is_recovery_point); + } + let healed = w.note_h265(20, false, h265_sei(0)); + assert!(healed.is_recovery_point); + assert!(!healed.sei_here, "the target did not advance past 20"); + // A genuinely later wave DOES start, and is reported. + assert!(w.note_h265(21, false, h265_sei(8)).sei_here); + } + + /// An SEI that reaches SHORT of the outstanding target is a re-announcement + /// too, and the further target is what stands — the module's "late, never + /// early" rule. (A real new-but-shorter wave therefore goes unreported: one + /// heal missed, falling back to the consumer's backstop, which is the safe + /// side of this trade.) + #[test] + fn an_sei_that_reaches_short_of_the_outstanding_target_never_shortens_the_watch() { + let mut w = RecoveryWatch::new(); + w.note_h265(0, false, h265_sei(20)); // target 20 + let short = w.note_h265(1, false, h265_sei(2)); // would target 3 + assert!(!short.sei_here); + assert_eq!(w.note_h265(3, false, None), RecoveryMark::NONE, "not at 20"); + assert!(w.note_h265(20, false, None).is_recovery_point); + + let mut w = RecoveryWatch::new(); + w.note_h264(0, false, h264_sei(6)); + assert!(!w.note_h264(1, false, h264_sei(2)).sei_here); + // The original six increments are still what has to be walked off. + for fnum in 2..6 { + assert_eq!(w.note_h264(fnum, false, None), RecoveryMark::NONE); + } + assert!(w.note_h264(6, false, None).is_recovery_point); + } + + /// An IDR/IRAP clears the outstanding target, so the very next SEI is a wave + /// start again however small its count — the re-announcement rule must not + /// leave a stream unable to report a fresh wave after a keyframe. + #[test] + fn a_keyframe_makes_the_next_sei_a_wave_start_again() { + let mut w = RecoveryWatch::new(); + w.note_h264(10, false, h264_sei(9)); + w.note_h264(0, true, None); // IDR + assert!( + w.note_h264(1, false, h264_sei(1)).sei_here, + "a one-increment wave after an IDR is still a fresh wave" + ); + + let mut w = RecoveryWatch::new(); + w.note_h265(100, false, h265_sei(50)); + w.note_h265(0, true, None); // IRAP + assert!(w.note_h265(1, false, h265_sei(2)).sei_here); + } + + /// An IDR is a whole re-anchor and resets `frame_num`; a target counted + /// against the old numbering must not survive it. + #[test] + fn an_idr_drops_a_pending_h264_watch() { + let mut w = RecoveryWatch::new(); + w.note_h264(200, false, h264_sei(2)); + assert!(w.is_watching()); + let idr = w.note_h264(0, true, None); + assert_eq!( + idr, + RecoveryMark::NONE, + "the IDR is not a recovery-point mark" + ); + assert!(!w.is_watching()); + // The pictures after it are ordinary — no stale mark fires. + assert_eq!(w.note_h264(1, false, None), RecoveryMark::NONE); + assert_eq!(w.note_h264(2, false, None), RecoveryMark::NONE); + } + + /// H.265 counts in POC, which is exact arithmetic — the target is hit at or + /// past the announced value even when POC steps by more than one. + #[test] + fn an_h265_wave_marks_the_first_picture_at_or_past_the_target_poc() { + let mut w = RecoveryWatch::new(); + let start = w.note_h265(10, false, h265_sei(4)); + assert!(start.sei_here && !start.is_recovery_point); + assert_eq!(w.note_h265(11, false, None), RecoveryMark::NONE); + assert_eq!(w.note_h265(13, false, None), RecoveryMark::NONE); + // POC 14 is the target exactly. + assert!(w.note_h265(14, false, None).is_recovery_point); + assert!(!w.is_watching()); + + // …and a stream whose POC steps OVER the target still marks: "at or past" + // is the rule, because nothing guarantees the exact value is coded. + let mut w = RecoveryWatch::new(); + w.note_h265(0, false, h265_sei(3)); + assert!(w.note_h265(8, false, None).is_recovery_point); + } + + /// `recovery_poc_cnt` is se(v)-coded and may be negative (a recovery point + /// among leading pictures) — the target is then behind us and the SEI's own + /// picture is already the clean one. It must not become an infinite watch. + #[test] + fn a_negative_or_zero_poc_count_marks_immediately() { + for count in [0, -1, -7] { + let mut w = RecoveryWatch::new(); + let m = w.note_h265(30, false, h265_sei(count)); + assert!(m.sei_here && m.is_recovery_point, "count {count}"); + assert!(!w.is_watching(), "count {count}"); + } + } + + /// Any IRAP — not just an IDR — restarts prediction and re-bases POC, so a + /// target counted against the previous numbering cannot be compared past it. + #[test] + fn an_irap_drops_a_pending_h265_watch() { + let mut w = RecoveryWatch::new(); + w.note_h265(500, false, h265_sei(3)); + assert!(w.is_watching()); + assert_eq!(w.note_h265(0, true, None), RecoveryMark::NONE); + assert!(!w.is_watching()); + assert_eq!(w.note_h265(1, false, None), RecoveryMark::NONE); + } + + /// `exact_match_flag == 0` is the NORMAL value for a rolling wave (loop + /// filtering bleeds across the refresh boundary, so the encoder promises + /// approximate rather than bit-exact output). Requiring exactness would make + /// this whole module never fire on the streams it was written for — and an + /// approximately-correct picture is not the failure mode the freeze exists to + /// hide, which is a gray plate with motion painted on it. + #[test] + fn an_approximate_recovery_point_still_marks() { + let mut w = RecoveryWatch::new(); + assert!( + w.note_h264( + 1, + false, + Some(RecoveryPoint { + recovery_frame_cnt: 0, + exact_match: false, + broken_link: true, + }) + ) + .is_recovery_point, + "neither exact_match nor broken_link may veto the mark" + ); + let mut w = RecoveryWatch::new(); + assert!( + w.note_h265( + 1, + false, + Some(RecoveryPointHevc { + recovery_poc_cnt: 0, + exact_match: false, + broken_link: true, + }) + ) + .is_recovery_point + ); + } + + /// A stream with no recovery point SEI at all — every punktfunk host today — + /// produces no marks whatsoever. The signal is purely additive. + #[test] + fn a_stream_without_recovery_point_seis_never_marks() { + let mut w = RecoveryWatch::new(); + for n in 0..64u16 { + assert_eq!(w.note_h264(n, n == 0, None), RecoveryMark::NONE); + } + let mut w = RecoveryWatch::new(); + for n in 0..64i32 { + assert_eq!(w.note_h265(n, n == 0, None), RecoveryMark::NONE); + } + } +} diff --git a/crates/pf-vkdecode/tests/fault_detection.rs b/crates/pf-vkdecode/tests/fault_detection.rs new file mode 100644 index 00000000..5a776c3b --- /dev/null +++ b/crates/pf-vkdecode/tests/fault_detection.rs @@ -0,0 +1,447 @@ +//! Fault injection proves detection — M4's exit criterion, on the CPU. +//! +//! The native-decode program exists because a field corruption was +//! ARCHITECTURALLY undetectable: FFmpeg's Vulkan decoder creates no status queries, +//! never sets `AV_FRAME_FLAG_CORRUPT`, and reports trouble only as log lines. The +//! native lane now has the signals. This harness fires them on purpose, so +//! "detection works" is a test result rather than a belief. +//! +//! It drives the REAL detection path minus the GPU: [`pf_vkdecode::AuFault`] — the +//! very injector the client's `PUNKTFUNK_AU_FAULT` knob arms — damages a real +//! 250-AU host-shaped stream, the damaged AUs go through the planner exactly as +//! `VkH264Decoder::decode` / `VkH265Decoder::decode` feed it, and the verdict is +//! read with [`pf_vkdecode::is_integrity_warning`] / +//! [`pf_vkdecode::is_integrity_warning_h265`] — the same predicates the client +//! conceals on. Nothing is mocked; what is absent is only the Vulkan submission +//! below the planner, which cannot change a plan's warnings. +//! +//! **Both codecs**, because they detect a lost AU through genuinely different +//! predicates and a harness covering one proves nothing about the other. H.264 has +//! a `frame_num` gap — an explicit, cheap counter break that fires whether or not +//! the missing picture is ever referenced. HEVC has no analogue at all (POC is +//! coded per picture and simply jumps), so its entire drop detection rests on the +//! RPS resolving a reference the DPB does not hold: `MissingReference` or nothing. +//! HEVC ships in this milestone, so it is tested here on the same terms. +//! +//! What it therefore proves, and — just as deliberately — what it proves is +//! IMPOSSIBLE here: +//! +//! * **Proved**: a dropped AU is DETECTED as *integrity* damage within a bounded +//! number of AUs, on both codecs — the class that makes the client release the +//! picture unshown and ask for a re-anchor. That is M4's exit criterion for the +//! parser-visible half. +//! * **Proved**: the clean stream produces NO integrity warning anywhere, so the +//! detector cannot be passing by firing on everything (the failure mode that +//! would cost a keyframe round trip per second on healthy links). +//! * **Proved impossible**: truncation and payload flips are INVISIBLE to the +//! parser. Annex-B carries no NALU length, so a cut slice is just a shorter +//! slice, and a flipped payload byte is syntactically perfect. Both decode to a +//! wrong picture with nothing in the bitstream to object to. The only detector +//! left is the driver's per-op `RESULT_STATUS` verdict, which needs real hardware +//! — the GPU smoke/parity tests' ground. These assertions are the negative space +//! that justifies the status ring existing at all, and the reason a session on a +//! driver without `queryResultStatusSupport` must be reported as unmeasured +//! rather than clean. +//! * **Proved impossible, and correctly so**: a dropped SUB-LAYER NON-REFERENCE +//! picture is invisible to the planner on either codec, because nothing ever +//! references it. That is not a hole in detection — no later picture is damaged +//! — it is a missing OUTPUT frame, which is the wire's frame-index gap detector's +//! job (`punktfunk_core::reanchor::index_gap`), not the bitstream's. The HEVC +//! vector below contains both kinds and the test asserts both verdicts, so the +//! distinction cannot quietly become "HEVC misses drops". + +use pf_bitstream::h264::H264Planner; +use pf_bitstream::h265::H265Planner; +use pf_vkdecode::{ + is_integrity_warning, is_integrity_warning_h265, AuFault, FaultAction, FaultMode, +}; +use std::io::Cursor; + +/// The same vendored vectors the WP-A conversion tests and the GPU tests use: 250 +/// AUs of real encoder output each, an IDR then P-frames — the punktfunk +/// envelope's shape, in both codecs. +const TEST_25FPS_H264: &[u8] = include_bytes!( + "../../pf-bitstream/vendor/cros-codecs/src/codec/h264/test_data/test-25fps.h264" +); +const TEST_25FPS_H265: &[u8] = include_bytes!( + "../../pf-bitstream/vendor/cros-codecs/src/codec/h265/test_data/test-25fps.h265" +); + +/// Test-only H.264 AU splitter, mirroring pf-bitstream's (`#[cfg(test)]`-private +/// there) and the GPU tests'. +fn split_h264(stream: &[u8]) -> Vec<&[u8]> { + use cros_codecs::codec::h264::parser::{Nalu, NaluType}; + let mut aus = Vec::new(); + let mut cursor = Cursor::new(stream); + let mut au_start = 0usize; + let mut au_has_slice = false; + + while let Ok(nalu) = Nalu::next(&mut cursor) { + let nalu_offset = cursor.position() as usize; + let start = nalu_offset - nalu.offset; + let is_slice = matches!(nalu.header.type_, NaluType::Slice | NaluType::SliceIdr); + let first_mb_zero = is_slice && stream.get(nalu_offset + 1).is_some_and(|b| b & 0x80 != 0); + + if au_has_slice && (!is_slice || first_mb_zero) { + aus.push(&stream[au_start..start]); + au_start = start; + au_has_slice = false; + } + au_has_slice |= is_slice; + } + aus.push(&stream[au_start..]); + aus +} + +/// The H.265 twin, mirroring `pic_h265`'s test splitter: a new AU starts at a +/// non-VCL NALU following slices, or at a slice segment whose +/// `first_slice_segment_in_pic_flag` is set (the first bit of the byte after the +/// 2-byte NAL header) when the current AU already has slices. +fn split_h265(stream: &[u8]) -> Vec<&[u8]> { + use cros_codecs::codec::h265::parser::Nalu; + let mut aus = Vec::new(); + let mut cursor = Cursor::new(stream); + let mut au_start = 0usize; + let mut au_has_slice = false; + + while let Ok(nalu) = Nalu::next(&mut cursor) { + let header_start = cursor.position() as usize; + let start = header_start - nalu.offset; + let is_slice = (nalu.header.type_ as u32) < 32; + let first_slice_flag = + is_slice && stream.get(header_start + 2).is_some_and(|b| b & 0x80 != 0); + + if au_has_slice && (!is_slice || first_slice_flag) { + aus.push(&stream[au_start..start]); + au_start = start; + au_has_slice = false; + } + au_has_slice |= is_slice; + } + aus.push(&stream[au_start..]); + aus +} + +/// What the client would do with each AU, condensed to the one bit that matters: +/// did THIS AU make the client conceal and ask for recovery? +/// +/// Mirrors `video_vk_native::NativeVulkanDecoder::decode`'s CPU half exactly — a +/// plan that fails outright is trouble, and a plan whose warnings include an +/// integrity warning is damage. (The third source, a driver `Failed` verdict on a +/// prior frame, has no CPU analogue and is the GPU tests' business.) +fn damaged_h264(planner: &mut H264Planner, au: &[u8]) -> bool { + match planner.plan_au(au) { + Ok(plan) => plan.warnings.iter().any(is_integrity_warning), + // A refusal is the loudest possible detection: the client turns it into an + // Err, releases nothing, and asks. Counting it as detected is what the + // client does. + Err(_) => true, + } +} + +/// The H.265 twin — with the one arm that is NOT damage. +/// +/// `RaslSkipped` is the spec's own answer (8.1.3 NOTE) for a leading picture whose +/// references precede an open-GOP join: `VkH265Decoder::decode` turns it into +/// `Ok(None)` and clears the warning ledger, so the client neither drops a frame +/// nor asks for anything. Counting it here would let this harness "detect" a fault +/// through a path production treats as a non-event. +fn damaged_h265(planner: &mut H265Planner, au: &[u8]) -> bool { + use pf_bitstream::h265::PlanError; + match planner.plan_au(au) { + Ok(plan) => plan.warnings.iter().any(is_integrity_warning_h265), + Err(PlanError::RaslSkipped { .. }) => false, + Err(_) => true, + } +} + +/// Replay a vector with `fault` armed, returning for each AU index whether the +/// client would have flagged damage — plus how many AUs were actually faulted, so +/// a test can prove the injector fired at all rather than passing vacuously. +/// +/// Generic over the codec through the split + verdict pair, so both legs replay +/// through ONE driver: the injector, the AU cadence and the "a dropped AU has no +/// verdict of its own" rule are codec-independent, and forking them per codec is +/// how the two legs would quietly stop testing the same thing. +fn replay<'a>( + stream: &'a [u8], + split: fn(&'a [u8]) -> Vec<&'a [u8]>, + mut verdict: impl FnMut(&[u8]) -> bool, + fault: Option, +) -> (Vec, usize) { + let aus = split(stream); + let mut fault = fault; + let mut flags = Vec::with_capacity(aus.len()); + let mut faulted = 0usize; + for au in aus { + let action = match &mut fault { + Some(f) => f.apply(au), + None => FaultAction::Pass, + }; + match action { + FaultAction::Pass => flags.push(verdict(au)), + FaultAction::Drop => { + faulted += 1; + // The AU never reaches the decoder — exactly what the client does + // for a dropped AU, and exactly what the network does for a lost + // one. No verdict for this index. + flags.push(false); + } + FaultAction::Corrupt(bytes) => { + faulted += 1; + flags.push(verdict(&bytes)); + } + } + } + (flags, faulted) +} + +fn replay_h264(fault: Option) -> (Vec, usize) { + let mut planner = H264Planner::new(); + replay( + TEST_25FPS_H264, + split_h264, + move |au| damaged_h264(&mut planner, au), + fault, + ) +} + +fn replay_h265(fault: Option) -> (Vec, usize) { + let mut planner = H265Planner::new(); + replay( + TEST_25FPS_H265, + split_h265, + move |au| damaged_h265(&mut planner, au), + fault, + ) +} + +/// How many AUs after the fault detection is allowed to take on H.264. ONE: a +/// dropped reference is visible to the planner on the very next AU that references +/// it. The bound is stated as a number rather than "eventually" because +/// "eventually" is what the 500 ms freeze backstop already provides — the whole +/// point of local detection is that it is immediate. +const DETECT_WITHIN: usize = 1; + +/// The fault period the tests replay at: five faults over the 250-AU vector, far +/// enough apart that each one's detection is unambiguously about ITS fault. +const PERIOD: u32 = 50; + +/// The 0-based indices [`AuFault`] faults at `PERIOD`, over `total` AUs. The +/// injector counts the AUs it is OFFERED, 1-based, so the first fault lands on +/// index `PERIOD - 1` — which is also why any period above 1 leaves a session's +/// opening parameter sets and IDR untouched. +fn fault_indices(total: usize) -> Vec { + (PERIOD as usize - 1..total) + .step_by(PERIOD as usize) + .collect() +} + +/// Every faulted AU is followed within [`DETECT_WITHIN`] AUs by a damage verdict. +/// Returns how many faults actually had a successor window to check, so the caller +/// can refuse a vacuous pass. +fn assert_detected_after_each_fault(flags: &[bool], what: &str) -> usize { + let mut checked = 0usize; + for dropped in fault_indices(flags.len()) { + // The last AU of the vector has no successor to detect on — the stream + // simply ends there. Skipping it is honest; asserting on it would be a + // statement about the fixture's length, not about detection. + let Some(window) = flags.get(dropped + 1..(dropped + 1 + DETECT_WITHIN).min(flags.len())) + else { + continue; + }; + if window.is_empty() { + continue; + } + checked += 1; + assert!( + window.iter().any(|&d| d), + "{what}: the AU(s) after dropped AU {dropped} must read as damaged — \ + the reference it needs was never decoded" + ); + } + checked +} + +/// The indices that read as damaged — the message a failing assertion needs. +fn flagged(flags: &[bool]) -> Vec { + flags + .iter() + .enumerate() + .filter(|(_, &d)| d) + .map(|(i, _)| i) + .collect() +} + +/// The control: a healthy stream must produce NO damage verdict anywhere, on +/// either codec. Without this the fault tests below prove nothing — a detector +/// that fires on every AU would pass them and cost a keyframe round trip per +/// second in the field. +#[test] +fn a_clean_stream_never_reads_as_damaged() { + for (codec, (flags, faulted)) in [("h264", replay_h264(None)), ("h265", replay_h265(None))] { + assert_eq!(faulted, 0, "{codec}: no fault armed"); + assert_eq!(flags.len(), 250, "{codec}: the whole vector replayed"); + assert!( + flagged(&flags).is_empty(), + "{codec}: the clean vector must plan without a single integrity \ + warning — flagged AUs: {:?}", + flagged(&flags) + ); + } +} + +/// A DROPPED access unit — the everyday network-loss shape — is detected on the +/// next AU, because that AU references a picture the DPB never received. This is +/// the exit criterion's first half: deliberately corrupted input, detection within +/// a bounded number of frames. +#[test] +fn a_dropped_access_unit_is_detected_on_the_very_next_one() { + let (flags, faulted) = replay_h264(Some(AuFault::new(FaultMode::Drop, PERIOD))); + assert!(faulted >= 4, "the injector fired ({faulted} AUs dropped)"); + let checked = assert_detected_after_each_fault(&flags, "h264"); + assert!( + checked >= 4, + "{checked} drops actually had a successor to check" + ); +} + +/// The H.265 leg of the same criterion, and the reason it is a separate test +/// rather than a loop over both codecs: HEVC detects a dropped AU through a +/// DIFFERENT predicate, and it has a class of AU whose loss is legitimately +/// invisible. +/// +/// There is no `frame_num` gap to notice — POC is coded per picture and a jump in +/// it is legal — so everything rests on the RPS asking for a picture the DPB does +/// not hold (`MissingReference`). If that ever stopped firing, HEVC would lose +/// drop detection entirely while the H.264 leg above stayed green. +/// +/// And the RPS can only speak for pictures something REFERENCES. A sub-layer +/// non-reference picture (`TRAIL_N` and friends — 3 of the 5 faults this vector +/// takes) is referenced by nothing, so its loss damages no later picture and the +/// planner is right to stay silent: it is a missing output frame, caught by the +/// wire's frame-index gap, not by the bitstream. Asserting BOTH verdicts is what +/// stops that correct silence from being mistaken for a detection hole — or a +/// detection hole from hiding behind it. +#[test] +fn a_dropped_hevc_reference_picture_is_detected_through_the_rps() { + // Which AUs carry a picture something can reference? Read off a CLEAN replay, + // so the classification is the stream's own and not this test's guess. + let mut planner = H265Planner::new(); + let referenced: Vec = split_h265(TEST_25FPS_H265) + .into_iter() + .map(|au| { + planner + .plan_au(au) + .map(|plan| !plan.picture.nalu_type.is_slnr()) + .unwrap_or(false) + }) + .collect(); + + let (flags, faulted) = replay_h265(Some(AuFault::new(FaultMode::Drop, PERIOD))); + assert!(faulted >= 4, "the injector fired ({faulted} AUs dropped)"); + + let (mut checked_refs, mut checked_slnr) = (0usize, 0usize); + for dropped in fault_indices(flags.len()) { + let Some(window) = flags.get(dropped + 1..(dropped + 1 + DETECT_WITHIN).min(flags.len())) + else { + continue; + }; + if window.is_empty() { + continue; + } + if referenced[dropped] { + checked_refs += 1; + assert!( + window.iter().any(|&d| d), + "h265: the AU after dropped REFERENCE picture {dropped} must read \ + as damaged — its RPS names a picture the DPB never received" + ); + } else { + checked_slnr += 1; + assert!( + !window.iter().any(|&d| d), + "h265: dropping sub-layer non-reference picture {dropped} damages \ + nothing — if the planner starts complaining here it is reporting \ + damage that did not happen, and every such report costs a frame \ + and a keyframe round trip" + ); + } + } + assert!( + checked_refs >= 2 && checked_slnr >= 2, + "the vector must exercise BOTH classes ({checked_refs} reference drops, \ + {checked_slnr} non-reference drops) or this test proves only half of what \ + it claims" + ); +} + +/// A TRUNCATED access unit is NOT parser-visible, and the assertion is that it +/// stays that way — on both codecs. +/// +/// Annex-B has no NALU length field: a slice cut at a byte boundary is +/// indistinguishable from a shorter slice. Its header parses, the picture plans, +/// it enters the DPB, and every later AU resolves its reference against an entry +/// that exists — so nothing in the syntax is ever wrong. (pf-bitstream's +/// `TruncatedAu` warning is a narrower thing entirely: a NALU whose HEADER is +/// malformed with real data still behind it.) The hardware, meanwhile, is handed a +/// slice whose bitstream ends mid-picture, which is a decode error it can report — +/// so this mode is how a lab run fires the driver's `RESULT_STATUS` detector +/// deterministically, and it is useless without one. +#[test] +fn a_truncated_access_unit_is_invisible_to_the_parser_and_needs_the_driver_verdict() { + for (codec, (flags, faulted)) in [ + ( + "h264", + replay_h264(Some(AuFault::new(FaultMode::Truncate, PERIOD))), + ), + ( + "h265", + replay_h265(Some(AuFault::new(FaultMode::Truncate, PERIOD))), + ), + ] { + assert!( + faulted >= 4, + "{codec}: the injector fired ({faulted} AUs truncated)" + ); + assert!( + flagged(&flags).is_empty(), + "{codec}: a mid-slice cut carries no syntax error — if this starts \ + firing, the cut is landing on a NALU header and the mode has stopped \ + exercising the driver-only path it exists for (flagged AUs: {:?})", + flagged(&flags) + ); + } +} + +/// The mode that shows what the DRIVER's status query is for: a byte flipped deep +/// in a slice payload leaves a bitstream that parses perfectly, resolves every +/// reference, and decodes to a wrong picture. The planner is silent — as it should +/// be, since nothing about the syntax is wrong — and on the FFmpeg rungs that +/// silence is the end of the story (`nb_queries = 0`, no `AV_FRAME_FLAG_CORRUPT`). +/// This is the Xbox Ally X class exactly, on both codecs. +#[test] +fn a_payload_bit_flip_is_invisible_to_the_parser_which_is_why_the_status_query_exists() { + for (codec, (flags, faulted)) in [ + ( + "h264", + replay_h264(Some(AuFault::new(FaultMode::Flip, PERIOD))), + ), + ( + "h265", + replay_h265(Some(AuFault::new(FaultMode::Flip, PERIOD))), + ), + ] { + assert!( + faulted >= 4, + "{codec}: the injector fired ({faulted} AUs flipped)" + ); + assert!( + flagged(&flags).is_empty(), + "{codec}: a payload flip must not be parser-visible — if this ever \ + starts firing, the flip is landing in syntax rather than payload and \ + the mode has stopped testing what it claims (flagged AUs: {:?})", + flagged(&flags) + ); + } +} diff --git a/crates/punktfunk-core/src/reanchor.rs b/crates/punktfunk-core/src/reanchor.rs index 795630d9..d005cc6c 100644 --- a/crates/punktfunk-core/src/reanchor.rs +++ b/crates/punktfunk-core/src/reanchor.rs @@ -13,6 +13,13 @@ //! and — over the C ABI — the Apple client). The state machine is time-driven but takes `now` as a //! parameter so it is unit-testable without a clock; the C ABI wrappers supply `Instant::now()`. //! +//! A client whose decoder parses the bitstream ITSELF has a fourth, independent way to see a clean +//! re-anchor: the **recovery point SEI**, which an intra-refresh encoder emits to name the picture at +//! which its wave has healed the frame. [`on_local_recovery`](ReanchorGate::on_local_recovery) is +//! that path. It is strictly ADDITIONAL — a client without a local parser (Android MediaCodec, Apple +//! VideoToolbox, every FFmpeg rung, which exposes no SEI) simply never calls it and every wire +//! behaviour above is bit-for-bit unchanged. +//! //! [`USER_FLAG_RECOVERY_POINT`]: crate::packet::USER_FLAG_RECOVERY_POINT //! [`USER_FLAG_RECOVERY_ANCHOR`]: crate::packet::USER_FLAG_RECOVERY_ANCHOR @@ -100,6 +107,37 @@ fn reanchor_after_frame( } } +/// What a client's OWN bitstream parser saw about intra-refresh recovery on one decoded frame — the +/// in-band counterpart of the wire's [`USER_FLAG_RECOVERY_POINT`](crate::packet::USER_FLAG_RECOVERY_POINT). +/// +/// Two facts rather than one verdict, because the gate needs both and only the gate knows how to +/// combine them. A recovery point SEI promises: *a decoder that starts at THIS AU has a correct +/// picture N frames later*. That promise covers a decoder which lost references BEFORE the SEI (the +/// wave re-codes every stripe after it, so the stale content is fully overwritten) and says nothing +/// at all about one which lost references AFTER it (the already-swept stripes still reference the +/// lost picture). So a recovery point may only lift a freeze when its SEI was observed at or after +/// the loss — which is the pairing [`ReanchorGate::on_local_recovery`] performs, since the gate is +/// the only party that knows when the loss was. +/// +/// Produced by pf-vkdecode's `RecoveryWatch` on the native decode lane. Every other lane leaves it +/// [`Default`] and nothing changes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct LocalRecovery { + /// The AU that produced this frame carried a recovery point SEI — a heal starts here. + pub sei_here: bool, + /// This frame IS the recovery point a previously-seen SEI named — the heal completed. + pub is_recovery_point: bool, +} + +impl LocalRecovery { + /// Nothing observed — what every frame of a stream without recovery point SEIs reports, and what + /// every client without a local parser passes. + pub const NONE: LocalRecovery = LocalRecovery { + sei_here: false, + is_recovery_point: false, + }; +} + /// Whether a decoded frame should be shown or withheld while the gate is (or isn't) frozen. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GateVerdict { @@ -136,6 +174,17 @@ pub struct ReanchorGate { /// The last `frames_dropped` value [`poll`](Self::poll) observed; a climb means the reassembler /// declared an AU unrecoverable and the following deltas will conceal, so arm. last_dropped: u64, + /// A local recovery point SEI has been observed SINCE the latest arm — the precondition for a + /// later recovery point to be trusted ([`LocalRecovery`]). Zeroed at every arm, so a heal that + /// began before the loss can never lift the freeze the loss raised. + local_sei_since_arm: bool, + /// How many times the freeze has been armed ([`Self::arms`]) — a monotonic counter, never + /// reset. Only a client with its OWN bitstream parser needs it: `local_sei_since_arm` pairs an + /// SEI against the arm in TIME, but a decoder can hand back a frame it decoded *before* the + /// loss (a post-failure DPB flush does exactly that), and no wall clock separates those. Such + /// a client stamps the decoder's decode-order watermark whenever this counter moves and + /// discards the local recovery of anything older. Every other client ignores it. + arms: u64, } impl ReanchorGate { @@ -148,9 +197,24 @@ impl ReanchorGate { deadline: None, no_output_streak: 0, last_dropped: frames_dropped, + local_sei_since_arm: false, + arms: 0, } } + /// How many times the freeze has been armed since the gate was created — monotonic, never + /// reset, and moved by EVERY arm site including the ones inside [`Self::on_no_output`] and + /// [`Self::poll`] (but not by the overdue backstop, which re-asks without re-arming). + /// + /// A client whose decoder parses the bitstream itself watches this so it can pair a frame's + /// [`LocalRecovery`] against the arm by DECODE ORDER rather than by arrival: a decoder that + /// flushes its DPB after a failed AU hands back pictures decoded before the loss, and their + /// recovery marks describe a wave that completed before it. Every other client can ignore + /// this entirely — nothing in the gate's own behaviour reads it. + pub fn arms(&self) -> u64 { + self.arms + } + /// Arm the freeze: a loss was detected (a frame-index gap, a dropped-count climb, or a decoder /// wedge/demotion). Zeroes the mark count so a fresh loss waits out two fresh recovery marks, and /// (re-)sets the backstop deadline. Idempotent while already frozen (re-arming just re-zeroes the @@ -158,9 +222,60 @@ impl ReanchorGate { pub fn arm(&mut self, now: Instant) { self.awaiting = true; self.marks = 0; + self.arms = self.arms.saturating_add(1); + // A heal that was already in flight when this loss landed proves nothing about it: the + // stripes the wave already swept still reference the picture we just lost. Only an SEI seen + // from here on may be trusted ([`LocalRecovery`]). + self.local_sei_since_arm = false; self.deadline = Some(now + REANCHOR_FREEZE_MAX); } + /// Fold the client's OWN recovery-point observation for one decoded frame, BEFORE handing that + /// frame to [`on_decoded`](Self::on_decoded). Returns `true` when it lifted the freeze. + /// + /// This is the only re-anchor signal that does not come off the wire, and it exists because + /// intra-refresh sessions otherwise have NO clean point a client can see. The host's wave never + /// emits an IDR; libavcodec flags `AV_FRAME_FLAG_KEY` only for true IDRs; and the wire mark + /// ([`USER_FLAG_RECOVERY_POINT`](crate::packet::USER_FLAG_RECOVERY_POINT)) is set by exactly one + /// encoder backend — pf-encode's Linux libav-NVENC under its `PUNKTFUNK_INTRA_REFRESH` opt-in — + /// while the other two that run a wave (Windows AMF, QSV) leave it off pending on-glass GDR + /// validation. So a client on one of those sessions freezes on loss and holds until + /// [`REANCHOR_FREEZE_MAX`] expires, then forces the very IDR the wave exists to avoid: half a + /// second of frozen picture followed by a 20-40× frame, on a stream that healed itself long + /// before. A decoder that parses the recovery point SEI can simply watch it happen — and unlike + /// the wire flag, that signal cannot be lost separately from the picture it describes. + /// + /// The rule, and the reason it is a pairing rather than a single flag: a recovery point is + /// trustworthy only when its SEI arrived at or after the loss ([`LocalRecovery`] carries the + /// argument). A mark whose SEI predates the arm is IGNORED — silently and deliberately; the + /// backstop still covers it, exactly as today. + /// + /// It lifts on the FIRST trusted recovery point, unlike the wire mark's two + /// ([`REANCHOR_MARKS_TO_LIFT`]), and the difference is not a relaxation. The wire mark is a + /// phase-fixed WAVE BOUNDARY with no knowledge of when the loss was, so the first boundary after + /// a loss is only partially healed and a second must be waited out. The SEI names the recovery + /// point of a specific wave and is only honoured for a wave that STARTED after the loss, so the + /// picture at that point is fully swept by construction — the same guarantee an + /// [`USER_FLAG_RECOVERY_ANCHOR`](crate::packet::USER_FLAG_RECOVERY_ANCHOR) gives, derived + /// locally instead of trusted from the host. + /// + /// Called on a gate that is not frozen it only records the SEI; there is nothing to lift. + pub fn on_local_recovery(&mut self, obs: LocalRecovery) -> bool { + if obs.sei_here { + self.local_sei_since_arm = true; + } + if !(obs.is_recovery_point && self.local_sei_since_arm && self.awaiting) { + return false; + } + self.awaiting = false; + self.deadline = None; + self.marks = 0; + // Spent: the next heal needs its own SEI. Without this a single wave's recovery point could + // lift a freeze armed by a LATER loss, which is the one thing the pairing exists to prevent. + self.local_sei_since_arm = false; + true + } + /// Fold one decoded frame and decide whether to present or withhold it. /// /// `wire_flags` is the AU's `user_flags` word ([`crate::session::Frame::flags`] / @@ -456,6 +571,167 @@ mod tests { assert!(g.is_holding(), "but never resumes to the concealed picture"); } + // ---- the local (recovery point SEI) path ---- + + /// One decoded frame's local observation, spelled as the two facts it is. + fn local(sei_here: bool, is_recovery_point: bool) -> LocalRecovery { + LocalRecovery { + sei_here, + is_recovery_point, + } + } + + /// The headline: an intra-refresh session heals and the freeze lifts on the SEI's own recovery + /// point — no wire flag, no IDR, and above all no waiting out REANCHOR_FREEZE_MAX. This is the + /// half-second of frozen picture M4 exists to remove. + #[test] + fn a_local_recovery_point_lifts_the_freeze_without_the_backstop() { + let mut g = ReanchorGate::new(0); + let start = t0(); + g.arm(start); // a frame-index gap + assert_eq!(g.on_decoded(0, false, start), GateVerdict::Hold); + + // The wave starts (SEI) and sweeps; the client keeps holding meanwhile. + assert!(!g.on_local_recovery(local(true, false))); + assert_eq!(g.on_decoded(0, false, start), GateVerdict::Hold); + assert!(!g.on_local_recovery(local(false, false))); + assert_eq!(g.on_decoded(0, false, start), GateVerdict::Hold); + + // The recovery point: healed, and the very next frame is presented — while the backstop + // deadline is still far away, which is the whole point. + let mid = start + Duration::from_millis(120); + assert!(g.on_local_recovery(local(false, true)), "the heal lifts it"); + assert!(!g.is_holding()); + assert_eq!(g.on_decoded(0, false, mid), GateVerdict::Present); + assert!( + !g.poll(0, mid), + "and no keyframe is ever asked for — no IDR spike on a stream that healed itself" + ); + } + + /// The pairing rule. A recovery point whose SEI arrived BEFORE the loss guarantees nothing: the + /// stripes that wave already swept still reference the picture that was lost, so lifting there + /// would flash a half-stale frame. It must be ignored and the freeze must hold. + #[test] + fn a_recovery_point_from_a_wave_that_predates_the_loss_is_ignored() { + let mut g = ReanchorGate::new(0); + let now = t0(); + // A wave is in flight when the loss lands. + assert!(!g.on_local_recovery(local(true, false))); + g.arm(now); + // Its recovery point arrives — about a wave that started before the loss. + assert!( + !g.on_local_recovery(local(false, true)), + "a pre-loss wave's recovery point must not lift" + ); + assert!(g.is_holding()); + assert_eq!(g.on_decoded(0, false, now), GateVerdict::Hold); + // The NEXT wave — started after the loss — does lift. + assert!(!g.on_local_recovery(local(true, false))); + assert!(g.on_local_recovery(local(false, true))); + assert_eq!(g.on_decoded(0, false, now), GateVerdict::Present); + } + + /// A single recovery point is spent when it lifts: it must not also lift a freeze armed by a + /// LATER loss, which is exactly what a sticky "an SEI was seen once" flag would do. + #[test] + fn a_spent_recovery_point_cannot_lift_the_next_loss() { + let mut g = ReanchorGate::new(0); + let now = t0(); + g.arm(now); + g.on_local_recovery(local(true, false)); + assert!(g.on_local_recovery(local(false, true))); + // A fresh loss, and a stray recovery point with no new SEI behind it. + g.arm(now); + assert!( + !g.on_local_recovery(local(false, true)), + "the previous wave's credit is gone" + ); + assert!(g.is_holding()); + } + + /// An SEI whose count is zero puts both facts on ONE frame ("start here, this picture is already + /// exact"). It must still lift — the pairing is about ORDER, not about needing two frames. + #[test] + fn an_sei_that_is_its_own_recovery_point_lifts_on_that_frame() { + let mut g = ReanchorGate::new(0); + let now = t0(); + g.arm(now); + assert!(g.on_local_recovery(local(true, true))); + assert_eq!(g.on_decoded(0, false, now), GateVerdict::Present); + } + + /// The whole path is inert for every client that has no local parser — Android MediaCodec, Apple + /// VideoToolbox, and all four FFmpeg rungs (libavcodec exposes no SEI). They never call it, and + /// even if they did with an empty observation nothing may change. + #[test] + fn a_client_without_a_local_parser_sees_no_behaviour_change() { + let mut g = ReanchorGate::new(0); + let now = t0(); + g.arm(now); + for _ in 0..8 { + assert!(!g.on_local_recovery(LocalRecovery::NONE)); + assert_eq!(g.on_decoded(0, false, now), GateVerdict::Hold); + } + assert!( + g.is_holding(), + "still frozen — only the wire can lift this one" + ); + assert_eq!(g.on_decoded(SOF, false, now), GateVerdict::Present); + } + + /// A local recovery point on a gate that is NOT frozen changes nothing: there is no freeze to + /// lift, and the observation must not become a stored credit that pre-lifts the next loss. + #[test] + fn a_recovery_point_on_an_unfrozen_gate_is_not_banked() { + let mut g = ReanchorGate::new(0); + let now = t0(); + assert!(!g.on_local_recovery(local(true, true))); + assert!(!g.is_holding()); + g.arm(now); + assert!( + !g.on_local_recovery(local(false, true)), + "the pre-arm SEI was cleared by the arm" + ); + assert!(g.is_holding()); + } + + /// `arms()` has to move at EVERY arm site — including the two that arm from inside the gate — + /// because a client pairing local recovery by decode order re-stamps its watermark off exactly + /// this counter. An arm site that did not move it would leave that client trusting the + /// recovery marks of pictures decoded before the loss. The overdue backstop is the one thing + /// that must NOT move it: it re-asks without re-arming, and re-stamping there would discard a + /// heal that is legitimately in flight. + #[test] + fn every_arm_site_moves_the_arm_counter_and_the_backstop_does_not() { + let mut g = ReanchorGate::new(0); + let start = t0(); + assert_eq!(g.arms(), 0, "a fresh gate has never armed"); + + g.arm(start); + assert_eq!(g.arms(), 1); + // Re-arming mid-freeze is a second loss — it counts. + g.arm(start); + assert_eq!(g.arms(), 2); + + // The drop-climb arm inside `poll`. + assert!(g.poll(1, start)); + assert_eq!(g.arms(), 3); + + // The overdue backstop re-asks and keeps holding — but does not re-arm. + let later = start + REANCHOR_FREEZE_MAX + Duration::from_millis(1); + assert!(g.poll(1, later)); + assert_eq!(g.arms(), 3, "the backstop re-asks, it does not re-arm"); + + // The no-output streak's arm. + let mut g = ReanchorGate::new(0); + assert!(!g.on_no_output(start)); + assert!(!g.on_no_output(start)); + assert_eq!(g.arms(), 0, "the streak has not tripped yet"); + assert!(g.on_no_output(start)); + assert_eq!(g.arms(), 1); + } + #[test] fn a_live_mark_stream_pushes_the_deadline_out() { // A healing wave (marks arriving) must not be pre-empted by the overdue IDR floor. diff --git a/include/punktfunk_core.h b/include/punktfunk_core.h index 847b97ee..247b63e0 100644 --- a/include/punktfunk_core.h +++ b/include/punktfunk_core.h @@ -1635,6 +1635,22 @@ typedef struct ColorInfo ColorInfo; // else happened to be using the audio graph that day. typedef struct JitterTuning JitterTuning; +// What a client's OWN bitstream parser saw about intra-refresh recovery on one decoded frame — the +// in-band counterpart of the wire's [`USER_FLAG_RECOVERY_POINT`](crate::packet::USER_FLAG_RECOVERY_POINT). +// +// Two facts rather than one verdict, because the gate needs both and only the gate knows how to +// combine them. A recovery point SEI promises: *a decoder that starts at THIS AU has a correct +// picture N frames later*. That promise covers a decoder which lost references BEFORE the SEI (the +// wave re-codes every stripe after it, so the stale content is fully overwritten) and says nothing +// at all about one which lost references AFTER it (the already-swept stripes still reference the +// lost picture). So a recovery point may only lift a freeze when its SEI was observed at or after +// the loss — which is the pairing [`ReanchorGate::on_local_recovery`] performs, since the gate is +// the only party that knows when the loss was. +// +// Produced by pf-vkdecode's `RecoveryWatch` on the native decode lane. Every other lane leaves it +// [`Default`] and nothing changes. +typedef struct LocalRecovery LocalRecovery; + #if defined(PUNKTFUNK_FEATURE_QUIC) // Opaque handle to a live `punktfunk/1` connection (QUIC control plane + UDP data plane, all // pumped on internal threads). @@ -2037,6 +2053,8 @@ typedef struct { + + // The multipliers a picker offers. `1.0` (Native) is the default; the rest are the round stops // users reason about. Shared so every client's list stays identical. #define PUNKTFUNK_PRESETS { 0.5, 0.67, 0.75, 1.0, 1.25, 1.5, 2.0, 3.0, 4.0, }