diff --git a/crates/pf-client-core/Cargo.toml b/crates/pf-client-core/Cargo.toml index 0d39da70..4c42bc45 100644 --- a/crates/pf-client-core/Cargo.toml +++ b/crates/pf-client-core/Cargo.toml @@ -18,9 +18,10 @@ punktfunk-core = { path = "../punktfunk-core", features = ["quic"] } # FFmpeg's Vulkan hwcontext surface (Vulkan Video decode on the presenter's device). pf-ffvk = { path = "../pf-ffvk" } # Native Vulkan Video decode (WP-C of the native-decode program, HEVC added by M3 -# WP-2): auto's rung immediately above FFmpeg-Vulkan (2026-08-05 ladder decision), also -# pinnable via `PUNKTFUNK_DECODER=native-vulkan` — video_vk_native.rs, running -# pf-vkdecode's VkH264Decoder/VkH265Decoder on the presenter's shared device. +# WP-2, AV1 by M7 — pin only): auto's rung immediately above FFmpeg-Vulkan (2026-08-05 +# ladder decision), also pinnable via `PUNKTFUNK_DECODER=native-vulkan` — +# video_vk_native.rs, running pf-vkdecode's VkH264Decoder/VkH265Decoder/VkAv1Decoder on +# the presenter's shared device. pf-vkdecode = { path = "../pf-vkdecode" } async-channel = "2" diff --git a/crates/pf-client-core/src/lib.rs b/crates/pf-client-core/src/lib.rs index e95e9ff9..ebc60cb2 100644 --- a/crates/pf-client-core/src/lib.rs +++ b/crates/pf-client-core/src/lib.rs @@ -84,10 +84,11 @@ mod video_vaapi; #[cfg(target_os = "linux")] pub mod video_vaapi_native; // Native Vulkan Video decode (WP-C of the native-decode program, HEVC added by M3 -// WP-2): pf-vkdecode's H.264/H.265 decoders on the presenter's shared device — auto's -// rung immediately above FFmpeg-Vulkan (2026-08-05 ladder decision; the program is -// dropping FFmpeg from the client), also pinnable via -// `PUNKTFUNK_DECODER=native-vulkan`. +// WP-2, AV1 by M7): pf-vkdecode's H.264/H.265/AV1 decoders on the presenter's shared +// device — auto's rung immediately above FFmpeg-Vulkan (2026-08-05 ladder decision; +// the program is dropping FFmpeg from the client), also pinnable via +// `PUNKTFUNK_DECODER=native-vulkan`. The AV1 leg is PIN ONLY until it has hardware +// evidence, so an `auto` AV1 session still lands on the FFmpeg rungs. #[cfg(any(target_os = "linux", windows))] mod video_vk_native; #[cfg(any(target_os = "linux", windows))] diff --git a/crates/pf-client-core/src/video.rs b/crates/pf-client-core/src/video.rs index 86a1ea9a..40e9a6ce 100644 --- a/crates/pf-client-core/src/video.rs +++ b/crates/pf-client-core/src/video.rs @@ -18,7 +18,9 @@ //! `native-d3d11va` (Windows) pins M5's pf-dxvadec `ID3D11VideoDecoder` rung and //! `native-vaapi` (Linux) pins M6's pf-vaadec libva rung. Both of those are reachable //! ONLY by their pin — they are absent from every `auto` arm until they have the -//! hardware evidence M2's native rung had before IT joined `auto`): +//! hardware evidence M2's native rung had before IT joined `auto`, and M7's AV1 leg of +//! the native Vulkan rung is pin-only for the same reason: `native-vulkan` reaches it, +//! `auto` never does, so an AV1 session still lands on the FFmpeg rungs by default): //! //! * **Vulkan Video**: FFmpeg's Vulkan decoder running on the PRESENTER's own VkDevice //! (its handles arrive via [`VulkanDecodeDevice`]) — the decoded VkImage feeds the @@ -109,13 +111,14 @@ pub enum DecodedImage { #[cfg(all(any(target_os = "linux", windows), feature = "pyrowave"))] PyroWave(crate::video_pyrowave::PyroWavePlanarFrame), /// Native Vulkan Video output (pf-vkdecode — auto's H.264/HEVC rung immediately - /// above FFmpeg-Vulkan, also pinnable via `PUNKTFUNK_DECODER=native-vulkan`): a - /// decoded image + per-plane views already on the PRESENTER's device — same - /// zero-copy contract as [`DecodedImage::VkFrame`], no FFmpeg involved. The - /// picture format is the stream's, carried on the frame - /// ([`NativeVkFrame::vk_format`] — NV12 for H.264 and HEVC Main, P010 for Main - /// 10, the two-plane 4:4:4 formats for RExt), never assumed. The presenter waits - /// the frame's timeline pair, transitions the layer for sampling and BACK to + /// above FFmpeg-Vulkan, plus M7's pin-only AV1 leg; pinnable via + /// `PUNKTFUNK_DECODER=native-vulkan`): a decoded image + per-plane views already + /// on the PRESENTER's device — same zero-copy contract as + /// [`DecodedImage::VkFrame`], no FFmpeg involved. The picture format is the + /// stream's, carried on the frame ([`NativeVkFrame::vk_format`] — NV12 for H.264, + /// HEVC Main and AV1 Main 8-bit, P010 for Main 10, the two-plane 4:4:4 formats for + /// RExt and AV1 High), never assumed. The presenter waits the frame's timeline + /// pair, transitions the layer for sampling and BACK to /// [`NativeVkFrame::layout`], and releases the decoder's slot by dropping the /// frame (its guard sends the release token). NativeVk(NativeVkFrame), @@ -181,6 +184,24 @@ pub struct DecodeHealth { /// The longest [`Self::run`] of the session — the worst moment, which a /// once-per-second sample of `run` will usually miss entirely. pub worst_run: u32, + /// Frames that decoded CORRECTLY and were then discarded without ever being + /// shown, because the backend's deliverable queue overflowed + /// (`video_vk_native::MAX_DELIVERABLE` — a decoder making more pictures + /// display-ready per access unit than the pump can take one at a time). + /// + /// Deliberately its own number and not folded into any of the three above: + /// nothing was damaged, nothing was refused and no driver failed, so counting + /// it as any of those would put a damage report on a healthy stream — and the + /// AU it happened on still showed a picture, so it must not extend + /// [`Self::run`] either. But it cannot be nothing at all: a session quietly + /// discarding a frame per AU is one running at half the frame rate it thinks + /// it is, and before this counter existed it read as perfectly clean. + /// + /// Structurally 0 on every rung but native Vulkan — it is the only one with a + /// deliverable queue — and not on the session stats line today; the + /// rate-limited `warn` at the drop site is the field signal, and this is the + /// number a stats field would read. + pub dropped: u64, /// This device answers per-op decode-status queries /// (`queryResultStatusSupport`). When FALSE — RADV, where recording a query /// anyway HANGS the VCN ring — [`Self::failed`] can only ever read 0, because @@ -225,6 +246,17 @@ impl DecodeHealth { self.run = 0; } } + + /// Note one correctly-decoded frame discarded unshown — see [`Self::dropped`]. + /// + /// Separate from [`Self::note`] because it is not an AU verdict: several frames + /// can be dropped within one access unit, and the access unit itself may well + /// have shipped a picture. It touches nothing but its own counter, and in + /// particular never [`Self::run`], which answers "did the picture come back" + /// and here it did. + pub(crate) fn note_dropped(&mut self) { + self.dropped = self.dropped.saturating_add(1); + } } /// A raw `VkFormat` code point, carried across the ash-free boundary. @@ -640,13 +672,15 @@ impl Drop for DrmFrameGuard { enum Backend { Vulkan(VulkanDecoder), - /// Native Vulkan Video H.264/HEVC (pf-vkdecode) on the presenter's device — + /// Native Vulkan Video H.264/HEVC/AV1 (pf-vkdecode) on the presenter's device — /// auto's rung immediately above FFmpeg-Vulkan since the 2026-08-05 ladder /// decision (WP-D closed bit-exact; the program's goal is dropping FFmpeg from /// the client), also pinnable by name (`PUNKTFUNK_DECODER=native-vulkan`) — see /// [`native_vulkan_gate`]. The negotiated codec picks the decoder once, at - /// construction; everything else about this backend is codec-agnostic. Errors - /// ride the SAME streak/demotion machinery as the FFmpeg-Vulkan rung. + /// construction; everything else about this backend is codec-agnostic. Its AV1 + /// leg (M7) is reachable by the PIN only and never through `auto`, which is the + /// gate's decision, not this variant's. Errors ride the SAME streak/demotion + /// machinery as the FFmpeg-Vulkan rung. /// Boxed: the decoder (planner + shipped-frame ledger) dwarfs the other variants, /// same as PyroWave below. NativeVulkan(Box), @@ -804,14 +838,12 @@ fn clears_demotion_streak(delivered: bool, concealed: bool) -> bool { /// `VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR` — the raw flag bit within /// [`VulkanDecodeDevice::decode_video_caps`] (this crate stays ash-free). const VIDEO_CODEC_OP_DECODE_H264: u32 = 0x0000_0001; -/// `VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR` — its H.265 sibling. (AV1 is 0x4 -/// and deliberately has no constant here: pf-vkdecode has no AV1 decoder, so the bit -/// would only invite a gate that admits a session nothing can decode.) +/// `VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR` — its H.265 sibling. const VIDEO_CODEC_OP_DECODE_H265: u32 = 0x0000_0002; /// `VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR`. The Deck's VanGogh advertises -/// it alongside H.264/H.265/VP9, and it is what -/// [`av1_hardware_decodable`] reads. +/// it alongside H.264/H.265/VP9; it is what [`av1_hardware_decodable`] reads and, +/// since M7, the caps bit [`native_codec`] demands for an AV1 session. const VIDEO_CODEC_OP_DECODE_AV1: u32 = 0x0000_0004; /// The native decoder for a negotiated wire codec, plus the @@ -821,13 +853,16 @@ const VIDEO_CODEC_OP_DECODE_AV1: u32 = 0x0000_0004; /// The two are returned together on purpose: "which decoder" and "which caps bit" /// are one fact, and splitting them is how a gate ends up admitting HEVC on an /// H.264-only decode family (`vkCreateVideoSessionKHR` for a codec operation the -/// family cannot run is undefined behaviour, not an error). AV1 has a Vulkan decode -/// op and real hardware advertises it — but there is no AV1 decoder in pf-vkdecode, -/// so those sessions must keep falling through to the FFmpeg rungs. +/// family cannot run is undefined behaviour, not an error). +/// +/// ⚠ Being here is "pf-vkdecode has a decoder", NOT "the automatic ladder may pick +/// it". AV1 (M7) is pin-only; [`native_vulkan_gate`] holds that decision, and it +/// reads this map for the codec/caps pair only. fn native_codec(codec_id: ffmpeg::codec::Id) -> Option<(NativeCodec, u32)> { match codec_id { ffmpeg::codec::Id::H264 => Some((NativeCodec::H264, VIDEO_CODEC_OP_DECODE_H264)), ffmpeg::codec::Id::HEVC => Some((NativeCodec::H265, VIDEO_CODEC_OP_DECODE_H265)), + ffmpeg::codec::Id::AV1 => Some((NativeCodec::Av1, VIDEO_CODEC_OP_DECODE_AV1)), _ => None, } } @@ -862,8 +897,9 @@ fn native_vaapi_codec(codec_id: ffmpeg::codec::Id) -> Option { } /// The native Vulkan Video admission gate (WP-C of the native-decode program, widened -/// by the 2026-08-05 ladder decision and again by M3 WP-2's HEVC wiring): the -/// pf-vkdecode backend engages when `choice` asks for it — by name +/// by the 2026-08-05 ladder decision, again by M3 WP-2's HEVC wiring and again — for +/// the pin only — by M7's AV1 wiring): the pf-vkdecode backend engages when `choice` +/// asks for it — by name /// (`PUNKTFUNK_DECODER=native-vulkan` — `choice` is env-first, so that's what carries /// it) or as the auto family (`auto`/``/`hardware`), where native is the rung /// immediately ABOVE FFmpeg-Vulkan: WP-D closed with bit-exact parity against @@ -879,10 +915,19 @@ fn native_vaapi_codec(codec_id: ffmpeg::codec::Id) -> Option { /// pin — refuses. /// /// Beyond the choice: the negotiated wire codec must be one pf-vkdecode speaks — -/// H.264 or H.265 ([`native_codec`]) — and the presenter's decode family must +/// H.264, H.265 or AV1 ([`native_codec`]) — and the presenter's decode family must /// advertise THAT codec's decode operation. `video_decode` alone proves the extension /// stack, never the codec: an AV1-only decode family exists on real hardware, and -/// H.264-only ones are the common case on older silicon. AV1 sessions refuse outright. +/// H.264-only ones are the common case on older silicon. +/// +/// **AV1 (M7) is admitted by the PIN ONLY** and is absent from the `auto` family, on +/// exactly the rule M5's native D3D11VA and M6's native VAAPI rungs follow: `auto` +/// admission is earned with hardware parity and a soak, and the AV1 rung has decoded +/// nothing on hardware. An `auto` AV1 session therefore keeps landing where it landed +/// before M7 — the FFmpeg rungs — and the pin is what a lab run uses to reach the new +/// one. The per-codec choice test is the one thing that makes this gate more than a +/// codec lookup, so it lives here rather than in [`native_codec`], which stays the +/// answer to "does a decoder exist and which caps bit does it need". /// /// What the gate deliberately does NOT check is the stream's picture SHAPE — that is /// [`NativeVulkanDecoder::new`]'s construction-time probe, which has the negotiated @@ -894,12 +939,18 @@ fn native_vulkan_gate( video_decode: bool, decode_video_caps: u32, ) -> bool { - let Some((_, codec_op)) = native_codec(codec_id) else { + let Some((codec, codec_op)) = native_codec(codec_id) else { return false; }; - matches!(choice, "native-vulkan" | "auto" | "" | "hardware") - && video_decode - && decode_video_caps & codec_op != 0 + let chosen = match codec { + // Hardware-proven rungs: the pin AND the whole auto family. + NativeCodec::H264 | NativeCodec::H265 => { + matches!(choice, "native-vulkan" | "auto" | "" | "hardware") + } + // Pin only, until this rung has decoded a frame on real hardware. + NativeCodec::Av1 => choice == "native-vulkan", + }; + chosen && video_decode && decode_video_caps & codec_op != 0 } /// Map a negotiated `quic` codec bit to the FFmpeg decoder id the client opens. @@ -1136,7 +1187,8 @@ impl Decoder { /// ([`VulkanDecodeDevice::prefer_vulkan_first`]); on H.264 and HEVC sessions the /// native pf-vkdecode rung sits immediately above FFmpeg-Vulkan wherever the /// ladder reaches it ([`native_vulkan_gate`] — the program is dropping FFmpeg, and - /// a native INIT failure falls through to FFmpeg-Vulkan). Linux: native → Vulkan → + /// a native INIT failure falls through to FFmpeg-Vulkan). An AV1 session does NOT + /// take it in `auto` — that leg is pin-only (M7). Linux: native → Vulkan → /// VAAPI → software on NVIDIA and ALL AMD (`prefer_vulkan_first` is vendor-wide — /// desktop RADV included, on-glass verdict — not just the Deck's VanGogh); /// VAAPI → native → Vulkan → software on Intel/unknown. Windows (no VAAPI @@ -1302,9 +1354,9 @@ impl Decoder { tracing::warn!( ?codec_id, video_decode = vk.is_some_and(|v| v.video_decode), - "PUNKTFUNK_DECODER=native-vulkan refused (needs an H.264 or HEVC session \ - and a presenter device whose decode family advertises that codec) — \ - standard ladder" + "PUNKTFUNK_DECODER=native-vulkan refused (needs an H.264, HEVC or AV1 \ + session and a presenter device whose decode family advertises that \ + codec) — standard ladder" ); } choice = "auto".to_string(); @@ -2173,6 +2225,51 @@ mod tests { "three driver errors interleaved with concealment must still reach the \ demotion threshold — they got to {fails}" ); + + // ---- The AV1 shape (M7), and the reason its recovery wait is an `Err` ---- + // + // A native rung waiting to re-anchor after a failure produces no picture for + // every AU of the wait, and all three codecs say so with an ERROR: H.264 and + // H.265 through their planners' `PlanError::AwaitingIdr`, AV1 through + // `VkDecodeError::AwaitingKeyAv1`. So the streak ticks for the whole wait and + // a rung that never recovers reaches the threshold. + let mut fails = 0u32; + for errored in [true; 5] { + // the failing AU, then four skipped ones + if errored { + fails += 1; + } else if clears_demotion_streak(false, false) { + fails = 0; + } + } + assert!(fails >= VAAPI_DEMOTE_AFTER); + + // The counterfactual is the whole point, and it is what the AV1 rung was + // first wired as: answer the skipped AUs with a CLEAN `Ok(None)` instead — + // no picture, no warnings, nothing to object to — and every one of them + // clears the streak. The `Err` from each failure is then alone, and + // `VAAPI_DEMOTE_AFTER` is unreachable no matter how long the session runs. + // + // The stream this strands is real and named in `NativeVulkanDecoder::new`: + // an AV1 sequence with `film_grain_params_present = 1` on a device without + // the grain decode profile fails at `ensure_state` — at EVERY key frame, and + // only at a key frame. Key frame `Err`, inter frames "clean", next key frame + // `Err`: a frozen screen for the whole session, `refused N · damaged 0 · + // run 0` on the stats line, and the `!delivered` fall-through to + // FFmpeg-Vulkan below never reached. + let mut fails = 0u32; + for errored in [true, false, false, true, false, false, true, false, false] { + if errored { + fails += 1; + } else if clears_demotion_streak(false, false) { + fails = 0; + } + } + assert!( + fails < VAAPI_DEMOTE_AFTER, + "a recovery wait answered as a CLEAN AU zeroes the streak once per frame \ + — which is why it must not be answered that way; it got to {fails}" + ); } /// Auto's hardware order (both OSes): Vulkan-first on NVIDIA (on Linux: no usable @@ -2198,10 +2295,6 @@ mod tests { assert!(!decode_device(0x8086, "Intel(R) Arc(TM) Pro Graphics").prefer_vulkan_first()); } - /// The native-Vulkan admission gate (WP-C, widened by the 2026-08-05 ladder - /// decision and again by M3 WP-2's HEVC wiring): the pin AND the auto family - /// admit on a capable H.264 or HEVC session — native sits immediately above - /// FFmpeg-Vulkan because the program is dropping FFmpeg — while every explicit /// AV1 is advertised on a HARDWARE fact, never on a decoder existing. /// /// The standing open item M7 closes. `ffmpeg::decoder::find(AV1)` says yes @@ -2238,11 +2331,16 @@ mod tests { assert!(!av1_hardware_decodable(Some(&dev))); } - /// backend pin refuses (`vulkan` names the FFmpeg-Vulkan backend specifically and - /// must keep meaning exactly that), and the codec/device legs still refuse for - /// every choice. The codec's OWN caps bit is the device leg: admitting HEVC on an - /// H.264-only decode family would create a video session for an operation the - /// family cannot run, which is undefined behaviour rather than an error. + /// The native-Vulkan admission gate (WP-C, widened by the 2026-08-05 ladder + /// decision, again by M3 WP-2's HEVC wiring and again — pin only — by M7's AV1 + /// wiring): the pin AND the auto family admit on a capable H.264 or HEVC session + /// (native sits immediately above FFmpeg-Vulkan because the program is dropping + /// FFmpeg), the PIN ALONE admits AV1, every explicit other-backend pin refuses + /// (`vulkan` names the FFmpeg-Vulkan backend specifically and must keep meaning + /// exactly that), and the codec/device legs still refuse for every choice. The + /// codec's OWN caps bit is the device leg: admitting HEVC on an H.264-only decode + /// family would create a video session for an operation the family cannot run, + /// which is undefined behaviour rather than an error. #[test] fn native_vulkan_gate_admits_pin_and_auto_family_per_codec_on_a_capable_family() { use ffmpeg::codec::Id; @@ -2258,11 +2356,13 @@ mod tests { VIDEO_CODEC_OP_DECODE_H265, 0x2, "VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR" ); + assert_eq!( + VIDEO_CODEC_OP_DECODE_AV1, 0x4, + "VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR" + ); const H264_OP: u32 = VIDEO_CODEC_OP_DECODE_H264; const H265_OP: u32 = VIDEO_CODEC_OP_DECODE_H265; - // `VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR` — a real bit on real - // hardware, and never enough on its own (no AV1 decoder exists here). - const AV1_OP: u32 = 0x4; + const AV1_OP: u32 = VIDEO_CODEC_OP_DECODE_AV1; for choice in ["native-vulkan", "auto", "", "hardware"] { // The pin and the whole auto family admit both codecs pf-vkdecode // speaks, on a family that advertises the matching op… @@ -2293,14 +2393,31 @@ mod tests { !native_vulkan_gate(choice, Id::H264, true, H265_OP), "{choice:?}" ); - // AV1 refuses whatever the family advertises — pf-vkdecode has no AV1 - // decoder, so the session must fall through to the FFmpeg rungs. + // AV1 (M7) is PIN ONLY: `native-vulkan` reaches it, and the whole auto + // family must keep landing on the FFmpeg rungs exactly as it did before + // this rung existed. That is not a caps question — the family below + // advertises AV1 — it is the "auto admission is earned with hardware + // parity and a soak" rule, and this rung has decoded nothing. + let av1_pin = choice == "native-vulkan"; + assert_eq!( + native_vulkan_gate(choice, Id::AV1, true, AV1_OP), + av1_pin, + "{choice:?}" + ); + assert_eq!( + native_vulkan_gate(choice, Id::AV1, true, H264_OP | H265_OP | AV1_OP), + av1_pin, + "{choice:?}" + ); + // …and the pin is still not a licence to skip the device leg: an AV1 + // session on a family that does not advertise the AV1 op would create a + // video session for an operation the family cannot run. assert!( - !native_vulkan_gate(choice, Id::AV1, true, AV1_OP), + !native_vulkan_gate(choice, Id::AV1, true, H264_OP | H265_OP), "{choice:?}" ); assert!( - !native_vulkan_gate(choice, Id::AV1, true, H264_OP | H265_OP | AV1_OP), + !native_vulkan_gate(choice, Id::AV1, false, AV1_OP), "{choice:?}" ); // No Vulkan-Video-capable presenter device. @@ -2336,6 +2453,10 @@ mod tests { !native_vulkan_gate(choice, Id::HEVC, true, H265_OP), "{choice:?}" ); + assert!( + !native_vulkan_gate(choice, Id::AV1, true, AV1_OP), + "{choice:?}" + ); } // The decoder the gate implies — the construction sites `expect()` this // exact agreement, so a codec admitted with no decoder behind it would be a @@ -2348,7 +2469,13 @@ mod tests { native_codec(Id::HEVC).map(|(c, _)| c), Some(NativeCodec::H265) ); - assert!(native_codec(Id::AV1).is_none()); + // AV1 has a decoder AND the caps bit here — being in this map is what the + // pin construction path reads. Whether `auto` may pick it is the gate's + // decision above, and deliberately not this one's. + assert_eq!( + native_codec(Id::AV1), + Some((NativeCodec::Av1, VIDEO_CODEC_OP_DECODE_AV1)) + ); assert!(native_codec(Id::VP9).is_none()); } diff --git a/crates/pf-client-core/src/video_vk_native.rs b/crates/pf-client-core/src/video_vk_native.rs index 1be7d591..10b2a0f4 100644 --- a/crates/pf-client-core/src/video_vk_native.rs +++ b/crates/pf-client-core/src/video_vk_native.rs @@ -1,26 +1,51 @@ //! Native Vulkan Video decode backend (WP-C of the native-decode program, widened to -//! HEVC by M3 WP-2): pf-vkdecode's [`VkH264Decoder`]/[`VkH265Decoder`] running on the -//! PRESENTER's own VkDevice — the same zero-copy shape as the FFmpeg-Vulkan backend, -//! with no FFmpeg in the path. Auto's rung immediately ABOVE FFmpeg-Vulkan since the -//! 2026-08-05 ladder decision (WP-D closed bit-exact — the program is dropping FFmpeg -//! from the client), also pinnable via `PUNKTFUNK_DECODER=native-vulkan`; -//! `video::native_vulkan_gate` is the admission either way, and a failure falls -//! through to the FFmpeg-Vulkan rung. +//! HEVC by M3 WP-2 and to AV1 by M7): pf-vkdecode's +//! [`VkH264Decoder`]/[`VkH265Decoder`]/[`VkAv1Decoder`] running on the PRESENTER's own +//! VkDevice — the same zero-copy shape as the FFmpeg-Vulkan backend, with no FFmpeg in +//! the path. Auto's rung immediately ABOVE FFmpeg-Vulkan since the 2026-08-05 ladder +//! decision (WP-D closed bit-exact — the program is dropping FFmpeg from the client), +//! also pinnable via `PUNKTFUNK_DECODER=native-vulkan`; `video::native_vulkan_gate` is +//! the admission either way, and a failure falls through to the FFmpeg-Vulkan rung. +//! **AV1 is reachable by the PIN only** — that rung has decoded nothing on hardware, +//! so it is absent from every `auto` arm on the same rule M5's native D3D11VA and M6's +//! native VAAPI rungs follow (`video::native_vulkan_gate` is where that lives). //! //! **Codec dispatch:** the negotiated codec picks the decoder ONCE, at construction -//! ([`Codec`]) — H.264 or H.265, the two codecs pf-vkdecode speaks. The negotiated -//! picture SHAPE (chroma format + bit depth) is checked there too, against the -//! device: an H.265 session this GPU has no decode format for is refused at +//! ([`Codec`]) — H.264, H.265 or AV1, the three codecs pf-vkdecode speaks. The +//! negotiated picture SHAPE (chroma format + bit depth) is checked there too, against +//! the device: an H.265 or AV1 session this GPU has no decode format for is refused at //! construction, where the ladder answers with FFmpeg-Vulkan, rather than at the //! first AU, where the only exit is an error streak PAST that rung //! ([`NativeVulkanDecoder::new`]). Nothing below the codec enum is per-codec: the //! shipped-frame ledger, the release tokens, the //! decode-status reads, the timeline waits and the teardown drain are shared, because -//! both decoders deliver the identical [`DecodedVkFrame`] contract (same pool/slot +//! all three decoders deliver the identical [`DecodedVkFrame`] contract (same pool/slot //! lifecycle, same `value + 1` write-back, same query slots, same generations). //! Forking that machinery per codec would fork the one part of this backend hardware //! has already proven. //! +//! **One AU, several FRAMES (AV1 only).** An AV1 access unit is a TEMPORAL UNIT and may +//! carry more than one frame — the vendored conformance vector puts 274 frames in 250 +//! units. [`VkAv1Decoder::decode`] walks them all internally and hands back the first +//! picture the planner declared DISPLAYABLE; the rest come out of `take_ready`, which +//! this backend already drains for H.265's burst output. Two AV1 facts make that walk +//! invisible from here, and both are the decoder's doing rather than this module's: +//! a HIDDEN frame (`show_frame = 0`) is decoded but never declared an output, so it +//! never enters `take_ready` and can never be shipped; and a `show_existing_frame` +//! (`dpb.stored == None`) decodes nothing at all and merely declares an +//! already-decoded picture displayable. So the contract this backend keeps is +//! unchanged — ONE access unit in, at most one displayable frame out. +//! +//! That contract is the SPEC's, not an assumption about punktfunk hosts: AV1 admits +//! exactly one shown frame per temporal unit, and the 24 two-frame units of the +//! vendored vector are a hidden ALTREF plus the frame that shows — one output +//! between them (`pf_bitstream::av1`'s conformance golden pins `shown = 250` across +//! 250 units, with `show_existing = 0`). [`NativeVulkanDecoder::decode`]'s +//! deliverable bound is therefore defence in depth against a stream that is NOT +//! that — a non-conformant encoder, or a scalable stream whose temporal unit carries +//! several operating points — and not the routine case it would be if a +//! `show_existing_frame` could ride alongside a shown frame. It cannot. +//! //! **A skipped RASL picture is NOT a decode error.** An HEVC stream joined at a CRA //! carries leading pictures whose references precede the join; the spec's own answer //! (8.1.3 NOTE) is to decode and output nothing for them. [`VkH265Decoder::decode`] @@ -33,6 +58,22 @@ //! the field today — punktfunk hosts emit IDR-only re-entry points — but it is the //! contract pf-bitstream's `h265` module docs record for this wiring.) //! +//! AV1's post-failure wait is NOT that shape, and the difference is deliberate. +//! After a failed frame the decoder empties its own slot ledger and skips every +//! frame until the next key frame (`VkAv1Decoder::awaiting_key`), because the +//! planner's eight-slot store still believes the flushed pictures are resident. But +//! a temporal unit in which every frame was skipped comes back as an ERROR +//! (`VkDecodeError::AwaitingKeyAv1`), once per access unit — exactly what H.264 and +//! H.265 answer for the same wait through their planners' `PlanError::AwaitingIdr`, +//! and for a reason this module owns: an `Ok(None)` with an empty warning ledger is +//! read here as a CLEAN access unit, and a clean AU clears `video.rs`'s demotion +//! streak. A rung whose every key frame fails would then never demote — one error +//! per key frame, zeroed by the skipped frames between them — and the `!delivered` +//! fall-through to FFmpeg-Vulkan, the documented backstop for a level above +//! `maxLevelIdc`, a sequence header disagreeing with the Welcome and (AV1 only) +//! film grain, would be unreachable. All three codecs demote identically here, and +//! only the H.26x paths have hardware evidence. +//! //! **Queue lock:** pf-vkdecode submits on queue 0 of the decode family //! ([`DECODE_QUEUE_INDEX`] — the presenter creates exactly one queue per family). When //! the decode family IS the presenter's graphics family, that is the very `VkQueue` the @@ -86,13 +127,33 @@ //! damaged and I coped" and "I could not run" are opposite statements about the //! rung, and only the second one means the session is looking at a frozen screen. //! -//! Because concealment is not an error, it must not clear the demotion streak -//! either — `video::Decoder::decode_frame` leaves the streak untouched on a -//! concealed `Ok(None)` and resets it only on a shipped frame or a clean AU. -//! Otherwise a driver failing every other AU on a lossy link has its errors zeroed -//! by the concealment between them, and a rung that conceals FOREVER (a host -//! framing regression: every AU damaged, no frame ever shipped) has no escape -//! hatch at all. +//! **AV1 answers a LOST REFERENCE as a refusal, not as concealment**, and that is the +//! codec's doing rather than a policy difference here. AV1's reference array is indexed +//! by reference NAME, so a lost reference leaves a HOLE and there is no legal substitute +//! to write into it — `-1` for a name the frame really references is a spec violation +//! whose firmware behaviour is undefined, so [`VkAv1Decoder::decode`] refuses the AU +//! (`MissingReferenceAv1`) instead of concealing. The refusal counts in +//! [`DecodeHealth::refused`], the `Err` sets `want_keyframe` through `video::Decoder`'s +//! own error arm, and the decoder then skips to the next key frame — answering an +//! `Err` for every access unit of that wait, so the streak keeps ticking until the +//! re-anchor lands (the paragraph above). What must not be done is to launder either +//! answer into a concealment, or into a clean AU: the pictures really were not +//! decoded, and reporting "damaged, coped" — or "nothing to object to" — would put a +//! clean-looking bill of health on a rung that produced no picture. +//! +//! **The invariant all of the above serves: an answer may clear the demotion +//! streak only if it PROVES the rung works.** `video::Decoder::decode_frame` resets +//! the streak on a shipped frame or a CLEAN access unit, and on nothing else — so +//! every state in which this backend produces no picture has to reach it as either +//! a concealment (`Ok(None)` + a recovery request) or an `Err`, never as a clean +//! `Ok(None)`. Concealment is therefore left untouched by the reset (otherwise a +//! driver failing every other AU on a lossy link has its errors zeroed by the +//! concealment between them, and a rung that conceals FOREVER — a host framing +//! regression: every AU damaged, no frame ever shipped — has no escape hatch at +//! all), and the AV1 key-frame wait is an `Err` rather than the clean `Ok(None)` it +//! superficially resembles. The one genuinely clean `Ok(None)` is the decoder that +//! ran and had nothing to object to: it buffered, or it skipped an H.265 RASL +//! picture after an open-GOP join. //! //! Neither can storm, for two independent reasons. The ask is throttled to one per //! 100 ms per session whatever the damage rate; and once the freeze is armed the gate @@ -124,7 +185,8 @@ use anyhow::{anyhow, bail, Result}; use pf_vkdecode::ash::vk; use pf_vkdecode::ash::vk::Handle as _; use pf_vkdecode::{ - DecodeStatus, DecodedVkFrame, DeviceHandles, VkDecodeError, VkH264Decoder, VkH265Decoder, + DecodeStatus, DecodedVkFrame, DeviceHandles, VkAv1Decoder, VkDecodeError, VkH264Decoder, + VkH265Decoder, }; use std::sync::mpsc; use std::time::{Duration, Instant}; @@ -182,27 +244,30 @@ impl pf_vkdecode::QueueLock for NativeQueueLock { /// The codecs pf-vkdecode has a decoder for — the native rung's whole vocabulary, /// named ash-free so `video.rs` can pick one from the negotiated wire codec without /// this module knowing about FFmpeg's codec ids (and `video::native_vulkan_gate` -/// stays the single admission decision). AV1 is deliberately absent: the Vulkan -/// decode op exists and real hardware advertises it, but there is no AV1 decoder in -/// pf-vkdecode, so those sessions must keep falling through to the FFmpeg rungs. +/// stays the single admission decision). +/// +/// Being IN this enum is not the same as being in `auto`: `Av1` is pin-only until it +/// has hardware evidence, and `video::native_vulkan_gate` — not this list — is where +/// that decision lives. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum NativeCodec { H264, H265, + Av1, } /// The decoder this backend drives, chosen ONCE from the negotiated codec. /// /// Dispatch stops here. Everything the backend does around a decoder — the /// shipped-frame ledger, release tokens, status-query settling, timeline waits, -/// teardown — is codec-agnostic, because [`VkH264Decoder`] and [`VkH265Decoder`] -/// expose the same surface over the same [`DecodedVkFrame`] contract (same pool/slot -/// lifecycle, same `value + 1` write-back, same query slots, same generations). The -/// forwarders below are therefore mechanically identical per arm on purpose: the -/// H.264 path is hardware-verified bit-exact, and dispatch must not be able to change -/// its behaviour. -// Unboxed on purpose, against `large_enum_variant`: the arms differ by ~1.7 KB (both -// decoders carry a planner, a slot ledger and pinned Std parameter sets), and exactly +/// teardown — is codec-agnostic, because [`VkH264Decoder`], [`VkH265Decoder`] and +/// [`VkAv1Decoder`] expose the same surface over the same [`DecodedVkFrame`] contract +/// (same pool/slot lifecycle, same `value + 1` write-back, same query slots, same +/// generations). The forwarders below are therefore mechanically identical per arm on +/// purpose: the H.264 path is hardware-verified bit-exact, and dispatch must not be +/// able to change its behaviour. +// Unboxed on purpose, against `large_enum_variant`: the arms differ by ~1.7 KB (every +// decoder carries a planner, a slot ledger and pinned Std parameter sets), and exactly // ONE of these exists per session — inside the `Box` the backend // already lives in. So the "waste" is 1.7 KB of slack in a single session-lifetime // allocation, while boxing would put a second indirection between the pump and the @@ -212,25 +277,41 @@ pub(crate) enum NativeCodec { enum Codec { H264(VkH264Decoder), H265(VkH265Decoder), + Av1(VkAv1Decoder), } impl Codec { /// Feed one access unit — see [`VkH264Decoder::decode`] / - /// [`VkH265Decoder::decode`]. `Ok(None)` means "no display-ready picture from - /// this AU", which for H.265 also covers a RASL picture skipped after an - /// open-GOP join (the module doc's contract: never an error). + /// [`VkH265Decoder::decode`] / [`VkAv1Decoder::decode`]. `Ok(None)` means "no + /// display-ready picture from this AU", which for H.265 also covers a RASL + /// picture skipped after an open-GOP join (the module doc's contract: never an + /// error). + /// + /// What `Ok(None)` deliberately does NOT cover on any arm is a decoder waiting + /// to re-anchor after a failure: H.264/H.265 answer that with their planners' + /// `PlanError::AwaitingIdr` and AV1 with `VkDecodeError::AwaitingKeyAv1`, one + /// `Err` per access unit for as long as the wait lasts. A clean `Ok(None)` + /// would clear the demotion streak once per frame and strand the session on a + /// rung that produces nothing (module doc). + /// + /// AV1 is the one arm where "an access unit" is not "a frame": its AU is a + /// TEMPORAL UNIT, the decoder walks every frame in it, and what comes back is + /// the FIRST displayable picture of the walk — the rest, if any, through + /// [`Self::take_ready`], exactly like H.265's burst output. fn decode(&mut self, au: &[u8]) -> Result, VkDecodeError> { match self { Codec::H264(d) => d.decode(au), Codec::H265(d) => d.decode(au), + Codec::Av1(d) => d.decode(au), } } - /// Drain the plan warnings of the AU just decoded, TYPED — the two planners + /// Drain the plan warnings of the AU just decoded, TYPED — the three planners /// have genuinely different enums ([`pf_vkdecode::PlanWarning`] has /// `FrameNumGap`/`Mmco5Rebase`, [`pf_vkdecode::H265PlanWarning`] has - /// `NonZeroReorder`, neither a subset of the other), so the pair is carried as - /// a two-armed value rather than flattened. + /// `NonZeroReorder`, [`pf_vkdecode::Av1PlanWarning`] has `MissingShowExisting`, + /// none a subset of another), so the set is carried as a three-armed value + /// rather than flattened. /// /// Typed and not rendered because the backend must BRANCH on them: only some /// warnings mean the picture is damaged ([`PlanWarnings::integrity`]), and @@ -241,15 +322,23 @@ impl Codec { match self { Codec::H264(d) => PlanWarnings::H264(d.take_warnings()), Codec::H265(d) => PlanWarnings::H265(d.take_warnings()), + // The AV1 decoder concatenates the WHOLE temporal unit's warnings, in + // decode order — one unit, one concealment verdict, which is what this + // backend already assumes for an AU. + Codec::Av1(d) => PlanWarnings::Av1(d.take_warnings()), } } /// Pull the next already display-ready frame the last AU did not return - /// directly (burst output). + /// directly (H.265 burst output; on AV1 only a non-conformant or + /// multi-operating-point unit, since the spec admits one shown frame per + /// temporal unit — [`MAX_DELIVERABLE`]). Drained after EVERY decode, so a + /// burst can never be stranded inside the decoder. fn take_ready(&mut self) -> Option { match self { Codec::H264(d) => d.take_ready(), Codec::H265(d) => d.take_ready(), + Codec::Av1(d) => d.take_ready(), } } @@ -263,6 +352,7 @@ impl Codec { match self { Codec::H264(d) => d.release_frame(frame, presented), Codec::H265(d) => d.release_frame(frame, presented), + Codec::Av1(d) => d.release_frame(frame, presented), } } @@ -272,6 +362,7 @@ impl Codec { match self { Codec::H264(d) => d.generation(), Codec::H265(d) => d.generation(), + Codec::Av1(d) => d.generation(), } } @@ -280,6 +371,7 @@ impl Codec { match self { Codec::H264(d) => d.poll_status(frame), Codec::H265(d) => d.poll_status(frame), + Codec::Av1(d) => d.poll_status(frame), } } @@ -289,6 +381,7 @@ impl Codec { match self { Codec::H264(d) => d.wait_decoded(frame, timeout_ns), Codec::H265(d) => d.wait_decoded(frame, timeout_ns), + Codec::Av1(d) => d.wait_decoded(frame, timeout_ns), } } @@ -299,15 +392,19 @@ impl Codec { match self { Codec::H264(d) => d.status_queries(), Codec::H265(d) => d.status_queries(), + Codec::Av1(d) => d.status_queries(), } } /// The newest planned picture's DECODE-order ordinal — the watermark the /// pump stamps when it arms a freeze (see [`NativeVkFrame::decode_order`]). + /// On AV1 a `show_existing_frame` does not advance it, because it decodes + /// nothing — which is what the watermark is comparing against. fn decode_order(&self) -> u64 { match self { Codec::H264(d) => d.decode_order(), Codec::H265(d) => d.decode_order(), + Codec::Av1(d) => d.decode_order(), } } } @@ -346,7 +443,7 @@ impl StatusVerdicts { /// The plan warnings one AU produced, still in their codec's own enum. /// -/// The split that matters is INTEGRITY vs. spec-legal, not H.264 vs. H.265. Both +/// The split that matters is INTEGRITY vs. spec-legal, not which codec. The /// planners emit two kinds of warning through one channel: /// /// - **Integrity** — a reference the DPB does not hold, a `frame_num` gap, an AU @@ -364,9 +461,18 @@ impl StatusVerdicts { /// excludes `NonZeroReorder` from its integrity set for exactly this reason. /// /// Everything is logged either way; only integrity warnings drop the frame. +/// +/// AV1 (M7) has an EMPTY right-hand column: its planner reports nothing that is +/// spec-legal-but-notable, because AV1 has no reorder envelope to announce (no +/// bumping process, no `max_num_reorder_pics`) and no MMCO to rebase. Every AV1 +/// warning is damage, and `pf_vkdecode::is_integrity_warning_av1` says so +/// exhaustively so a warning added later cannot default to "clean". The branch +/// below is therefore not dead code on that arm — it is the place a future +/// spec-legal AV1 warning would land without costing a frame. enum PlanWarnings { H264(Vec), H265(Vec), + Av1(Vec), } impl PlanWarnings { @@ -374,6 +480,7 @@ impl PlanWarnings { match self { PlanWarnings::H264(w) => w.is_empty(), PlanWarnings::H265(w) => w.is_empty(), + PlanWarnings::Av1(w) => w.is_empty(), } } @@ -400,6 +507,12 @@ impl PlanWarnings { .cloned() .collect(), ), + PlanWarnings::Av1(w) => PlanWarnings::Av1( + w.iter() + .filter(|x| pf_vkdecode::is_integrity_warning_av1(x)) + .cloned() + .collect(), + ), } } @@ -407,6 +520,7 @@ impl PlanWarnings { match self { PlanWarnings::H264(w) => w.len(), PlanWarnings::H265(w) => w.len(), + PlanWarnings::Av1(w) => w.len(), } } @@ -437,12 +551,21 @@ impl PlanWarnings { "native decode planned with concealment — dropping the frame, \ requesting re-anchor" ), + PlanWarnings::Av1(w) => tracing::warn!( + concealed, + warnings = ?w, + "native decode planned with concealment — dropping the frame, \ + requesting re-anchor" + ), } } /// The spec-legal log: the planner flagged an envelope fact and planned the AU /// in full, so the frame is SHOWN. Rare by construction (SPS activation, MMCO - /// 5), which is why it is a `warn` and not a per-frame `debug`. + /// 5), which is why it is a `warn` and not a per-frame `debug`. Unreachable on + /// the AV1 arm today — every AV1 warning is damage — and spelled out anyway so + /// a future spec-legal AV1 warning gets the same treatment rather than the + /// concealment branch's. fn warn_planned_in_full(&self) { match self { PlanWarnings::H264(w) => tracing::warn!( @@ -455,6 +578,11 @@ impl PlanWarnings { "native decode: spec-legal envelope signal — the AU was planned in \ full and the frame is kept" ), + PlanWarnings::Av1(w) => tracing::warn!( + warnings = ?w, + "native decode: spec-legal envelope signal — the AU was planned in \ + full and the frame is kept" + ), } } } @@ -559,25 +687,32 @@ fn project_frame(frame: &DecodedVkFrame, guard: NativeReleaseGuard) -> NativeVkF } } -/// The picture format an H.265 session of the negotiated shape decodes to, or a named -/// refusal for a shape pf-vkdecode has no output format for at all. +/// The picture format a session of the negotiated shape decodes to, or a named +/// refusal for a shape pf-vkdecode has no output format for at all. `codec` names the +/// codec in the refusal text and nothing else — the map is the CRATE's one +/// (sampling, depth) → format table, shared by every codec it decodes. /// -/// The DEVICE-INDEPENDENT half of [`NativeVulkanDecoder::new`]'s shape check: 4:2:2 -/// and 12-bit are legal H.265 that no punktfunk host emits and this client has no -/// plumbing for, so no driver has to be asked about them. Pure, so the refusal is -/// CPU-testable — the device-dependent half (a shape with a format that THIS driver -/// does not advertise) is [`VkH265Decoder::probe_stream_support`], covered by -/// pf-vkdecode's `derive_caps_h265` refusal tests. -fn h265_picture_format(stream: crate::video::StreamFormat) -> Result { +/// The DEVICE-INDEPENDENT half of [`NativeVulkanDecoder::new`]'s shape check: 4:2:2, +/// monochrome and 12-bit are legal H.265/AV1 that no punktfunk host emits and this +/// client has no plumbing for, so no driver has to be asked about them. Pure, so the +/// refusal is CPU-testable — the device-dependent half (a shape with a format that +/// THIS driver does not advertise) is `probe_stream_support`, covered by +/// pf-vkdecode's `derive_caps_h265`/`derive_caps_av1` refusal tests. +/// +/// One function for both codecs because the ENVELOPE is identical: pf-vkdecode's +/// AV1 profile builder admits exactly the four (sampling, depth) pairs +/// [`pf_vkdecode::output_format_for`] maps, and refusing here in different terms +/// than the probe refuses one line later would be two gates to keep in agreement. +fn picture_format(codec: &str, stream: crate::video::StreamFormat) -> Result { let depth = stream.bit_depth_minus8().ok_or_else(|| { anyhow!( - "negotiated HEVC bit depth {} is outside the 8/10-bit decode envelope", + "negotiated {codec} bit depth {} is outside the 8/10-bit decode envelope", stream.bit_depth ) })?; pf_vkdecode::output_format_for(stream.chroma_format_idc, depth).ok_or_else(|| { anyhow!( - "no native picture format for the negotiated HEVC stream shape \ + "no native picture format for the negotiated {codec} stream shape \ (chroma_format_idc={}, {}-bit)", stream.chroma_format_idc, stream.bit_depth @@ -585,6 +720,137 @@ fn h265_picture_format(stream: crate::video::StreamFormat) -> Result }) } +/// The film-grain flag the AV1 construction-time probe asks the device about. +/// +/// Grain synthesis is part of the AV1 decode PROFILE — a device that decodes AV1 +/// need not offer the grain-enabled one — and the negotiation carries no grain bit, +/// so this is the one probe input that is an ASSUMPTION rather than a negotiated +/// fact. `false` is the right assumption and the safe one: +/// +/// * a punktfunk host encodes desktop capture, where film-grain synthesis is off +/// (it exists to re-add grain a denoiser removed from camera footage); +/// * and the failure directions are not symmetric. Probing `false` on a device that +/// only offers the grain profile would REFUSE a session it could have run — but +/// there is no such device (grain support is an added capability, never a +/// replacement). Probing `true` on the far more common device that offers only +/// the grain-LESS profile would refuse every session this rung can actually +/// decode. +/// +/// If a grain stream ever does arrive, `ensure_state` re-keys from the SEQUENCE +/// header (never softened to make a query pass) and refuses at the first AU — which +/// lands on the "never delivered a frame" arm in [`crate::video::Decoder`], the same +/// backstop that already covers a level above `maxLevelIdc` and a sequence header +/// disagreeing with the Welcome. +const AV1_PROBE_FILM_GRAIN: bool = false; + +/// What the CLIENT PIPELINE itself holds, at its worst moment: how many delivered +/// frames are unreleased between this backend and the screen at once. +/// +/// pf-vkdecode's [`pf_vkdecode::HOLD_HEADROOM`] docs enumerate them — two bounded(2) +/// channels, the FrameStore's 1..=3 preroll, the in-flight present, the retired-frame +/// slot — as 4-7 at steady state. Taken at the MAXIMUM, because a bound derived from +/// the average is a bound that fails exactly when it is needed. +const PIPELINE_HOLD: usize = 7; + +/// How many display-ready frames the backend will hold back for LATER access units +/// before it starts dropping the oldest (see [`trim_deliverable`]). +/// +/// **Derived, not chosen.** [`pf_vkdecode::HOLD_HEADROOM`] is the TOTAL number of +/// delivered-but-unreleased frames the pool is sized for (`picture_count = +/// required_slots + HOLD_HEADROOM`), and a queued frame counts against it exactly +/// like a shipped one: `build_frame` increments the picture's `held` the moment the +/// decoder declares it ready, and it stays held until [`Codec::release_frame`]. So +/// the queue's share of the headroom is whatever the pipeline does not already +/// occupy, and a queue bounded any deeper does not prevent the failure it names — +/// it merely caps the memory while the pool runs out anyway (8 queued + 7 in flight +/// against a headroom of 8 is `NoFreeSlot` on the next AU). +/// +/// The other bound it has to stay inside is the STATUS-QUERY ring, which is +/// `picture_count` deep (17 on AV1: nine DPB slots plus the headroom) and is +/// re-armed once per SUBMISSION — up to two per temporal unit. A frame's query is +/// first read the AU after it ships, so a frame that waits `MAX_DELIVERABLE` access +/// units in this queue burns roughly `2 * (MAX_DELIVERABLE + 1)` of those 17 slots +/// before anyone looks at it. Overrun the ring and `read_status` reports the +/// re-armed slot as `Failed`, which [`NativeVulkanDecoder::settle_statuses`] +/// attributes to `driver_failed` — a FABRICATED driver-corruption verdict polluting +/// the one signal [`DecodeHealth::failed`] exists to carry (the Xbox Ally X class). +/// At the derived depth the wait is ~4 of 17 and the question does not arise. +/// +/// The queue exists because a decoder can make several pictures display-ready from +/// one AU while the caller takes exactly one per AU: H.265 bumping outputs a burst +/// after a reordering stretch. AV1 cannot — the spec admits exactly one shown frame +/// per temporal unit, and pf-bitstream's conformance golden pins it (250 shown +/// frames across 250 units, no `show_existing_frame` at all) — so on that codec this +/// is defence in depth against a non-conformant or multi-operating-point stream, not +/// a routine case. Either way punktfunk hosts reorder nothing, so on the wire the +/// queue is empty every single AU and the bound never engages. +/// +/// It is a bound and not a plain queue because "transient" is an assumption about the +/// HOST, and the failure it fails into is silent: a stream that reliably made two +/// frames displayable per AU would grow this by one per AU until the pool ran out — +/// after which every AU refuses with `NoFreeSlot`, three in a second demote the rung, +/// and nothing in the log would say the cause was a queue that could never drain. +/// +/// What the derived depth costs, stated plainly: a stream that really does bump a +/// burst of more than two pictures at once loses the middle of the burst rather than +/// queueing it. That is the right way round. The frames are already several AUs late +/// by the time a burst exists, the stage after this one is newest-wins anyway, and +/// the alternative — a queue deep enough to hold the burst — spends the pool's whole +/// headroom on it and answers `NoFreeSlot` on the next access unit, which is a frozen +/// screen and a demotion rather than a hitch. Reachable only on a reordering stream, +/// which punktfunk hosts do not emit and which the planner already flags +/// (`H265PlanWarning::NonZeroReorder`). +const MAX_DELIVERABLE: usize = pf_vkdecode::HOLD_HEADROOM as usize - PIPELINE_HOLD; + +// The derivation must leave the queue able to do its job: carry the one frame a +// two-output access unit strands. A `PIPELINE_HOLD` raised to the headroom (or past +// it) would silently turn every burst into a dropped frame — or underflow the const. +const _: () = assert!( + MAX_DELIVERABLE >= 1, + "the deliverable queue must be able to carry at least one frame between AUs" +); + +/// One `warn` per this many dropped deliverable frames, after the first. The shape +/// that drops at all drops on EVERY access unit, and a warn per frame at frame rate +/// buries the log it exists to explain — while a single line at the start of a +/// session that then goes quiet reads as a one-off. So: the first drop in full, then +/// a heartbeat with the running total (~every 5 s at 60 fps). +const DROP_WARN_EVERY: u64 = 300; + +/// Trim the deliverable queue to `cap` by dropping from the FRONT, returning the +/// dropped frames so the caller can release them unshown. +/// +/// Oldest-first, because by the time a queue this deep exists the front frame is +/// several AUs stale and the consumer one stage on is itself newest-wins (the pump's +/// `force_send` overwrites an unconsumed frame). Dropping the NEWEST would keep the +/// stalest picture and present the stream in ever-lagging order; dropping the oldest +/// keeps display order for everything that survives and costs the frames that were +/// already too late to matter. +/// +/// ⚠ Called AFTER this AU's own frame has been taken off the front, so `cap` bounds +/// the CARRY-OVER — what is held back for later access units — exactly as +/// [`MAX_DELIVERABLE`] says. Trimming before the take would make an AU that produced +/// two outputs drop the FIRST of them and ship the second, which is display order +/// inverted inside a single access unit. +/// +/// Pure over the queue, so the bound is CPU-testable without a GPU. +fn trim_deliverable( + queue: &mut std::collections::VecDeque, + cap: usize, +) -> Vec { + let mut dropped = Vec::new(); + while queue.len() > cap { + match queue.pop_front() { + Some(frame) => dropped.push(frame), + // Unreachable: `len() > cap >= 0` means the queue is non-empty. Written + // as a break rather than an `expect` so a bound of 0 on an empty queue + // could never be a panic in the decode path. + None => break, + } + } + dropped +} + /// The native backend: the decoder plus the shipped-frame ledger and release channel. pub(crate) struct NativeVulkanDecoder { dec: Codec, @@ -593,8 +859,10 @@ pub(crate) struct NativeVulkanDecoder { /// the last guard is gone — the teardown short-circuit signal. release_tx: Option>, release_rx: mpsc::Receiver, - /// Display-ready frames not yet handed to the pump (burst outputs — decode - /// delivers one per call; the rest wait here, oldest first). + /// Display-ready frames not yet handed to the pump (an H.265 burst output, or a + /// temporal unit that declared more pictures displayable than AV1 permits — + /// decode delivers one per call; the rest wait here, oldest first, bounded by + /// [`MAX_DELIVERABLE`]). Every frame in here holds a picture-pool image. deliverable: std::collections::VecDeque, outstanding: Vec, next_seq: u64, @@ -639,19 +907,22 @@ impl NativeVulkanDecoder { /// /// The difference is which rung a refusal lands on. pf-vkdecode's picture format /// is the STREAM's (Main → NV12, Main 10 → P010, RExt 4:4:4 → the two-plane 4:4:4 - /// formats) and a device that advertises H.265 decode need not advertise a format - /// for every shape of it: 4:4:4 is absent everywhere but NVIDIA. Discovered + /// formats) and a device that advertises H.265 or AV1 decode need not advertise a + /// format for every shape of it: 4:4:4 is absent everywhere but NVIDIA. Discovered /// lazily, that is a mid-stream ERROR STREAK, and the streak machinery demotes a /// Vulkan rung to VAAPI/D3D11VA — PAST FFmpeg-Vulkan, which on NVIDIA/Linux (no /// usable VAAPI) means a 4K HEVC session lands on SOFTWARE. Refused here it is an /// ordinary construction failure, and `video::Decoder::new` falls through to /// FFmpeg-Vulkan — the rung that session ran on before this backend existed. /// - /// Two legs the probe cannot see, because they are stream facts no negotiation - /// carries: a level above the device's `maxLevelIdc`, and an SPS that disagrees - /// with the Welcome. Those still surface at the first decode — and are caught by - /// the "never delivered a frame" arm in [`crate::video::Decoder::decode_frame`], - /// which routes exactly that state to FFmpeg-Vulkan instead of past it. + /// Three legs the probe cannot see, because they are stream facts no negotiation + /// carries: a level above the device's `maxLevelIdc`, an SPS (or AV1 sequence + /// header) that disagrees with the Welcome, and — AV1 only — a sequence that + /// enables FILM GRAIN, which is part of the decode profile and which the probe + /// therefore has to assume ([`AV1_PROBE_FILM_GRAIN`]). All three still surface at + /// the first decode — and are caught by the "never delivered a frame" arm in + /// [`crate::video::Decoder::decode_frame`], which routes exactly that state to + /// FFmpeg-Vulkan instead of past it. /// /// H.264 is deliberately NOT probed: its envelope is fixed at 8-bit 4:2:0, so the /// only fact a probe could add is a profile idc guess — on the one path in this @@ -709,7 +980,7 @@ impl NativeVulkanDecoder { // The device-independent half of the shape check, first: a stream // shape pf-vkdecode has NO picture format for (4:2:2, 12-bit) needs // no driver to refuse it. - let wanted = h265_picture_format(stream)?; + let wanted = picture_format("HEVC", stream)?; // SAFETY: the handle contract stated directly above. let d = unsafe { VkH265Decoder::new(&handles, lock) } .map_err(|e| anyhow!("VkH265Decoder init: {e}"))?; @@ -719,7 +990,7 @@ impl NativeVulkanDecoder { // timing differs, and the timing is the whole point. let depth = stream .bit_depth_minus8() - .expect("h265_picture_format accepted the depth"); + .expect("picture_format accepted the depth"); d.probe_stream_support(stream.chroma_format_idc, depth) .map_err(|e| { anyhow!( @@ -731,6 +1002,36 @@ impl NativeVulkanDecoder { })?; Codec::H265(d) } + NativeCodec::Av1 => { + // Exactly the H.265 shape check, one codec over — AV1's decode + // profile is a (seq_profile, sampling, depth, film grain) tuple and + // a device that advertises the AV1 decode OPERATION need not offer + // every profile of it. Refused here, the ladder answers with + // FFmpeg-Vulkan; discovered at the first AU, the only exit is an + // error streak PAST that rung. + let wanted = picture_format("AV1", stream)?; + // SAFETY: the handle contract stated directly above. + let d = unsafe { VkAv1Decoder::new(&handles, lock) } + .map_err(|e| anyhow!("VkAv1Decoder init: {e}"))?; + d.probe_stream_support( + stream.chroma_format_idc, + // AV1's profile key takes the ABSOLUTE bit depth (8/10), not + // H.265's `bit_depth_luma_minus8` — the two probes really do + // want different numbers, and `picture_format` above is the + // one that proved this depth is in the envelope at all. + stream.bit_depth, + AV1_PROBE_FILM_GRAIN, + ) + .map_err(|e| { + anyhow!( + "device cannot decode the negotiated AV1 stream shape \ + (chroma_format_idc={}, {}-bit, needs {wanted:?}): {e}", + stream.chroma_format_idc, + stream.bit_depth + ) + })?; + Codec::Av1(d) + } }; let (release_tx, release_rx) = mpsc::channel(); let status_queries = dec.status_queries(); @@ -807,14 +1108,25 @@ impl NativeVulkanDecoder { /// Feed one complete access unit. /// + /// One access unit, at most one DISPLAYABLE frame out. On AV1 the access unit is + /// a temporal unit and the decoder may decode several frames from it — hidden + /// frames included, which are never declared displayable and therefore never + /// reach this ledger at all. Anything a single AU makes displayable beyond the + /// first waits in [`Self::deliverable`] for the next call, bounded by + /// [`MAX_DELIVERABLE`]. + /// /// `Ok(Some)` = a display-ready picture. `Ok(None)` = no picture this AU, which /// covers three unrelated things and the caller treats all three the same /// (its no-output/re-anchor machinery, exactly as for FFmpeg): the decoder /// buffered without output, an H.265 RASL picture was skipped after an open-GOP /// join, or the AU's plan needed CONCEALMENT and its output was released - /// unshown. `Err` = the DECODER is in trouble — a Vulkan/session error, or a - /// driver `RESULT_STATUS` verdict of Failed on a prior frame — which the - /// caller's streak/demotion machinery is entitled to act on. + /// unshown. `Err` = the DECODER is in trouble — a Vulkan/session error, an AV1 + /// reference the plan could not resolve (which AV1 refuses rather than + /// conceals — module doc), an AV1 temporal unit skipped in full while the + /// decoder waits for the next key frame (the H.26x planners' `AwaitingIdr` + /// under another name), or a driver `RESULT_STATUS` verdict of Failed on a + /// prior frame — which the caller's streak/demotion machinery is entitled to + /// act on. /// /// That split is the M4 recovery policy and it is deliberate (module doc): /// concealment says the STREAM lost data, not that this decoder is failing, so @@ -892,6 +1204,38 @@ impl NativeVulkanDecoder { let delivered = match self.dec.decode(au) { Ok(delivered) => delivered, Err(e) => { + // NOTHING from a refused AU reaches the screen — the same rule the + // concealment branch below keeps, and it has to be enforced here + // too because a refusal can STRAND a frame inside the decoder. + // AV1's `decode_inner` settles each plan of a multi-frame temporal + // unit in turn, so frame 1 can already sit in `ready` when frame 2 + // fails; left there, the NEXT access unit pops it out of + // `take_ready` and ships it with an empty warning ledger. That + // would put a picture from a REFUSED unit on screen, clear the + // demotion streak with it, and set `video.rs`'s `delivered` — + // permanently disabling the never-delivered fall-through to + // FFmpeg-Vulkan, which is the backstop for exactly the stream + // shapes that refuse every AU. + // + // On H.264/H.265 this also catches the pictures `recover_dpb` + // FLUSHES out of the DPB on the AU after a failure (that AU then + // answers `AwaitingIdr`, so it lands here). Releasing them costs + // nothing observable: they were decoded BEFORE the loss, they + // arrive while the pump's freeze is armed, and the freeze gate + // withholds a non-keyframe there anyway — `session.rs` goes further + // and discards their recovery marks by `decode_order` for exactly + // this reason. The pool image comes back an access unit sooner. On + // a punktfunk stream the set is empty regardless: zero reorder means + // the DPB buffers no output to flush. + while let Some(frame) = self.dec.take_ready() { + if let Err(e) = self.dec.release_frame(&frame, false) { + tracing::debug!(error = %e, "releasing a stranded frame failed"); + } + } + // …and the unit's warnings go with it. They describe a plan whose + // frames were all released unshown; carried over, the NEXT AU would + // read them as its own fresh damage. + let _ = self.dec.take_warnings(); let verdicts = self.settle_statuses(); self.health.note(false, true, verdicts.total()); tracing::warn!( @@ -950,7 +1294,38 @@ impl NativeVulkanDecoder { } self.deliverable.extend(fresh); - Ok(self.deliverable.pop_front().map(|frame| self.ship(frame))) + // This AU's frame comes off the FRONT first: the bound is on the CARRY-OVER + // (see [`trim_deliverable`]), so a unit that produced two outputs ships the + // first and holds the second rather than dropping the first to ship the + // second. + let shipped = self.deliverable.pop_front().map(|frame| self.ship(frame)); + // The queue can only ever hand ONE frame per AU to the caller, so anything + // it cannot drain is a frame holding a pool image forever — see + // [`MAX_DELIVERABLE`]. Inert on every stream a punktfunk host emits. + let queued = self.deliverable.len(); + for frame in trim_deliverable(&mut self.deliverable, MAX_DELIVERABLE) { + self.health.note_dropped(); + // Rate-limited, because the shape this fires on is a stream producing a + // surplus frame on EVERY access unit: unthrottled that is a warn per + // frame at frame rate, which buries the log it is supposed to explain. + // The first one carries the diagnosis; the rest are a running count. + // `queued` is the PRE-trim depth — the number that says how far past the + // bound the queue actually got. Read after the trim it would be the + // constant `MAX_DELIVERABLE` every single time. + if self.health.dropped == 1 || self.health.dropped % DROP_WARN_EVERY == 0 { + tracing::warn!( + queued, + dropped_total = self.health.dropped, + poc = frame.poc, + "native decode: more display-ready frames than the pump can take — \ + dropping the oldest so its pool image is not held forever" + ); + } + if let Err(e) = self.dec.release_frame(&frame, false) { + tracing::debug!(error = %e, "releasing an over-queued frame failed"); + } + } + Ok(shipped) } /// Wrap a delivered [`DecodedVkFrame`] for the presenter and enter it into the @@ -1440,10 +1815,13 @@ mod tests { fn a_stream_shape_with_no_native_picture_format_is_refused_at_construction() { use crate::video::StreamFormat; let f = |chroma, bit_depth| { - h265_picture_format(StreamFormat { - chroma_format_idc: chroma, - bit_depth, - }) + picture_format( + "HEVC", + StreamFormat { + chroma_format_idc: chroma, + bit_depth, + }, + ) }; // What the envelope DOES admit resolves, and to the right format — Main, // Main 10 and both RExt 4:4:4 depths. @@ -1452,7 +1830,7 @@ mod tests { assert_eq!(f(3, 8).unwrap(), pf_vkdecode::YUV444_8); assert_eq!(f(3, 10).unwrap(), pf_vkdecode::YUV444_10); assert_eq!( - h265_picture_format(StreamFormat::SDR_420_8).unwrap(), + picture_format("HEVC", StreamFormat::SDR_420_8).unwrap(), pf_vkdecode::NV12, "the default/older-host shape is the ordinary one" ); @@ -1469,6 +1847,184 @@ mod tests { assert!(f(3, 6).is_err()); } + /// AV1's construction-time shape gate is the SAME envelope, and it has to stay + /// that way: [`picture_format`] is what refuses first, and one line later + /// `VkAv1Decoder::probe_stream_support` builds an `Av1ProfileKey`, which refuses + /// exactly the same set (monochrome, 4:2:2, the planner's 4:4:0 sentinel, any + /// depth but 8/10). Two gates that disagreed would mean either a shape refused + /// here that the device could have decoded, or — worse — a shape admitted here + /// and then refused mid-stream, where the exit is an error streak past + /// FFmpeg-Vulkan. + /// + /// The label is checked too, because it is the only thing a support engineer + /// reading the refusal has to tell an AV1 session's refusal from an HEVC one. + #[test] + fn the_av1_shape_gate_admits_exactly_what_pf_vkdecodes_av1_profile_key_admits() { + use crate::video::StreamFormat; + let f = |chroma, bit_depth| { + picture_format( + "AV1", + StreamFormat { + chroma_format_idc: chroma, + bit_depth, + }, + ) + }; + // AV1 Main 8-bit / Main 10 / High 4:4:4 at both depths — every combination + // `Av1ProfileKey::from_negotiated` maps to a profile. + assert_eq!(f(1, 8).unwrap(), pf_vkdecode::NV12); + assert_eq!(f(1, 10).unwrap(), pf_vkdecode::P010); + assert_eq!(f(3, 8).unwrap(), pf_vkdecode::YUV444_8); + assert_eq!(f(3, 10).unwrap(), pf_vkdecode::YUV444_10); + // …and the ones it refuses. + assert!(f(0, 8).is_err(), "monochrome"); + assert!(f(2, 8).is_err(), "4:2:2"); + assert!( + f(4, 8).is_err(), + "the planner's 4:4:0 sentinel is not 4:4:4" + ); + assert!(f(1, 12).is_err(), "12-bit"); + assert!( + f(1, 0).is_err(), + "an absurd depth refuses, never underflows" + ); + + // The refusal names the codec — the label is the whole reason this function + // takes one. + let err = format!("{:#}", f(2, 8).unwrap_err()); + assert!(err.contains("AV1"), "{err}"); + let hevc = format!( + "{:#}", + picture_format( + "HEVC", + StreamFormat { + chroma_format_idc: 2, + bit_depth: 8 + } + ) + .unwrap_err() + ); + assert!(hevc.contains("HEVC"), "{hevc}"); + + // The probe's OWN gate, asked the same questions through pf-vkdecode's + // profile key: this is the agreement the comment above claims, asserted + // rather than assumed. + for (chroma, depth) in [(1u8, 8u8), (1, 10), (3, 8), (3, 10)] { + assert!( + pf_vkdecode::Av1ProfileKey::from_negotiated(chroma, depth, AV1_PROBE_FILM_GRAIN) + .is_ok(), + "{chroma}/{depth} passes here, so it must pass the probe's key too" + ); + } + for (chroma, depth) in [(0u8, 8u8), (2, 8), (4, 8), (1, 12), (1, 0)] { + assert!( + pf_vkdecode::Av1ProfileKey::from_negotiated(chroma, depth, AV1_PROBE_FILM_GRAIN) + .is_err(), + "{chroma}/{depth} refuses here, so the probe's key must refuse it too" + ); + } + } + + /// The deliverable queue can only hand ONE frame per AU to the pump, and every + /// frame waiting in it pins a picture-pool image. A stream that made two + /// pictures displayable per access unit would grow it by one per AU until the + /// pool ran out, after which every AU refuses with `NoFreeSlot`, three in a + /// second demote the rung, and nothing in the log would name a queue that could + /// never drain as the cause. + /// + /// ⚠ Which stream that is, precisely — the earlier claim here was wrong and the + /// crate's own golden disproves it. AV1 permits exactly ONE shown frame per + /// temporal unit, so a `show_existing_frame` can never ride alongside a shown + /// frame; pf-bitstream's conformance test pins `shown = 250` across 250 units + /// with `show_existing = 0`, and its 24 two-frame units are a hidden ALTREF plus + /// the frame that shows it. The real producers are H.265 bumping after a + /// reordering stretch, and — as defence in depth — a non-conformant or + /// multi-operating-point AV1 stream. The bound is worth having for those; it is + /// not the routine case. + /// + /// So the bound drops from the FRONT: by the time the queue is this deep the + /// oldest frame is several AUs stale, and the stage after this one (the pump's + /// `force_send`) is itself newest-wins. Dropping the newest instead would keep + /// the stalest picture and present the stream in ever-lagging order. + /// + /// ⚠ What this exercises is [`trim_deliverable`] ALONE — the pure half. The + /// wiring it cannot see is the caller's: that the trim runs AFTER this AU's + /// frame is taken off the front, that every dropped frame is handed to + /// `release_frame(.., false)`, and that [`DecodeHealth::dropped`] counts it. + /// Replace those with `mem::forget` and this test stays green; only a device + /// (or the `NoFreeSlot` a leaked pool image eventually produces) would notice. + #[test] + fn the_deliverable_queue_drops_its_oldest_rather_than_pinning_pool_images_forever() { + let mut q: std::collections::VecDeque = (0..5) + .map(|i| { + let mut f = decoded(pf_vkdecode::NV12, vk::ImageLayout::VIDEO_DECODE_DST_KHR, 1); + // The only field this test reads — distinct per frame so "which + // ones were dropped" is decidable rather than merely counted. + f.poc = i; + f + }) + .collect(); + + let dropped = trim_deliverable(&mut q, 3); + assert_eq!( + dropped.iter().map(|f| f.poc).collect::>(), + vec![0, 1], + "the OLDEST two come back for release — not the newest" + ); + assert_eq!( + q.iter().map(|f| f.poc).collect::>(), + vec![2, 3, 4], + "…and what survives stays in display order" + ); + + // At or below the bound nothing moves: on every stream a punktfunk host + // emits this queue is empty, and the bound must be invisible there. + assert!(trim_deliverable(&mut q, 3).is_empty()); + assert_eq!(q.len(), 3); + + // The bound is DERIVED, and this is the arithmetic it is derived from: a + // queued frame holds a picture-pool image exactly like a shipped one, and + // pf-vkdecode sizes that pool at `required_slots + HOLD_HEADROOM`. So the + // queue at its bound PLUS what the pipeline itself holds must fit inside + // the headroom — otherwise the pool runs out, every AU refuses with + // `NoFreeSlot`, and the bound caps memory without preventing the failure it + // names. Pinned against pf-vkdecode's own constant so a hardcoded depth here + // (this shipped at 8, against a headroom of 8) fails the build rather than a + // field session. + assert!( + MAX_DELIVERABLE + PIPELINE_HOLD <= pf_vkdecode::HOLD_HEADROOM as usize, + "a queue of {MAX_DELIVERABLE} on top of the pipeline's {PIPELINE_HOLD} \ + exceeds the {} frames the picture pool is sized for", + pf_vkdecode::HOLD_HEADROOM + ); + + // The PRODUCTION bound, at the carry-over depth it is derived to: one AU's + // surplus frame is held (the burst this queue exists for), a second AU's is + // not. Asserted against `MAX_DELIVERABLE` itself so a change to + // `PIPELINE_HOLD` lands here rather than in a field log. + let mut q: std::collections::VecDeque = (0..MAX_DELIVERABLE) + .map(|i| { + let mut f = decoded(pf_vkdecode::NV12, vk::ImageLayout::VIDEO_DECODE_DST_KHR, 1); + f.poc = i as i32; + f + }) + .collect(); + assert!( + trim_deliverable(&mut q, MAX_DELIVERABLE).is_empty(), + "a queue AT the bound is exactly what a two-output AU leaves behind" + ); + assert_eq!(q.len(), MAX_DELIVERABLE); + + // A zero bound drains rather than looping or panicking. Reachable only + // through a caller that asks for it — teardown does NOT come through here + // (`Drop` empties the queue with `mem::take` and releases each frame), so + // this pins termination and the empty-queue edge, not a production path. + let drained = q.len(); + assert_eq!(trim_deliverable(&mut q, 0).len(), drained); + assert!(q.is_empty()); + assert!(trim_deliverable(&mut q, 0).is_empty(), "and it terminates"); + } + #[test] fn release_tokens_mark_their_frame_and_tolerate_strays() { let mut outstanding = vec![shipped(0, 1), shipped(1, 1)]; @@ -1656,6 +2212,58 @@ mod tests { assert_eq!(mixed.integrity().len(), 1); } + /// The AV1 arm of the same split (M7). AV1's planner has no spec-legal + /// companion to `NonZeroReorder`/`Mmco5Rebase` — it announces no reorder + /// envelope and has no MMCO to rebase — so every warning it emits is damage and + /// every one of them must conceal. + /// + /// Worth asserting despite being "all true", because the failure it catches is + /// silent: an arm wired to the wrong predicate (or to an empty vector) would + /// SHOW a picture the stream lost data for and ask for no re-anchor, which is + /// exactly the invisible-damage shape this program exists to end. The one that + /// carries most of the weight is `MissingShowExisting` — a frame that decoded + /// nothing and displayed nothing — because it is the one an author is most + /// likely to read as harmless. + #[test] + fn every_av1_warning_conceals_because_av1_has_no_spec_legal_signal() { + use pf_vkdecode::Av1PlanWarning as Av1; + + for w in [ + Av1::MissingReference { + slot: 3, + ref_index: 1, + }, + Av1::MissingShowExisting { slot: 5 }, + Av1::TruncatedAu { offset: 900 }, + ] { + let warnings = PlanWarnings::Av1(vec![w.clone()]); + assert!(!warnings.is_empty()); + assert_eq!( + warnings.integrity().len(), + 1, + "{w:?} means the picture is not fit to present" + ); + } + + // The whole vocabulary at once — the count the concealment log reports is + // the damage count, and here it is the full list. + let all = PlanWarnings::Av1(vec![ + Av1::MissingReference { + slot: 0, + ref_index: 0, + }, + Av1::MissingShowExisting { slot: 1 }, + Av1::TruncatedAu { offset: 4 }, + ]); + assert_eq!((all.len(), all.integrity().len()), (3, 3)); + + // A clean AU is clean: the AV1 arm must not manufacture concealment out of + // an empty ledger, which is what a stream with no damage produces on every + // single access unit. + assert!(PlanWarnings::Av1(Vec::new()).is_empty()); + assert!(PlanWarnings::Av1(Vec::new()).integrity().is_empty()); + } + /// The counter a support engineer reads first. A total alone cannot tell a /// lossy link that keeps recovering apart from a stream that went down and /// stayed down — `damaged 40 · run 0` and `damaged 40 · run 40` are the same diff --git a/crates/pf-vkdecode/src/decoder.rs b/crates/pf-vkdecode/src/decoder.rs index 87287f3d..7dad434f 100644 --- a/crates/pf-vkdecode/src/decoder.rs +++ b/crates/pf-vkdecode/src/decoder.rs @@ -234,6 +234,26 @@ pub enum VkDecodeError { /// produce. `ref_index` is the AV1 reference name (`LAST_FRAME` = 0 through /// `ALTREF_FRAME` = 6), `slot` the reference slot it pointed at. MissingReferenceAv1 { slot: u8, ref_index: u8 }, + /// Every frame of this AV1 temporal unit was SKIPPED because the decoder is + /// waiting for the next key frame after a failure — nothing decoded, nothing + /// displayed. + /// + /// AV1's answer to [`pf_bitstream::h264::PlanError::AwaitingIdr`], and + /// deliberately the same KIND of answer: an error, once per access unit, for + /// as long as the wait lasts. The AV1 planner has no `flush`, so the wait is + /// held in [`crate::VkAv1Decoder`] rather than in the planner — but a consumer + /// must not be able to tell the two codecs apart here, because the consumer's + /// demotion streak is what turns "this rung produces no picture" into "fall + /// through to the next rung". Answering the wait with a clean `Ok(None)` + /// instead RESETS that streak once per frame, and a rung whose every key frame + /// fails (a film-grain sequence on a device without the grain profile, a level + /// above `maxLevelIdc`, a sequence header disagreeing with the negotiation) + /// then never demotes at all: one error per key frame, cleared by the inter + /// frames between them, and a frozen screen for the whole session. + /// + /// A key frame ANYWHERE in the unit clears the wait and decodes, so this is + /// returned only when the unit produced nothing at all. + AwaitingKeyAv1, /// Plan-to-Vulkan conversion failed (caller/session bugs; `CapacityMismatch` /// is consumed internally by the rebuild path and only surfaces if the rebuilt /// session STILL mismatches). @@ -302,6 +322,11 @@ impl std::fmt::Display for VkDecodeError { no picture — the surviving references would renumber" ) } + VkDecodeError::AwaitingKeyAv1 => write!( + f, + "every frame of this AV1 temporal unit was skipped — the decoder is \ + waiting for the next key frame after a failure" + ), VkDecodeError::Convert(e) => write!(f, "plan conversion failed: {e}"), VkDecodeError::ConvertH265(e) => write!(f, "H.265 plan conversion failed: {e}"), VkDecodeError::Caps(e) => write!(f, "decode capabilities unusable: {e}"), diff --git a/crates/pf-vkdecode/src/decoder_av1.rs b/crates/pf-vkdecode/src/decoder_av1.rs index b62cb291..f6508af2 100644 --- a/crates/pf-vkdecode/src/decoder_av1.rs +++ b/crates/pf-vkdecode/src/decoder_av1.rs @@ -539,6 +539,25 @@ pub(crate) fn lost_reference(warnings: &[PlanWarning]) -> Option<(u8, u8)> { }) } +/// What [`VkAv1Decoder::decode_planned`] did with one frame of a temporal unit. +/// +/// Two outcomes rather than a bare `Ok(())`, because "the plan was honoured" and +/// "the decoder is waiting for a key frame and did nothing" are opposite +/// statements about the rung, and the caller has to count the second: a unit in +/// which EVERY frame was skipped produced no picture at all and comes back as +/// [`VkDecodeError::AwaitingKeyAv1`], while a unit where a key frame cleared the +/// wait partway through decoded normally (see [`VkAv1Decoder::awaiting_key`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FrameOutcome { + /// The plan was carried out: submitted, or (a `show_existing_frame`) settled + /// into a display verdict without a submission. Either way the unit produced + /// this frame. + Decoded, + /// Skipped: the decoder is waiting for the next key frame after a failure and + /// this frame is undecodable by construction. + SkippedAwaitingKey, +} + /// Everything tied to ONE AV1 session generation. A stream renegotiation (extent /// or profile — including a bit-depth, sampling or film-grain switch) retires it /// and builds fresh. @@ -606,21 +625,33 @@ pub struct VkAv1Decoder { /// ([`RecoveryLatch`] docs for the whole argument). recovery: RecoveryLatch, /// Every frame until the next KEY frame is undecodable, and is skipped rather - /// than failed. + /// than converted. /// /// This exists because AV1's planner has no `flush`: when a failure forces /// [`Self::recover_dpb`] to empty this decoder's slot ledger and image /// bindings, the PLANNER's own eight-slot store still believes those pictures /// are resident and keeps handing out inter frames that reference them. Each - /// would fail in `plan_to_vk_av1` with `UnresolvedReference` — a real error - /// per frame, at frame rate, which reads to the integration layer as a decoder - /// that has stopped working rather than a stream waiting to re-anchor. + /// would fail in `plan_to_vk_av1` with `UnresolvedReference` — a per-frame + /// failure whose message describes a phantom reference gap rather than the + /// wait that is really in progress, and which would drag every one of those + /// frames through a conversion that cannot succeed. /// - /// So the frames between the failure and the key frame are ANSWERED like the - /// H.265 decoder answers a RASL picture after an open-GOP join: `Ok` with - /// whatever was already display-ready, planner untouched, no error and no - /// second keyframe request. A key frame (which references nothing and refreshes - /// all eight slots) clears it and decoding resumes. + /// So the frames are skipped. What they are NOT is laundered into a clean + /// answer: a temporal unit in which every frame was skipped comes back as + /// [`VkDecodeError::AwaitingKeyAv1`], once per access unit, exactly as the + /// H.264/H.265 decoders answer the same wait with their planners' + /// `PlanError::AwaitingIdr`. The three codecs must be indistinguishable here, + /// because the consumer's demotion streak is the only thing that turns "this + /// rung produces no picture" into "fall through to the next rung": a clean + /// `Ok(None)` RESETS that streak once per frame, so a rung whose every key + /// frame fails would never reach the threshold and the session would keep a + /// frozen screen with a clean bill of health. During a recovery wait the + /// decoder really has stopped working, and that is what the streak must see. + /// + /// A DECODED key frame (which references nothing and refreshes all eight + /// slots) clears it and decoding resumes — including one that arrives partway + /// through a temporal unit, which is why the skip is per FRAME while the error + /// is per ACCESS UNIT. awaiting_key: bool, } @@ -704,9 +735,11 @@ impl VkAv1Decoder { /// Returns the next display-ready frame, if the planner declared one; drain the /// rest with [`Self::take_ready`]. /// - /// A frame skipped while [`Self::awaiting_key`] is set is NOT an error (its - /// docs carry the argument); nor is a `show_existing_frame` naming an empty - /// slot, which the planner reports as a warning and which simply displays + /// A temporal unit whose every frame was skipped while [`Self::awaiting_key`] + /// is set comes back as [`VkDecodeError::AwaitingKeyAv1`] — the same kind of + /// answer the H.264/H.265 decoders give for the same wait, and for the reason + /// [`Self::awaiting_key`]'s docs carry. A `show_existing_frame` naming an empty + /// slot is NOT that: the planner reports it as a warning and it simply displays /// nothing. /// /// Never panics. `VkDecodeError::DeviceLost` latches: every later call fails @@ -746,22 +779,40 @@ impl VkAv1Decoder { self.last_warnings.extend(plan.warnings.iter().cloned()); } + let mut skipped = 0usize; for plan in &plans { // From here the PLANNER has already advanced past this frame — its // store holds the picture whatever happens next — so any failure below // leaves the planner's store and this decoder's ledgers able to // disagree. Latch the recovery rather than returning into a permanently // wedged state. - if let Err(e) = self.decode_planned(plan, au) { - self.recovery.latch(); - return Err(e); + match self.decode_planned(plan, au) { + Ok(FrameOutcome::Decoded) => {} + Ok(FrameOutcome::SkippedAwaitingKey) => skipped += 1, + Err(e) => { + self.recovery.latch(); + return Err(e); + } } } + // Nothing in this unit decoded and nothing was displayed, because the + // decoder is still waiting for a key frame. That is an ERROR per access + // unit — [`VkDecodeError::AwaitingKeyAv1`] and [`Self::awaiting_key`] carry + // the argument — and deliberately not a latch: `recover_dpb` has already + // run, the ledgers are consistent, and re-latching would re-flush an empty + // ledger once per frame for the whole wait. + // + // Counted rather than short-circuited inside the loop, because a key frame + // may sit BEHIND a skipped frame in the same temporal unit: returning at + // the first skip would never reach it, and the wait would never end. + if whole_unit_skipped(plans.len(), skipped) { + return Err(VkDecodeError::AwaitingKeyAv1); + } Ok(self.ready.pop_front()) } /// One planned frame of a temporal unit. - fn decode_planned(&mut self, plan: &AuPlan, au: &[u8]) -> Result<(), VkDecodeError> { + fn decode_planned(&mut self, plan: &AuPlan, au: &[u8]) -> Result { // A key frame re-anchors everything: it references nothing and refreshes // all eight slots, so it is decodable no matter what came before. // @@ -779,7 +830,7 @@ impl VkAv1Decoder { show_existing = plan.dpb.stored.is_none(), "frame skipped while awaiting the next AV1 key frame" ); - return Ok(()); + return Ok(FrameOutcome::SkippedAwaitingKey); } // `show_existing_frame`: no decode at all. It displays a slot's contents — @@ -792,7 +843,9 @@ impl VkAv1Decoder { state.slots.release(id); } } - return Ok(()); + // Decoded: nothing was submitted, but the plan was HONOURED — it + // declared a picture displayable, which is a frame the unit produced. + return Ok(FrameOutcome::Decoded); }; // A reference the planner could not resolve: refuse before anything is @@ -1031,7 +1084,7 @@ impl VkAv1Decoder { state.pool.pictures[entry.image].pending = false; } } - Ok(()) + Ok(FrameOutcome::Decoded) } /// Apply one plan's DPB verdicts: outputs become ready frames (their images @@ -1622,6 +1675,28 @@ fn clears_awaiting_key(plan: &AuPlan) -> bool { plan.picture.is_key && plan.dpb.stored.is_some() } +/// Did a temporal unit of `planned` frames produce NOTHING because every one of +/// them was skipped waiting for a key frame — the +/// [`VkDecodeError::AwaitingKeyAv1`] condition? +/// +/// A named function rather than the expression inlined at the call site because +/// both of its edges are load-bearing and neither is obvious: +/// +/// * `planned == 0` is not a skip. A temporal unit can plan no frames at all (one +/// carrying only metadata or a sequence header), and that is an ordinary +/// `Ok(None)` — turning it into an error would fail access units on a perfectly +/// healthy stream. +/// * `skipped < planned` is not a skip either, and this is the case an early +/// return inside the loop would have got wrong: a key frame may sit BEHIND a +/// skipped frame in the same unit, clears the wait when it is reached, and +/// decodes. Reporting the unit as skipped there would answer an error for an +/// access unit that really did decode a picture. +/// +/// Pure, so the aggregation is CPU-testable without a device. +fn whole_unit_skipped(planned: usize, skipped: usize) -> bool { + planned > 0 && skipped == planned +} + /// The stream's level, as the sequence header's FIRST operating point states it. /// /// Operating point 0 is the full stream — the one a non-scalable decoder decodes @@ -2789,6 +2864,57 @@ mod tests { assert!(!clears_awaiting_key(&inter)); } + /// A recovery WAIT must reach the consumer as an ERROR, once per access unit — + /// the same answer H.264/H.265 give through their planners' + /// `PlanError::AwaitingIdr`, and the reason [`VkAv1Decoder::awaiting_key`]'s + /// docs carry: a clean `Ok(None)` resets the consumer's demotion streak once + /// per frame, so a rung whose every key frame fails (film grain on a device + /// without the grain profile; a level above `maxLevelIdc`; a sequence header + /// disagreeing with the negotiation) would never demote and the session would + /// hold a frozen screen with a clean bill of health. + /// + /// What this pins is the AGGREGATION, which is where the naive fix goes wrong: + /// the error is per ACCESS UNIT while the skip is per FRAME, because a key + /// frame can sit behind a skipped frame in the same temporal unit — the + /// vendored vector has 24 units carrying two frames each. + #[test] + fn a_unit_reports_the_key_frame_wait_only_when_it_decoded_nothing_at_all() { + // The wait itself: every frame of the unit skipped. + assert!(whole_unit_skipped(1, 1), "a single-frame unit"); + assert!(whole_unit_skipped(2, 2), "and a two-frame one"); + + // A key frame arrived partway through the unit and decoded: NOT the wait, + // whatever came before it. An early return at the first skip would have + // answered an error here and never reached the key frame at all. + assert!(!whole_unit_skipped(2, 1)); + assert!(!whole_unit_skipped(3, 2)); + // Nothing was skipped: the ordinary decoding case. + assert!(!whole_unit_skipped(2, 0)); + // A unit that planned no frames (metadata / a sequence header on its own) + // is a clean `Ok(None)`, never an error. + assert!(!whole_unit_skipped(0, 0)); + } + + /// The wait's error must be DISTINGUISHABLE from the failure that started it — + /// a support engineer reading a field log has to be able to tell "the AU could + /// not be decoded" from "the decoder is waiting to re-anchor", and the two ride + /// the same `Err` channel. + #[test] + fn the_key_frame_wait_names_itself_in_the_error_text() { + let waiting = format!("{}", VkDecodeError::AwaitingKeyAv1); + assert!(waiting.contains("key frame"), "{waiting}"); + assert!(waiting.contains("skipped"), "{waiting}"); + // …and it is not the same message as the loss that latched the recovery. + let lost = format!( + "{}", + VkDecodeError::MissingReferenceAv1 { + slot: 3, + ref_index: 2 + } + ); + assert_ne!(waiting, lost); + } + /// The `refresh_frame_flags == 0` leg is real AV1 and this vector has none of /// it — which is worth PROVING rather than assuming, because it is exactly the /// sort of "cannot happen" that quietly exhausts a nine-slot ledger in the diff --git a/crates/pf-vkdecode/src/integrity.rs b/crates/pf-vkdecode/src/integrity.rs index 14c57195..35892e85 100644 --- a/crates/pf-vkdecode/src/integrity.rs +++ b/crates/pf-vkdecode/src/integrity.rs @@ -1,7 +1,7 @@ //! Which planner warnings mean the PICTURE is damaged (M4 of the native-decode //! program). //! -//! Both planners emit two very different kinds of thing through one warning +//! The planners emit two very different kinds of thing through one warning //! channel, and the split is what a consumer must branch on: //! //! * **Integrity** — a reference the DPB does not hold, a `frame_num` gap, an AU @@ -24,7 +24,7 @@ //! does not actually perform — the exact shape of the `nb_queries = 0` failure the //! program exists to end. -use crate::{H265PlanWarning, PlanWarning}; +use crate::{Av1PlanWarning, H265PlanWarning, PlanWarning}; /// Does this H.264 planner warning mean the PICTURE is damaged? /// @@ -59,6 +59,41 @@ pub fn is_integrity_warning_h265(w: &H265PlanWarning) -> bool { } } +/// The AV1 twin (M7). Every variant the AV1 planner has today IS damage, and that +/// is a fact about the codec rather than an oversight: AV1 puts nothing in this +/// channel that resembles h265's `NonZeroReorder` or h264's `Mmco5Rebase`. It has +/// no reorder envelope to report (no bumping process, no `max_num_reorder_pics`) +/// and no MMCO to rebase — the frame header states the whole reference update +/// outright — so the only things left to warn about are pictures that went +/// missing and an OBU walk that stopped early. +/// +/// `MissingShowExisting` is the one that could be argued, and it is damage: a +/// `show_existing_frame` naming an empty slot means the picture the STREAM chose +/// to display was lost upstream. Nothing is displayed for that frame, so the +/// screen keeps the previous one — exactly the "silently stale picture" state a +/// re-anchor exists to end. +/// +/// ⚠ `MissingReference` is classified here for completeness and does NOT normally +/// reach a consumer through this predicate: [`crate::VkAv1Decoder`] refuses the +/// whole access unit for it ([`crate::VkDecodeError::MissingReferenceAv1`]), +/// because AV1's `refs` array is indexed by reference NAME and there is no legal +/// substitute to write into a hole — a `-1` for a name the frame really references +/// is a spec violation whose firmware behaviour is undefined. So the AV1 rung +/// answers a lost reference as a REFUSAL, not as concealment, and it is the +/// refusal counter that moves. Classifying it as damage here anyway keeps the two +/// statements consistent for any consumer that does see the warning (and for the +/// fault harness, which asserts detection against exactly this list). +/// +/// Exhaustive for the same reason as [`is_integrity_warning`]: a new AV1 warning +/// must not be able to mean "damaged" and read as clean. +pub fn is_integrity_warning_av1(w: &Av1PlanWarning) -> bool { + match w { + Av1PlanWarning::MissingReference { .. } + | Av1PlanWarning::MissingShowExisting { .. } + | Av1PlanWarning::TruncatedAu { .. } => true, + } +} + #[cfg(test)] mod tests { use super::*; @@ -105,4 +140,32 @@ mod tests { every ABR renegotiation's IDR" ); } + + /// AV1's whole warning vocabulary is damage. Note plainly what this test does + /// and does not guard, because the two are easy to confuse: + /// + /// * A NEW variant is caught by the EXHAUSTIVE MATCH in + /// [`is_integrity_warning_av1`], not here — this loop enumerates the variants + /// by hand, so a fourth one would simply not appear in it. That is the whole + /// reason the function is written as a match with no `_` arm. + /// * What this test does guard is a RECLASSIFICATION: split one of these names + /// out of the `|` chain and give it a `false` arm — the shape a future + /// "spec-legal AV1 signal" would arrive in — and the assertion below fires. + /// `MissingShowExisting` is the one most likely to be argued down that way (a + /// frame that decoded nothing and displayed nothing reads as harmless), and + /// reading it as clean would leave the previous picture on the screen with no + /// re-anchor asked for. + #[test] + fn every_av1_warning_is_damage_because_av1_has_no_envelope_signal() { + for w in [ + Av1PlanWarning::MissingReference { + slot: 3, + ref_index: 1, + }, + Av1PlanWarning::MissingShowExisting { slot: 5 }, + Av1PlanWarning::TruncatedAu { offset: 900 }, + ] { + assert!(is_integrity_warning_av1(&w), "{w:?} is damage"); + } + } } diff --git a/crates/pf-vkdecode/src/lib.rs b/crates/pf-vkdecode/src/lib.rs index 635310e8..986bb597 100644 --- a/crates/pf-vkdecode/src/lib.rs +++ b/crates/pf-vkdecode/src/lib.rs @@ -96,10 +96,10 @@ //! [`DecodedVkFrame::recovery`]). The only clean point an intra-refresh session //! has — its wave emits no IDR — so without it a client freezes for its full //! backstop and then forces the very IDR the wave exists to avoid. -//! - [`integrity`]: [`is_integrity_warning`] / [`is_integrity_warning_h265`], the -//! one list of warnings that mean the PICTURE is damaged. Here rather than in the -//! client so the fault harness asserts against the predicate production conceals -//! on. +//! - [`integrity`]: [`is_integrity_warning`] / [`is_integrity_warning_h265`] / +//! [`is_integrity_warning_av1`], the one list of warnings that mean the PICTURE +//! is damaged. Here rather than in the client so the fault harness asserts +//! against the predicate production conceals on. //! - [`fault`]: [`AuFault`], deliberate decoder-input corruption //! (`PUNKTFUNK_AU_FAULT`), inert unless armed. A detector nobody can fire is //! exactly as trustworthy as no detector at all. @@ -209,6 +209,7 @@ pub use images::plan_pools; pub use images::PoolPlan; pub use images::HOLD_HEADROOM; pub use integrity::is_integrity_warning; +pub use integrity::is_integrity_warning_av1; pub use integrity::is_integrity_warning_h265; pub use params::pps_to_std; pub use params::sps_to_std;