diff --git a/clients/session/README.md b/clients/session/README.md index 8cf1dcfd..e9104632 100644 --- a/clients/session/README.md +++ b/clients/session/README.md @@ -51,11 +51,12 @@ only, no Skia anywhere in the dependency tree. Decode follows the Settings preference (auto is vendor-ordered: hardware Vulkan Video → VAAPI → software on Linux, hardware Vulkan Video → D3D11VA → software on Windows, with -VAAPI/D3D11VA first on Intel; on H.264 the native pf-vkdecode Vulkan decoder is tried -immediately before FFmpeg-Vulkan): the Vulkan decoders run on the presenter's own +VAAPI/D3D11VA first on Intel; on H.264 and HEVC the native pf-vkdecode Vulkan decoder +is tried immediately before FFmpeg-Vulkan): the Vulkan decoders run on the presenter's own device where the stack supports it (every vendor, zero copy); VAAPI dmabufs import -per-plane elsewhere (D3D11VA textures on Windows); software is the universal fallback. 10-bit Main10 and HDR10 are advertised -(`VIDEO_CAP_10BIT|HDR`): P010 decodes through all three paths, and PQ streams present +per-plane elsewhere (D3D11VA textures on Windows); software is the universal fallback. +10-bit Main10 and HDR10 are advertised (`VIDEO_CAP_10BIT|HDR`): P010 decodes through the +native, FFmpeg-Vulkan, VAAPI/D3D11VA and software paths alike, and PQ streams present on an HDR10/ST.2084 swapchain when the desktop offers one (KDE HDR, gamescope) or tone-map in-shader to SDR when it doesn't (`PUNKTFUNK_TONEMAP_PEAK` tunes the rolloff, default ≈1000 nits). The host still gates the upgrade behind its `PUNKTFUNK_10BIT` diff --git a/crates/pf-client-core/Cargo.toml b/crates/pf-client-core/Cargo.toml index 1d4b2d51..271de520 100644 --- a/crates/pf-client-core/Cargo.toml +++ b/crates/pf-client-core/Cargo.toml @@ -17,10 +17,10 @@ repository.workspace = true punktfunk-core = { path = "../punktfunk-core", features = ["quic"] } # FFmpeg's Vulkan hwcontext surface (Vulkan Video decode on the presenter's device). pf-ffvk = { path = "../pf-ffvk" } -# Native Vulkan Video H.264 decode (WP-C of the native-decode program): auto's rung -# immediately above FFmpeg-Vulkan (2026-08-05 ladder decision), also pinnable via -# `PUNKTFUNK_DECODER=native-vulkan` — video_vk_native.rs, running pf-vkdecode's -# VkH264Decoder on the presenter's shared device. +# Native Vulkan Video decode (WP-C of the native-decode program, HEVC added by M3 +# WP-2): auto's rung immediately above FFmpeg-Vulkan (2026-08-05 ladder decision), also +# pinnable via `PUNKTFUNK_DECODER=native-vulkan` — video_vk_native.rs, running +# pf-vkdecode's VkH264Decoder/VkH265Decoder on the presenter's shared device. pf-vkdecode = { path = "../pf-vkdecode" } async-channel = "2" diff --git a/crates/pf-client-core/src/lib.rs b/crates/pf-client-core/src/lib.rs index 33e638ad..4e100303 100644 --- a/crates/pf-client-core/src/lib.rs +++ b/crates/pf-client-core/src/lib.rs @@ -77,10 +77,11 @@ mod video_software; mod video_libav; #[cfg(target_os = "linux")] mod video_vaapi; -// Native Vulkan Video H.264 decode (WP-C of the native-decode program): pf-vkdecode's -// decoder on the presenter's shared device — auto's rung immediately above -// FFmpeg-Vulkan (2026-08-05 ladder decision; the program is dropping FFmpeg from the -// client), also pinnable via `PUNKTFUNK_DECODER=native-vulkan`. +// Native Vulkan Video decode (WP-C of the native-decode program, HEVC added by M3 +// WP-2): pf-vkdecode's H.264/H.265 decoders on the presenter's shared device — auto's +// rung immediately above FFmpeg-Vulkan (2026-08-05 ladder decision; the program is +// dropping FFmpeg from the client), also pinnable via +// `PUNKTFUNK_DECODER=native-vulkan`. #[cfg(any(target_os = "linux", windows))] mod video_vk_native; #[cfg(any(target_os = "linux", windows))] diff --git a/crates/pf-client-core/src/session.rs b/crates/pf-client-core/src/session.rs index 9748e926..9723bcfa 100644 --- a/crates/pf-client-core/src/session.rs +++ b/crates/pf-client-core/src/session.rs @@ -442,6 +442,14 @@ fn pump( // Build the decoder for the codec the host resolved (never assume HEVC), honoring the // Settings backend preference (auto/vaapi/software). let codec_id = crate::video::ffmpeg_codec_id(connector.codec); + // The picture shape the host RESOLVED (not what we asked for) — the native + // Vulkan rung probes its device against it at construction, so a 4:4:4 or + // Main 10 session that this GPU has no decode format for refuses BEFORE it is + // chosen instead of error-streaking past FFmpeg-Vulkan mid-stream. + let stream_format = crate::video::StreamFormat { + chroma_format_idc: connector.chroma_format, + bit_depth: connector.bit_depth, + }; // The WIRE codec is the negotiated truth; the FFmpeg id is meaningful only where // FFmpeg decodes it. `ffmpeg_codec_id`'s fallthrough maps every unknown wire bit — // PyroWave included — to HEVC, so logging it unconditionally claimed @@ -497,10 +505,20 @@ fn pump( )), } } else { - Decoder::new(codec_id, ¶ms.decoder, params.vulkan.as_ref()) + Decoder::new( + codec_id, + ¶ms.decoder, + params.vulkan.as_ref(), + stream_format, + ) }; #[cfg(not(all(any(target_os = "linux", windows), feature = "pyrowave")))] - let built = Decoder::new(codec_id, ¶ms.decoder, params.vulkan.as_ref()); + let built = Decoder::new( + codec_id, + ¶ms.decoder, + params.vulkan.as_ref(), + stream_format, + ); let mut decoder = match built { Ok(d) => d, Err(e) => { diff --git a/crates/pf-client-core/src/trust.rs b/crates/pf-client-core/src/trust.rs index 4625ab85..f7d93592 100644 --- a/crates/pf-client-core/src/trust.rs +++ b/crates/pf-client-core/src/trust.rs @@ -938,9 +938,10 @@ pub struct Settings { #[serde(default = "default_codec")] pub codec: String, /// Video decoder preference: `"auto"` (vendor-ordered hardware ladder — on H.264 - /// the native pf-vkdecode rung sits immediately above FFmpeg-Vulkan; then - /// VAAPI/D3D11VA, then software — see `video::Decoder::new` for the per-vendor - /// order), `"vulkan"`, `"vaapi"`, `"d3d11va"`, `"native-vulkan"`, `"software"`. + /// and HEVC the native pf-vkdecode rung sits immediately above FFmpeg-Vulkan; + /// then VAAPI/D3D11VA, then software — see `video::Decoder::new` for the + /// per-vendor order), `"vulkan"`, `"vaapi"`, `"d3d11va"`, `"native-vulkan"`, + /// `"software"`. /// The `PUNKTFUNK_DECODER` env var overrides this (see `video::Decoder::new`). pub decoder: String, /// Decode/present GPU (multi-GPU boxes): the adapter's marketing name, as the WinUI diff --git a/crates/pf-client-core/src/video.rs b/crates/pf-client-core/src/video.rs index 84f798e1..efe65d61 100644 --- a/crates/pf-client-core/src/video.rs +++ b/crates/pf-client-core/src/video.rs @@ -1,13 +1,14 @@ //! Video decode: reassembled HEVC access units → frames for the presenter. //! //! Backends, picked at session start (auto is vendor-ordered on BOTH desktop OSes — -//! see [`VulkanDecodeDevice::prefer_vulkan_first`]; on H.264 sessions the native -//! pf-vkdecode decoder (`video_vk_native`, gated by [`native_vulkan_gate`]) slots in -//! immediately ABOVE the FFmpeg-Vulkan rung wherever the ladder reaches it — the -//! program's goal is dropping FFmpeg from the client, and a native INIT failure -//! falls through to FFmpeg-Vulkan; a runtime error streak instead demotes past it, -//! same as FFmpeg-Vulkan's own streaks do — see `decode_frame`). Linux: native → -//! vulkan → vaapi → software on NVIDIA and ALL AMD (VanGogh included), vaapi → +//! see [`VulkanDecodeDevice::prefer_vulkan_first`]; on H.264 AND HEVC sessions the +//! native pf-vkdecode decoder (`video_vk_native`, gated by [`native_vulkan_gate`]) +//! slots in immediately ABOVE the FFmpeg-Vulkan rung wherever the ladder reaches it — +//! the program's goal is dropping FFmpeg from the client, and a native INIT failure +//! falls through to FFmpeg-Vulkan; a runtime error streak demotes past it, same as +//! FFmpeg-Vulkan's own streaks do — EXCEPT while the native rung has never delivered +//! a frame, which falls through to FFmpeg-Vulkan too, see `decode_frame`). Linux: +//! native → vulkan → vaapi → software on NVIDIA and ALL AMD (VanGogh included), vaapi → //! native → vulkan → software on Intel/unknown. Windows: native → vulkan → d3d11va → //! software on NVIDIA/AMD, d3d11va → native → vulkan → software on Intel/unknown. //! Override: `PUNKTFUNK_DECODER=vulkan|vaapi|d3d11va|software|native-vulkan` — @@ -52,7 +53,7 @@ pub use crate::video_color::{csc_rows, ColorDesc}; use crate::video_software::SoftwareDecoder; #[cfg(target_os = "linux")] use crate::video_vaapi::VaapiDecoder; -use crate::video_vk_native::NativeVulkanDecoder; +use crate::video_vk_native::{NativeCodec, NativeVulkanDecoder}; use crate::video_vulkan::VulkanDecoder; /// One decoded frame headed for the presenter, carrying the host capture timestamp so the @@ -88,16 +89,46 @@ pub enum DecodedImage { /// samples them directly (BT.709 limited, the codec's fixed colour contract). #[cfg(all(any(target_os = "linux", windows), feature = "pyrowave"))] PyroWave(crate::video_pyrowave::PyroWavePlanarFrame), - /// Native Vulkan Video output (pf-vkdecode — auto's H.264 rung immediately above - /// FFmpeg-Vulkan, also pinnable via `PUNKTFUNK_DECODER=native-vulkan`): - /// an NV12 image + per-plane views already on the PRESENTER's device — same + /// Native Vulkan Video output (pf-vkdecode — auto's H.264/HEVC rung immediately + /// above FFmpeg-Vulkan, also pinnable via `PUNKTFUNK_DECODER=native-vulkan`): a + /// decoded image + per-plane views already on the PRESENTER's device — same /// zero-copy contract as [`DecodedImage::VkFrame`], no FFmpeg involved. The - /// presenter waits the frame's timeline pair, transitions the layer for sampling - /// and BACK to [`NativeVkFrame::layout`], and releases the decoder's slot by - /// dropping the frame (its guard sends the release token). + /// picture format is the stream's, carried on the frame + /// ([`NativeVkFrame::vk_format`] — NV12 for H.264 and HEVC Main, P010 for Main + /// 10, the two-plane 4:4:4 formats for RExt), never assumed. The presenter waits + /// the frame's timeline pair, transitions the layer for sampling and BACK to + /// [`NativeVkFrame::layout`], and releases the decoder's slot by dropping the + /// frame (its guard sends the release token). NativeVk(NativeVkFrame), } +/// A raw `VkFormat` code point, carried across the ash-free boundary. +/// +/// A newtype rather than a bare `i32` because the two hardware frame types +/// ([`VkVideoFrame`], [`NativeVkFrame`]) carry OTHER `i32`s — `poc` foremost — and +/// the presenter's colour-math lookup takes exactly one number. Handed the wrong +/// one it compiles, warns once about an unmapped format, and renders every frame of +/// the session as 8-bit: decoded correctly, displayed wrong, silently. The wrapper +/// makes that a type error instead. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct RawVkFormat(pub i32); + +/// Every picture format the NATIVE decode lane can deliver, as raw `VkFormat` code +/// points — pf-vkdecode's own [`pf_vkdecode::OUTPUT_FORMATS`] vocabulary, not a copy +/// of it. +/// +/// It is public so the PRESENTER can pin its per-format colour-math table against the +/// real producer. pf-presenter has no pf-vkdecode dependency, so without this its only +/// available check is the FFmpeg lane's table against itself — which stays green if +/// pf-vkdecode grows a fifth output format (12-bit RExt) that the CSC pass has no +/// depth mapping for. This crate sees both, so the fact crosses here. +pub fn native_picture_formats() -> Vec { + pf_vkdecode::OUTPUT_FORMATS + .iter() + .map(|f| RawVkFormat(f.as_raw())) + .collect() +} + /// One Vulkan-decoded frame. The image lives on the presenter's own VkDevice (the /// decoder was built over its handles), so presenting is: plane views → CSC pass — no /// import, no copy. The live synchronization state (layout / timeline value / owning @@ -116,9 +147,9 @@ pub struct VkVideoFrame { /// writing back the incremented semaphore value around its submission. pub lock_frame: usize, pub unlock_frame: usize, - /// The frame pool's VkFormat (`AVVulkanFramesContext.format[0]`, raw i32) — the + /// The frame pool's VkFormat (`AVVulkanFramesContext.format[0]`) — the /// multiplanar format the presenter builds its per-plane views against. - pub vk_format: i32, + pub vk_format: RawVkFormat, /// The frame's timeline semaphore (raw VkSemaphore; creation-constant) and the /// value FFmpeg's decode submission signals on completion — the pump waits this /// pair AFTER shipping the frame to measure true GPU decode time (zero pipeline @@ -234,8 +265,21 @@ impl Drop for NativeReleaseGuard { pub struct NativeVkFrame { /// The decode image (raw `VkImage`); the picture occupies array layer [`Self::layer`]. pub image: u64, - /// `R8`/`R8G8` per-plane views (raw `VkImageView`s) — the presenter's planar CSC - /// sampling contract, same shape as the FFmpeg path's derived plane views. + /// The picture's own `VkFormat` (same shape as [`VkVideoFrame::vk_format`]): + /// what the image was created with and what [`Self::plane_views`] alias. + /// + /// Read it, never infer it from the codec. H.264 in this program is the 8-bit + /// 4:2:0 envelope, so its frames are always NV12 — but an H.265 session's format + /// is the STREAM's (Main → NV12, Main 10 → P010, RExt 4:4:4 → the two-plane 4:4:4 + /// formats) and can change mid-stream when the host renegotiates. The presenter + /// derives the CSC pass's bit depth and MSB-packing factor from this; an assumed + /// 8 bits over a P010 surface decodes correctly and displays wrong, which is the + /// failure class this program exists to refuse. + pub vk_format: RawVkFormat, + /// Per-plane views (raw `VkImageView`s) in the formats pf-vkdecode resolves for + /// [`Self::vk_format`] — `R8`/`R8G8` for the 8-bit families, `R10X6`/`R10X6G10X6` + /// for the 10-bit ones — the presenter's planar CSC sampling contract, same shape + /// as the FFmpeg path's derived plane views. pub plane_views: [u64; 2], pub layer: u32, /// The layout the layer is in when the semaphore signals; the presenter must @@ -262,13 +306,17 @@ pub struct NativeVkFrame { /// wrong window — carried so that assumption is checkable, not silent. pub crop_x: u32, pub crop_y: u32, - /// Colour signalling, read from the SPS active for THIS picture (H.264 VUI → - /// H.273 code points, with E.2.1's "unspecified" inference where the VUI is + /// Colour signalling, read from the SPS active for THIS picture (the H.264/H.265 + /// VUI → H.273 code points, with E.2.1's "unspecified" inference where the VUI is /// silent) — per frame, like the FFmpeg rungs' AVFrame CICP, because the host /// switches HDR in-band; "unspecified" resolves to the BT.709-limited SDR /// default (`csc_rows`' documented fallback). pub color: ColorDesc, - /// IDR — the stream's re-anchor point (the pump's post-loss resume signal). + /// IDR — the stream's re-anchor point (the pump's post-loss resume signal). Truly + /// IDR: on H.265 a CRA/BLA does NOT set this (pf-bitstream keys it off the NALU + /// type), which costs nothing against punktfunk hosts — they emit IDR-only + /// re-entry points — and is the conservative direction anyway, since a CRA's + /// leading pictures may be undecodable. pub keyframe: bool, pub poc: i32, /// Sends the release token on drop — see [`NativeReleaseGuard`]. @@ -395,12 +443,13 @@ impl Drop for DrmFrameGuard { enum Backend { Vulkan(VulkanDecoder), - /// Native Vulkan Video H.264 (pf-vkdecode) on the presenter's device — auto's - /// rung immediately above FFmpeg-Vulkan since the 2026-08-05 ladder decision - /// (WP-D closed bit-exact; the program's goal is dropping FFmpeg from the - /// client), also pinnable by name (`PUNKTFUNK_DECODER=native-vulkan`) — see - /// [`native_vulkan_gate`]. Errors ride the SAME streak/demotion machinery as - /// the FFmpeg-Vulkan rung. + /// Native Vulkan Video H.264/HEVC (pf-vkdecode) on the presenter's device — + /// auto's rung immediately above FFmpeg-Vulkan since the 2026-08-05 ladder + /// decision (WP-D closed bit-exact; the program's goal is dropping FFmpeg from + /// the client), also pinnable by name (`PUNKTFUNK_DECODER=native-vulkan`) — see + /// [`native_vulkan_gate`]. The negotiated codec picks the decoder once, at + /// construction; everything else about this backend is codec-agnostic. Errors + /// ride the SAME streak/demotion machinery as the FFmpeg-Vulkan rung. /// Boxed: the decoder (planner + shipped-frame ledger) dwarfs the other variants, /// same as PyroWave below. NativeVulkan(Box), @@ -417,6 +466,40 @@ enum Backend { Software(SoftwareDecoder), } +/// The picture shape the host resolved in its Welcome, before a single AU arrives. +/// +/// The in-band SPS stays authoritative — this is the NEGOTIATED answer, which is what +/// makes it available at decoder-construction time. It exists so a backend whose +/// support for a shape is device-dependent can refuse BEFORE it is chosen, where the +/// ladder's fall-through to the next rung is a plain construction failure, instead of +/// discovering it at the first decode where the only exit is an error-streak demotion +/// PAST that rung. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StreamFormat { + /// `chroma_format_idc` — [`punktfunk_core::quic::CHROMA_IDC_420`] (1) or + /// [`punktfunk_core::quic::CHROMA_IDC_444`] (3). An older host that omitted it + /// reads as 4:2:0, never 0. + pub chroma_format_idc: u8, + /// Bits per component: 8, or 10 for a Main10/HDR session (an older host reads 8). + pub bit_depth: u8, +} + +impl StreamFormat { + /// The 8-bit 4:2:0 envelope — what every H.264 session is, and what an older + /// host's Welcome decodes to. + pub const SDR_420_8: StreamFormat = StreamFormat { + chroma_format_idc: punktfunk_core::quic::CHROMA_IDC_420, + bit_depth: 8, + }; + + /// `bit_depth` as the `bit_depth_luma_minus8` the H.265 SPS (and pf-vkdecode's + /// profile key) speaks, or `None` for a depth outside the 8/10 envelope — which + /// is itself a refusal, not a "probe skipped". + pub(crate) fn bit_depth_minus8(self) -> Option { + self.bit_depth.checked_sub(8) + } +} + pub struct Decoder { backend: Backend, /// The negotiated codec (from the host's Welcome), so a mid-session VAAPI→software demotion @@ -436,6 +519,15 @@ pub struct Decoder { /// The pump drains it and asks the host — under the infinite GOP there is no periodic /// keyframe, so a rebuilt/erroring decoder would otherwise stay gray/frozen forever. want_keyframe: bool, + /// The CURRENT backend has delivered at least one frame. A backend that never did + /// is one the session never actually had, so its error streak must not cost the + /// session the rung BELOW it — see the native→FFmpeg-Vulkan arm in + /// [`Decoder::decode_frame`]. Reset on every backend swap. + delivered: bool, + /// The presenter's device, kept so that same arm can build the FFmpeg-Vulkan + /// decoder mid-stream. Cloned once per session; its handles outlive every pump + /// (see [`VulkanDecodeDevice`]). + vk: Option, /// The presenter has the win32 external-memory import path, so D3D11VA frames can reach /// the screen — kept for the mid-session Vulkan→D3D11VA demotion rung (the Windows /// analog of Linux's Vulkan→VAAPI rung). @@ -465,37 +557,68 @@ const HW_DEMOTE_MIN_STREAK: std::time::Duration = std::time::Duration::from_mill /// `VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR` — the raw flag bit within /// [`VulkanDecodeDevice::decode_video_caps`] (this crate stays ash-free). const VIDEO_CODEC_OP_DECODE_H264: u32 = 0x0000_0001; +/// `VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR` — its H.265 sibling. (AV1 is 0x4 +/// and deliberately has no constant here: pf-vkdecode has no AV1 decoder, so the bit +/// would only invite a gate that admits a session nothing can decode.) +const VIDEO_CODEC_OP_DECODE_H265: u32 = 0x0000_0002; + +/// The native decoder for a negotiated wire codec, plus the +/// `VkVideoCodecOperationFlagBitsKHR` the presenter's decode family must advertise +/// for it — or `None` for a codec pf-vkdecode cannot decode natively. +/// +/// The two are returned together on purpose: "which decoder" and "which caps bit" +/// are one fact, and splitting them is how a gate ends up admitting HEVC on an +/// H.264-only decode family (`vkCreateVideoSessionKHR` for a codec operation the +/// family cannot run is undefined behaviour, not an error). AV1 has a Vulkan decode +/// op and real hardware advertises it — but there is no AV1 decoder in pf-vkdecode, +/// so those sessions must keep falling through to the FFmpeg rungs. +fn native_codec(codec_id: ffmpeg::codec::Id) -> Option<(NativeCodec, u32)> { + match codec_id { + ffmpeg::codec::Id::H264 => Some((NativeCodec::H264, VIDEO_CODEC_OP_DECODE_H264)), + ffmpeg::codec::Id::HEVC => Some((NativeCodec::H265, VIDEO_CODEC_OP_DECODE_H265)), + _ => None, + } +} /// The native Vulkan Video admission gate (WP-C of the native-decode program, widened -/// by the 2026-08-05 ladder decision): the pf-vkdecode backend engages when `choice` -/// asks for it — by name (`PUNKTFUNK_DECODER=native-vulkan` — `choice` is env-first, -/// so that's what carries it) or as the auto family (`auto`/``/`hardware`), where -/// native is the rung immediately ABOVE FFmpeg-Vulkan: WP-D closed with bit-exact -/// parity against libavcodec (250/250 AUs on three drivers, clean 92-minute soak), -/// and the program's goal is dropping FFmpeg from the client, so native goes first -/// wherever the ladder would reach FFmpeg-Vulkan — a native INIT failure falls -/// through to that rung, so admission can't cost a session its decoder at start -/// (a runtime error streak demotes past FFmpeg-Vulkan to VAAPI/D3D11VA/software, -/// like every hardware rung's streaks do — a native→FFmpeg-Vulkan runtime rung is -/// deliberately absent; FFmpeg is on its way out). The explicit -/// `vulkan` pin still names the FFmpeg-Vulkan backend specifically; it — and every -/// other explicit backend pin — refuses. Beyond the choice: the negotiated wire codec -/// must be H.264 (the one codec pf-vkdecode speaks) and the presenter's device must -/// actually advertise Vulkan Video H.264 decode. Pure so the decision is -/// CPU-testable. +/// by the 2026-08-05 ladder decision and again by M3 WP-2's HEVC wiring): the +/// pf-vkdecode backend engages when `choice` asks for it — by name +/// (`PUNKTFUNK_DECODER=native-vulkan` — `choice` is env-first, so that's what carries +/// it) or as the auto family (`auto`/``/`hardware`), where native is the rung +/// immediately ABOVE FFmpeg-Vulkan: WP-D closed with bit-exact parity against +/// libavcodec (250/250 AUs on three drivers, clean 92-minute soak), and the program's +/// goal is dropping FFmpeg from the client, so native goes first wherever the ladder +/// would reach FFmpeg-Vulkan — a native INIT failure falls through to that rung, so +/// admission can't cost a session its decoder at start. A runtime error streak demotes +/// past FFmpeg-Vulkan to VAAPI/D3D11VA/software like every hardware rung's streaks do, +/// with ONE exception: a native backend that never delivered a single frame demotes to +/// FFmpeg-Vulkan first, because a rung the session never actually had must not cost it +/// the rung below (see [`Decoder::decode_frame`]). The explicit `vulkan` pin still +/// names the FFmpeg-Vulkan backend specifically; it — and every other explicit backend +/// pin — refuses. +/// +/// Beyond the choice: the negotiated wire codec must be one pf-vkdecode speaks — +/// H.264 or H.265 ([`native_codec`]) — and the presenter's decode family must +/// advertise THAT codec's decode operation. `video_decode` alone proves the extension +/// stack, never the codec: an AV1-only decode family exists on real hardware, and +/// H.264-only ones are the common case on older silicon. AV1 sessions refuse outright. +/// +/// What the gate deliberately does NOT check is the stream's picture SHAPE — that is +/// [`NativeVulkanDecoder::new`]'s construction-time probe, which has the negotiated +/// chroma format and bit depth and can ask the device directly. Keeping it there keeps +/// this decision pure (and CPU-testable) while still refusing before a decoder exists. fn native_vulkan_gate( choice: &str, codec_id: ffmpeg::codec::Id, video_decode: bool, decode_video_caps: u32, ) -> bool { + let Some((_, codec_op)) = native_codec(codec_id) else { + return false; + }; matches!(choice, "native-vulkan" | "auto" | "" | "hardware") - && codec_id == ffmpeg::codec::Id::H264 && video_decode - // The decode family must advertise the H264 op specifically — - // `video_decode` alone proves the extension stack, not the codec (an - // AV1-only decode family exists on real hardware). - && decode_video_caps & VIDEO_CODEC_OP_DECODE_H264 != 0 + && decode_video_caps & codec_op != 0 } /// Map a negotiated `quic` codec bit to the FFmpeg decoder id the client opens. @@ -649,10 +772,10 @@ impl Decoder { /// Precedence: the `PUNKTFUNK_DECODER` env override wins (support/debug escape /// hatch, and the documented knob), then the setting; both default to auto. /// Auto's hardware order depends on the device on BOTH desktop OSes - /// ([`VulkanDecodeDevice::prefer_vulkan_first`]); on H.264 sessions the native - /// pf-vkdecode rung sits immediately above FFmpeg-Vulkan wherever the ladder - /// reaches it ([`native_vulkan_gate`] — the program is dropping FFmpeg, and a - /// native INIT failure falls through to FFmpeg-Vulkan). Linux: native → Vulkan → + /// ([`VulkanDecodeDevice::prefer_vulkan_first`]); on H.264 and HEVC sessions the + /// native pf-vkdecode rung sits immediately above FFmpeg-Vulkan wherever the + /// ladder reaches it ([`native_vulkan_gate`] — the program is dropping FFmpeg, and + /// a native INIT failure falls through to FFmpeg-Vulkan). Linux: native → Vulkan → /// VAAPI → software on NVIDIA and ALL AMD (`prefer_vulkan_first` is vendor-wide — /// desktop RADV included, on-glass verdict — not just the Deck's VanGogh); /// VAAPI → native → Vulkan → software on Intel/unknown. Windows (no VAAPI @@ -660,10 +783,17 @@ impl Decoder { /// native → Vulkan → software on Intel/unknown (Intel's driver advertises Vulkan /// Video, but FFmpeg-Vulkan on it strobes/overruns the budget — B580 field /// report). + /// + /// `stream` is the picture shape the host resolved ([`StreamFormat`]). Only the + /// native rung reads it — as its construction-time device probe — because it is + /// the one backend whose support for a shape is a per-device fact the ladder must + /// learn BEFORE it commits (FFmpeg's rungs open a codec and discover the pool + /// format themselves). pub fn new( codec_id: ffmpeg::codec::Id, pref: &str, vk: Option<&VulkanDecodeDevice>, + stream: StreamFormat, ) -> Result { ffmpeg::init().context("ffmpeg init")?; quiet_ffmpeg_log(); @@ -684,6 +814,8 @@ impl Decoder { vaapi_fails: 0, first_fail: None, want_keyframe: false, + delivered: false, + vk: vk.cloned(), #[cfg(windows)] d3d11_import, #[cfg(windows)] @@ -711,7 +843,8 @@ impl Decoder { ) { native_tried = true; let vk = vk.expect("gate demands video_decode, so vk is Some"); - match NativeVulkanDecoder::new(vk) { + let (codec, _) = native_codec(codec_id).expect("the gate admitted this codec"); + match NativeVulkanDecoder::new(vk, codec, stream) { Ok(n) => { tracing::info!( ?codec_id, @@ -727,8 +860,9 @@ impl Decoder { tracing::warn!( ?codec_id, video_decode = vk.is_some_and(|v| v.video_decode), - "PUNKTFUNK_DECODER=native-vulkan refused (needs an H.264 session and a \ - Vulkan-Video-capable presenter device) — standard ladder" + "PUNKTFUNK_DECODER=native-vulkan refused (needs an H.264 or HEVC session \ + and a presenter device whose decode family advertises that codec) — \ + standard ladder" ); } choice = "auto".to_string(); @@ -821,7 +955,8 @@ impl Decoder { ) { let vk = vk.expect("gate demands video_decode, so vk is Some"); - match NativeVulkanDecoder::new(vk) { + let (codec, _) = native_codec(codec_id).expect("the gate admitted this codec"); + match NativeVulkanDecoder::new(vk, codec, stream) { Ok(n) => { tracing::info!( ?codec_id, @@ -980,9 +1115,12 @@ impl Decoder { vaapi_fails: 0, first_fail: None, want_keyframe: false, + delivered: false, // A PyroWave session never demotes (nothing else decodes it — a failure - // renegotiates the codec instead), so the D3D11VA rebuild facts are unused - // here; keep them well-formed rather than plumbing them in for nothing. + // renegotiates the codec instead), so the demotion-rebuild facts (the + // device here, the D3D11VA ones below) are unused; keep them well-formed + // rather than plumbing them in for nothing. + vk: None, #[cfg(windows)] d3d11_import: false, #[cfg(windows)] @@ -1008,6 +1146,7 @@ impl Decoder { self.backend = Backend::Software(SoftwareDecoder::new(self.codec_id)?); self.vaapi_fails = 0; self.first_fail = None; + self.delivered = false; self.want_keyframe = true; Ok(()) } @@ -1067,6 +1206,7 @@ impl Decoder { Ok(f) => { self.vaapi_fails = 0; self.first_fail = None; + self.delivered |= f.is_some(); Ok(f) } Err(e) => { @@ -1082,6 +1222,42 @@ impl Decoder { let first = *self.first_fail.get_or_insert_with(std::time::Instant::now); if self.vaapi_fails >= VAAPI_DEMOTE_AFTER && first.elapsed() >= HW_DEMOTE_MIN_STREAK { + // A NATIVE rung that never delivered a single frame is not a + // failing decoder — it is a decoder the session never had, and + // the cause is almost always a stream shape THIS DEVICE cannot + // host (`NativeVulkanDecoder::new`'s probe catches the ones the + // negotiation can see; a level above the device's `maxLevelIdc`, + // or an SPS that disagrees with the Welcome, only surface here). + // Demoting past FFmpeg-Vulkan for that would cost the session the + // rung it would have run on before this backend existed — on + // NVIDIA/Linux, where VAAPI is unusable, that means a 4K HEVC + // session on SOFTWARE. So the first streak in this state falls + // through to FFmpeg-Vulkan, exactly where a construction failure + // would have landed. Once a frame HAS been delivered the rung is + // proven and its streaks demote like every other Vulkan rung's. + if !self.delivered && matches!(self.backend, Backend::NativeVulkan(_)) { + // `take`: this arm is one-shot by construction (the native + // backend is gone after it), and taking is also what lets the + // rebuild borrow the device while `self.backend` is assigned. + if let Some(v) = self.vk.take().filter(|v| v.video_decode) { + match VulkanDecoder::new(self.codec_id, &v) { + Ok(fallback) => { + tracing::warn!(error = %e, fails = self.vaapi_fails, + decoder = fallback.name(), + "native Vulkan Video never delivered a frame — \ + demoting to FFmpeg Vulkan Video"); + self.backend = Backend::Vulkan(fallback); + self.vaapi_fails = 0; + self.first_fail = None; + self.delivered = false; + return Ok(None); + } + Err(fe) => tracing::info!(reason = %format!("{fe:#}"), + "FFmpeg Vulkan Video unavailable for demotion — \ + continuing down the ladder"), + } + } + } // A failing Vulkan backend (FFmpeg or native — the native rung // demotes exactly like the FFmpeg one) still has a hardware rung // below it on Linux — demote to VAAPI first (user-reported: @@ -1098,6 +1274,7 @@ impl Decoder { self.backend = Backend::Vaapi(v); self.vaapi_fails = 0; self.first_fail = None; + self.delivered = false; return Ok(None); } Err(va) => tracing::info!(reason = %va, @@ -1123,6 +1300,7 @@ impl Decoder { self.backend = Backend::D3d11va(d); self.vaapi_fails = 0; self.first_fail = None; + self.delivered = false; return Ok(None); } Err(dx) => tracing::info!(reason = %dx, @@ -1134,6 +1312,7 @@ impl Decoder { self.backend = Backend::Software(SoftwareDecoder::new(self.codec_id)?); self.vaapi_fails = 0; self.first_fail = None; + self.delivered = false; } else { tracing::debug!(backend = which, error = %e, "decode error — requesting keyframe, keeping hardware decode"); @@ -1412,15 +1591,18 @@ mod tests { } /// The native-Vulkan admission gate (WP-C, widened by the 2026-08-05 ladder - /// decision): the pin AND the auto family admit on a capable H.264 session — - /// native sits immediately above FFmpeg-Vulkan because the program is dropping - /// FFmpeg — while every explicit backend pin refuses (`vulkan` names the - /// FFmpeg-Vulkan backend specifically and must keep meaning exactly that), and - /// the codec/device legs still refuse for every choice. + /// decision and again by M3 WP-2's HEVC wiring): the pin AND the auto family + /// admit on a capable H.264 or HEVC session — native sits immediately above + /// FFmpeg-Vulkan because the program is dropping FFmpeg — while every explicit + /// backend pin refuses (`vulkan` names the FFmpeg-Vulkan backend specifically and + /// must keep meaning exactly that), and the codec/device legs still refuse for + /// every choice. The codec's OWN caps bit is the device leg: admitting HEVC on an + /// H.264-only decode family would create a video session for an operation the + /// family cannot run, which is undefined behaviour rather than an error. #[test] - fn native_vulkan_gate_admits_pin_and_auto_family_on_capable_h264_only() { + fn native_vulkan_gate_admits_pin_and_auto_family_per_codec_on_a_capable_family() { use ffmpeg::codec::Id; - // Pin the raw spec value, not the implementation constant — a typo'd bit + // Pin the raw spec values, not the implementation constants — a typo'd bit // would refuse every real driver's caps and native would silently never // engage (the program's own nb_queries=0 lesson: silent non-engagement is // the failure mode nothing flags). @@ -1428,20 +1610,53 @@ mod tests { VIDEO_CODEC_OP_DECODE_H264, 0x1, "VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR" ); + assert_eq!( + VIDEO_CODEC_OP_DECODE_H265, 0x2, + "VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR" + ); const H264_OP: u32 = VIDEO_CODEC_OP_DECODE_H264; + const H265_OP: u32 = VIDEO_CODEC_OP_DECODE_H265; + // `VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR` — a real bit on real + // hardware, and never enough on its own (no AV1 decoder exists here). + const AV1_OP: u32 = 0x4; for choice in ["native-vulkan", "auto", "", "hardware"] { - // The pin and the whole auto family admit… + // The pin and the whole auto family admit both codecs pf-vkdecode + // speaks, on a family that advertises the matching op… assert!( native_vulkan_gate(choice, Id::H264, true, H264_OP), "{choice:?}" ); - // …but only for the one codec pf-vkdecode speaks. + assert!( + native_vulkan_gate(choice, Id::HEVC, true, H265_OP), + "{choice:?}" + ); + // …including the ordinary case of a family that runs both. + assert!( + native_vulkan_gate(choice, Id::H264, true, H264_OP | H265_OP), + "{choice:?}" + ); + assert!( + native_vulkan_gate(choice, Id::HEVC, true, H264_OP | H265_OP), + "{choice:?}" + ); + // Each codec needs ITS OWN bit: an H.264-only family (the common case on + // older silicon) must not take an HEVC session, and vice versa. assert!( !native_vulkan_gate(choice, Id::HEVC, true, H264_OP), "{choice:?}" ); assert!( - !native_vulkan_gate(choice, Id::AV1, true, H264_OP), + !native_vulkan_gate(choice, Id::H264, true, H265_OP), + "{choice:?}" + ); + // AV1 refuses whatever the family advertises — pf-vkdecode has no AV1 + // decoder, so the session must fall through to the FFmpeg rungs. + assert!( + !native_vulkan_gate(choice, Id::AV1, true, AV1_OP), + "{choice:?}" + ); + assert!( + !native_vulkan_gate(choice, Id::AV1, true, H264_OP | H265_OP | AV1_OP), "{choice:?}" ); // No Vulkan-Video-capable presenter device. @@ -1449,11 +1664,21 @@ mod tests { !native_vulkan_gate(choice, Id::H264, false, H264_OP), "{choice:?}" ); - // A decode family WITHOUT the H264 op (e.g. AV1-only) refuses even with - // the extension stack present — the caps BIT is the codec gate. - assert!(!native_vulkan_gate(choice, Id::H264, true, 0), "{choice:?}"); assert!( - !native_vulkan_gate(choice, Id::H264, true, 0x4), + !native_vulkan_gate(choice, Id::HEVC, false, H265_OP), + "{choice:?}" + ); + // A decode family advertising NO codec op, or only a foreign one, + // refuses even with the extension stack present — the caps BIT is the + // codec gate, not `video_decode`. + assert!(!native_vulkan_gate(choice, Id::H264, true, 0), "{choice:?}"); + assert!(!native_vulkan_gate(choice, Id::HEVC, true, 0), "{choice:?}"); + assert!( + !native_vulkan_gate(choice, Id::H264, true, AV1_OP), + "{choice:?}" + ); + assert!( + !native_vulkan_gate(choice, Id::HEVC, true, AV1_OP), "{choice:?}" ); } @@ -1463,7 +1688,24 @@ mod tests { !native_vulkan_gate(choice, Id::H264, true, H264_OP), "{choice:?}" ); + assert!( + !native_vulkan_gate(choice, Id::HEVC, true, H265_OP), + "{choice:?}" + ); } + // The decoder the gate implies — the construction sites `expect()` this + // exact agreement, so a codec admitted with no decoder behind it would be a + // panic rather than a demotion. + assert_eq!( + native_codec(Id::H264).map(|(c, _)| c), + Some(NativeCodec::H264) + ); + assert_eq!( + native_codec(Id::HEVC).map(|(c, _)| c), + Some(NativeCodec::H265) + ); + assert!(native_codec(Id::AV1).is_none()); + assert!(native_codec(Id::VP9).is_none()); } /// Lock the DRM FourCC magic numbers against typos — these are the exact values diff --git a/crates/pf-client-core/src/video_vk_native.rs b/crates/pf-client-core/src/video_vk_native.rs index aad614bf..6efbb753 100644 --- a/crates/pf-client-core/src/video_vk_native.rs +++ b/crates/pf-client-core/src/video_vk_native.rs @@ -1,10 +1,37 @@ -//! Native Vulkan Video H.264 decode backend (WP-C of the native-decode program): -//! pf-vkdecode's [`VkH264Decoder`] running on the PRESENTER's own VkDevice — the same -//! zero-copy shape as the FFmpeg-Vulkan backend, with no FFmpeg in the path. Auto's -//! rung immediately ABOVE FFmpeg-Vulkan since the 2026-08-05 ladder decision (WP-D -//! closed bit-exact — the program is dropping FFmpeg from the client), also pinnable -//! via `PUNKTFUNK_DECODER=native-vulkan`; `video::native_vulkan_gate` is the -//! admission either way, and a failure falls through to the FFmpeg-Vulkan rung. +//! Native Vulkan Video decode backend (WP-C of the native-decode program, widened to +//! HEVC by M3 WP-2): pf-vkdecode's [`VkH264Decoder`]/[`VkH265Decoder`] running on the +//! PRESENTER's own VkDevice — the same zero-copy shape as the FFmpeg-Vulkan backend, +//! with no FFmpeg in the path. Auto's rung immediately ABOVE FFmpeg-Vulkan since the +//! 2026-08-05 ladder decision (WP-D closed bit-exact — the program is dropping FFmpeg +//! from the client), also pinnable via `PUNKTFUNK_DECODER=native-vulkan`; +//! `video::native_vulkan_gate` is the admission either way, and a failure falls +//! through to the FFmpeg-Vulkan rung. +//! +//! **Codec dispatch:** the negotiated codec picks the decoder ONCE, at construction +//! ([`Codec`]) — H.264 or H.265, the two codecs pf-vkdecode speaks. The negotiated +//! picture SHAPE (chroma format + bit depth) is checked there too, against the +//! device: an H.265 session this GPU has no decode format for is refused at +//! construction, where the ladder answers with FFmpeg-Vulkan, rather than at the +//! first AU, where the only exit is an error streak PAST that rung +//! ([`NativeVulkanDecoder::new`]). Nothing below the codec enum is per-codec: the +//! shipped-frame ledger, the release tokens, the +//! decode-status reads, the timeline waits and the teardown drain are shared, because +//! both decoders deliver the identical [`DecodedVkFrame`] contract (same pool/slot +//! lifecycle, same `value + 1` write-back, same query slots, same generations). +//! Forking that machinery per codec would fork the one part of this backend hardware +//! has already proven. +//! +//! **A skipped RASL picture is NOT a decode error.** An HEVC stream joined at a CRA +//! carries leading pictures whose references precede the join; the spec's own answer +//! (8.1.3 NOTE) is to decode and output nothing for them. [`VkH265Decoder::decode`] +//! implements exactly that: `h265::PlanError::RaslSkipped` never becomes a +//! `VkDecodeError`, so the AU comes back as `Ok` with whatever was ALREADY +//! display-ready (usually `None`) and with the warning ledger cleared. This backend +//! must therefore treat `Ok(None)` as "no picture this AU" and nothing more — no +//! release-unshown, no re-anchor request, no error. Mapping it to an error would make +//! every open-GOP join beg the host for a keyframe it has no reason to send. (Dead in +//! the field today — punktfunk hosts emit IDR-only re-entry points — but it is the +//! contract pf-bitstream's `h265` module docs record for this wiring.) //! //! **Queue lock:** pf-vkdecode submits on queue 0 of the decode family //! ([`DECODE_QUEUE_INDEX`] — the presenter creates exactly one queue per family). When @@ -24,12 +51,12 @@ //! returned token proves the GPU is done with the image; a frame dropped UNPRESENTED //! (newest-wins displacement, post-demotion drain) releases through the same drop. The //! backend drains the channel at every `decode` entry and calls -//! [`VkH264Decoder::release_frame`] — but only once the frame's decode-status query has +//! [`Codec::release_frame`] — but only once the frame's decode-status query has //! also been read (the slot stays pinned meanwhile, which is what makes re-polling the //! query safe: an unreleased slot can never be recycled under the poll). //! //! **Status queries:** every decode op carries a `RESULT_STATUS_ONLY` query — -//! [`VkH264Decoder::poll_status`], read non-blockingly here at each decode entry. A +//! [`Codec::poll_status`], read non-blockingly here at each decode entry. A //! `Failed` verdict is driver-reported decode corruption, the class FFmpeg's //! `vulkan_decode.c` (`nb_queries = 0`) architecturally cannot see — the Xbox Ally X //! field case. It surfaces as an `Err` from the CURRENT `decode_frame` call so the @@ -48,7 +75,9 @@ use crate::video::{ use anyhow::{anyhow, bail, Result}; use pf_vkdecode::ash::vk; use pf_vkdecode::ash::vk::Handle as _; -use pf_vkdecode::{DecodeStatus, DecodedVkFrame, DeviceHandles, VkH264Decoder}; +use pf_vkdecode::{ + DecodeStatus, DecodedVkFrame, DeviceHandles, VkDecodeError, VkH264Decoder, VkH265Decoder, +}; use std::sync::mpsc; use std::time::{Duration, Instant}; @@ -102,6 +131,233 @@ impl pf_vkdecode::QueueLock for NativeQueueLock { } } +/// The codecs pf-vkdecode has a decoder for — the native rung's whole vocabulary, +/// named ash-free so `video.rs` can pick one from the negotiated wire codec without +/// this module knowing about FFmpeg's codec ids (and `video::native_vulkan_gate` +/// stays the single admission decision). AV1 is deliberately absent: the Vulkan +/// decode op exists and real hardware advertises it, but there is no AV1 decoder in +/// pf-vkdecode, so those sessions must keep falling through to the FFmpeg rungs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NativeCodec { + H264, + H265, +} + +/// The decoder this backend drives, chosen ONCE from the negotiated codec. +/// +/// Dispatch stops here. Everything the backend does around a decoder — the +/// shipped-frame ledger, release tokens, status-query settling, timeline waits, +/// teardown — is codec-agnostic, because [`VkH264Decoder`] and [`VkH265Decoder`] +/// expose the same surface over the same [`DecodedVkFrame`] contract (same pool/slot +/// lifecycle, same `value + 1` write-back, same query slots, same generations). The +/// forwarders below are therefore mechanically identical per arm on purpose: the +/// H.264 path is hardware-verified bit-exact, and dispatch must not be able to change +/// its behaviour. +// Unboxed on purpose, against `large_enum_variant`: the arms differ by ~1.7 KB (both +// decoders carry a planner, a slot ledger and pinned Std parameter sets), and exactly +// ONE of these exists per session — inside the `Box` the backend +// already lives in. So the "waste" is 1.7 KB of slack in a single session-lifetime +// allocation, while boxing would put a second indirection between the pump and the +// decoder on the per-AU path and change how the hardware-verified H.264 decoder is +// reached. Neither trade is worth 1.7 KB. +#[allow(clippy::large_enum_variant)] +enum Codec { + H264(VkH264Decoder), + H265(VkH265Decoder), +} + +impl Codec { + /// Feed one access unit — see [`VkH264Decoder::decode`] / + /// [`VkH265Decoder::decode`]. `Ok(None)` means "no display-ready picture from + /// this AU", which for H.265 also covers a RASL picture skipped after an + /// open-GOP join (the module doc's contract: never an error). + fn decode(&mut self, au: &[u8]) -> Result, VkDecodeError> { + match self { + Codec::H264(d) => d.decode(au), + Codec::H265(d) => d.decode(au), + } + } + + /// Drain the plan warnings of the AU just decoded, TYPED — the two planners + /// have genuinely different enums ([`pf_vkdecode::PlanWarning`] has + /// `FrameNumGap`/`Mmco5Rebase`, [`pf_vkdecode::H265PlanWarning`] has + /// `NonZeroReorder`, neither a subset of the other), so the pair is carried as + /// a two-armed value rather than flattened. + /// + /// Typed and not rendered because the backend must BRANCH on them: only some + /// warnings mean the picture is damaged ([`PlanWarnings::integrity`]), and + /// dropping a frame for the others costs a visible hitch on a stream the + /// planner says it planned correctly. Strings would make that a substring + /// match on `Debug` output. + fn take_warnings(&mut self) -> PlanWarnings { + match self { + Codec::H264(d) => PlanWarnings::H264(d.take_warnings()), + Codec::H265(d) => PlanWarnings::H265(d.take_warnings()), + } + } + + /// Pull the next already display-ready frame the last AU did not return + /// directly (burst output). + fn take_ready(&mut self) -> Option { + match self { + Codec::H264(d) => d.take_ready(), + Codec::H265(d) => d.take_ready(), + } + } + + /// Hand a delivered frame back to its pool; `presented` reports whether the + /// consumer enqueued the frame's `value + 1` timeline signal. + fn release_frame( + &mut self, + frame: &DecodedVkFrame, + presented: bool, + ) -> Result<(), VkDecodeError> { + match self { + Codec::H264(d) => d.release_frame(frame, presented), + Codec::H265(d) => d.release_frame(frame, presented), + } + } + + /// The decoder's current session generation (a frame from an older one has an + /// unknowable status verdict — see [`NativeVulkanDecoder::settle_statuses`]). + fn generation(&self) -> u64 { + match self { + Codec::H264(d) => d.generation(), + Codec::H265(d) => d.generation(), + } + } + + /// Non-blocking read of a frame's `RESULT_STATUS_ONLY` query. + fn poll_status(&mut self, frame: &DecodedVkFrame) -> DecodeStatus { + match self { + Codec::H264(d) => d.poll_status(frame), + Codec::H265(d) => d.poll_status(frame), + } + } + + /// Bounded host wait for a frame's decode-complete timeline signal (the pump's + /// sampled decode-latency stat). + fn wait_decoded(&self, frame: &DecodedVkFrame, timeout_ns: u64) -> bool { + match self { + Codec::H264(d) => d.wait_decoded(frame, timeout_ns), + Codec::H265(d) => d.wait_decoded(frame, timeout_ns), + } + } +} + +/// The plan warnings one AU produced, still in their codec's own enum. +/// +/// The split that matters is INTEGRITY vs. spec-legal, not H.264 vs. H.265. Both +/// planners emit two kinds of warning through one channel: +/// +/// - **Integrity** — a reference the DPB does not hold, a `frame_num` gap, an AU +/// whose NALU walk stopped early. The plan was completed with a SUBSTITUTE in +/// place of something lost: the picture is damaged, so its output is released +/// unshown and a re-anchor is requested. +/// - **Spec-legal envelope signals** — h265's `NonZeroReorder` (the activated SPS +/// sets `sps_max_num_reorder_pics > 0`) and h264's `Mmco5Rebase`. pf-bitstream +/// documents both as "spec-legal and fully planned"; they exist as the field +/// signal that a punktfunk-host assumption broke, not as damage. `NonZeroReorder` +/// in particular fires on the AU that ACTIVATES an SPS — the opening IDR, and the +/// fresh IDR at every ABR resolution change — so treating it as concealment costs +/// a released-unshown frame plus a keyframe round trip at every renegotiation, on +/// a stream the planner planned correctly. pf-bitstream's own conformance harness +/// excludes `NonZeroReorder` from its integrity set for exactly this reason. +/// +/// Everything is logged either way; only integrity warnings drop the frame. +enum PlanWarnings { + H264(Vec), + H265(Vec), +} + +/// Does this H.264 warning mean the PICTURE is damaged? `Mmco5Rebase` does not: the +/// AU carried an MMCO 5 and pf-bitstream planned it in full (the plan holds the +/// pre-rebase 8.2.1 values, later AUs reference the rebased ones). +fn h264_is_integrity(w: &pf_vkdecode::PlanWarning) -> bool { + use pf_vkdecode::PlanWarning as W; + matches!( + w, + W::FrameNumGap { .. } | W::MissingReference { .. } | W::TruncatedAu { .. } + ) +} + +/// The H.265 twin — the same set pf-bitstream's `h265` conformance harness calls +/// integrity, `NonZeroReorder` deliberately excluded (see [`PlanWarnings`]). +fn h265_is_integrity(w: &pf_vkdecode::H265PlanWarning) -> bool { + use pf_vkdecode::H265PlanWarning as W; + matches!(w, W::MissingReference { .. } | W::TruncatedAu { .. }) +} + +impl PlanWarnings { + fn is_empty(&self) -> bool { + match self { + PlanWarnings::H264(w) => w.is_empty(), + PlanWarnings::H265(w) => w.is_empty(), + } + } + + /// Just the warnings that mean the picture is damaged — the concealment set. + /// Allocates, but only off the clean path: [`Self::is_empty`] is true for every + /// AU of a healthy stream. + fn integrity(&self) -> PlanWarnings { + match self { + PlanWarnings::H264(w) => { + PlanWarnings::H264(w.iter().filter(|x| h264_is_integrity(x)).cloned().collect()) + } + PlanWarnings::H265(w) => { + PlanWarnings::H265(w.iter().filter(|x| h265_is_integrity(x)).cloned().collect()) + } + } + } + + fn len(&self) -> usize { + match self { + PlanWarnings::H264(w) => w.len(), + PlanWarnings::H265(w) => w.len(), + } + } + + /// The concealment log. Per arm so the rendering is the codec's OWN enum — + /// `warnings=[FrameNumGap { .. }]`, exactly what the hardware-verified H.264 + /// path emitted before dispatch existed (a `Vec` renders + /// `["FrameNumGap { .. }"]`, and a wrapper enum would prefix the arm). + fn warn_concealment(&self) { + // Spelled out per arm rather than shared through a `const`: this is the + // H.264 path's PRODUCTION log line, and a literal is what keeps it a static + // tracing message rather than a formatted one. + match self { + PlanWarnings::H264(w) => tracing::warn!( + warnings = ?w, + "native decode planned with concealment — dropping the frame, \ + requesting re-anchor" + ), + PlanWarnings::H265(w) => tracing::warn!( + warnings = ?w, + "native decode planned with concealment — dropping the frame, \ + requesting re-anchor" + ), + } + } + + /// The spec-legal log: the planner flagged an envelope fact and planned the AU + /// in full, so the frame is SHOWN. Rare by construction (SPS activation, MMCO + /// 5), which is why it is a `warn` and not a per-frame `debug`. + fn warn_planned_in_full(&self) { + match self { + PlanWarnings::H264(w) => tracing::warn!( + warnings = ?w, + "native decode: spec-legal envelope signal — the AU was planned in \ + full and the frame is kept" + ), + PlanWarnings::H265(w) => tracing::warn!( + warnings = ?w, + "native decode: spec-legal envelope signal — the AU was planned in \ + full and the frame is kept" + ), + } + } +} + /// One frame shipped to the presenter and not yet fully settled: settled = its release /// token came back (GPU reads proven done) AND its status query was read. struct Shipped { @@ -137,9 +393,85 @@ fn note_token(outstanding: &mut [Shipped], token: NativeReleaseToken) -> bool { } } +/// Flatten a delivered [`DecodedVkFrame`] into the ash-free [`NativeVkFrame`] the +/// presenter consumes. Pure over the frame (the guard is the caller's), so the +/// projection — every fact the presenter can no longer look up for itself — is +/// CPU-testable. +/// +/// The one that is easy to get wrong is [`NativeVkFrame::vk_format`]: the picture +/// format is the STREAM's, not the codec's. H.264 in this program is always the 8-bit +/// 4:2:0 envelope (NV12), but an H.265 session decodes Main to NV12, Main 10 to P010 +/// and RExt 4:4:4 to the two-plane 4:4:4 formats — and can change format mid-stream +/// when the host renegotiates. A consumer that assumes 8-bit 4:2:0 renders a Main 10 +/// picture with 8-bit transfer/range math: plausible-looking and wrong. So the format +/// is carried, never inferred, all the way to the presenter's CSC pass. +fn project_frame(frame: &DecodedVkFrame, guard: NativeReleaseGuard) -> NativeVkFrame { + NativeVkFrame { + image: frame.image.as_raw(), + vk_format: crate::video::RawVkFormat(frame.format.as_raw()), + plane_views: [frame.plane_views[0].as_raw(), frame.plane_views[1].as_raw()], + layer: frame.layer, + layout: if frame.layout == vk::ImageLayout::VIDEO_DECODE_DPB_KHR { + NativeVkLayout::DecodeDpb + } else { + NativeVkLayout::DecodeDst + }, + semaphore: frame.semaphore.as_raw(), + semaphore_value: frame.value, + generation: frame.generation, + width: frame.crop.width, + height: frame.crop.height, + coded_width: frame.coded_width, + coded_height: frame.coded_height, + crop_x: frame.crop.x, + crop_y: frame.crop.y, + // H.273 code points straight off the picture's ACTIVE SPS/VUI — per frame, + // never latched, because the Windows host switches an HDR desktop to + // PQ/BT.2020 IN-BAND (the Welcome still says SDR). pf-bitstream applies + // E.2.1's "unspecified" inference (2/2/2, limited) where the VUI is + // silent, and `csc_rows` resolves "unspecified" to its BT.709-limited + // SDR default — same verdicts libavcodec's CICP passthrough produced. + color: ColorDesc { + primaries: frame.colour.colour_primaries, + transfer: frame.colour.transfer_characteristics, + matrix: frame.colour.matrix_coefficients, + full_range: frame.colour.video_full_range, + }, + keyframe: frame.is_idr, + poc: frame.poc, + guard, + } +} + +/// The picture format an H.265 session of the negotiated shape decodes to, or a named +/// refusal for a shape pf-vkdecode has no output format for at all. +/// +/// The DEVICE-INDEPENDENT half of [`NativeVulkanDecoder::new`]'s shape check: 4:2:2 +/// and 12-bit are legal H.265 that no punktfunk host emits and this client has no +/// plumbing for, so no driver has to be asked about them. Pure, so the refusal is +/// CPU-testable — the device-dependent half (a shape with a format that THIS driver +/// does not advertise) is [`VkH265Decoder::probe_stream_support`], covered by +/// pf-vkdecode's `derive_caps_h265` refusal tests. +fn h265_picture_format(stream: crate::video::StreamFormat) -> Result { + let depth = stream.bit_depth_minus8().ok_or_else(|| { + anyhow!( + "negotiated HEVC bit depth {} is outside the 8/10-bit decode envelope", + stream.bit_depth + ) + })?; + pf_vkdecode::output_format_for(stream.chroma_format_idc, depth).ok_or_else(|| { + anyhow!( + "no native picture format for the negotiated HEVC stream shape \ + (chroma_format_idc={}, {}-bit)", + stream.chroma_format_idc, + stream.bit_depth + ) + }) +} + /// The native backend: the decoder plus the shipped-frame ledger and release channel. pub(crate) struct NativeVulkanDecoder { - dec: VkH264Decoder, + dec: Codec, /// Cloned into every shipped frame's guard. `Option` so teardown can DROP the /// backend's own sender: only then does `release_rx` report Disconnected once /// the last guard is gone — the teardown short-circuit signal. @@ -154,14 +486,51 @@ pub(crate) struct NativeVulkanDecoder { // SAFETY: the decoder is used strictly serially through `&mut self` from whichever // single thread owns the enclosing `Decoder` (the session pump) — `Send` only moves -// that ownership. The `Rc`s inside pf-vkdecode's planner never escape it, so they all -// move together; every queue submission runs under the collision-aware queue lock; the -// mpsc endpoints are `Send`. Same contract, same shape as the `VulkanDecoder` and -// `PyroWaveDecoder` impls above/beside it. Deliberately NOT `Sync`. +// that ownership. The `Rc`s inside pf-vkdecode's planners (H.264 and H.265 alike) +// never escape them, so they all move together; every queue submission runs under the +// collision-aware queue lock; the mpsc endpoints are `Send`. Same contract, same shape +// as the `VulkanDecoder` and `PyroWaveDecoder` impls above/beside it. Deliberately NOT +// `Sync`. unsafe impl Send for NativeVulkanDecoder {} impl NativeVulkanDecoder { - pub(crate) fn new(vk: &VulkanDecodeDevice) -> Result { + /// Build the backend over the presenter's device for `codec` — the codec the + /// session negotiated, already admitted by `video::native_vulkan_gate` (which + /// checked that the decode family advertises this codec's decode op; the + /// decoders re-check it themselves rather than trust the caller, because + /// creating a video session for a codec operation the family cannot run is + /// undefined behaviour rather than an error). + /// + /// Sessions and pools are built lazily from the first AU's parameter sets, so + /// nothing BELOW this constructor depends on the stream's shape — which is why + /// the shape is checked HERE, against `stream` (the host's resolved Welcome + /// facts), rather than being discovered at the first decode. + /// + /// The difference is which rung a refusal lands on. pf-vkdecode's picture format + /// is the STREAM's (Main → NV12, Main 10 → P010, RExt 4:4:4 → the two-plane 4:4:4 + /// formats) and a device that advertises H.265 decode need not advertise a format + /// for every shape of it: 4:4:4 is absent everywhere but NVIDIA. Discovered + /// lazily, that is a mid-stream ERROR STREAK, and the streak machinery demotes a + /// Vulkan rung to VAAPI/D3D11VA — PAST FFmpeg-Vulkan, which on NVIDIA/Linux (no + /// usable VAAPI) means a 4K HEVC session lands on SOFTWARE. Refused here it is an + /// ordinary construction failure, and `video::Decoder::new` falls through to + /// FFmpeg-Vulkan — the rung that session ran on before this backend existed. + /// + /// Two legs the probe cannot see, because they are stream facts no negotiation + /// carries: a level above the device's `maxLevelIdc`, and an SPS that disagrees + /// with the Welcome. Those still surface at the first decode — and are caught by + /// the "never delivered a frame" arm in [`crate::video::Decoder::decode_frame`], + /// which routes exactly that state to FFmpeg-Vulkan instead of past it. + /// + /// H.264 is deliberately NOT probed: its envelope is fixed at 8-bit 4:2:0, so the + /// only fact a probe could add is a profile idc guess — on the one path in this + /// program that is hardware-verified bit-exact against libavcodec. It keeps the + /// never-delivered arm as its backstop. + pub(crate) fn new( + vk: &VulkanDecodeDevice, + codec: NativeCodec, + stream: crate::video::StreamFormat, + ) -> Result { if !vk.video_decode { bail!("presenter device lacks Vulkan Video decode"); } @@ -180,16 +549,58 @@ impl NativeVulkanDecoder { decode_queue_index: DECODE_QUEUE_INDEX, graphics_qf: vk.graphics_qf, }; - // SAFETY: the handles are the presenter's live instance/device, which outlives - // every session pump (the run loop tears the pump — and with it this decoder — - // down first: the exact liveness contract the FFmpeg and PyroWave backends - // already rely on over the same bundle). `video_decode` (checked above) is set - // only when the presenter enabled the Vulkan Video decode extension stack + - // synchronization2/timelineSemaphore at device creation, and - // `decode_qf`/`graphics_qf` mirror the families it created queues for (one - // queue, index 0, each). - let dec = unsafe { VkH264Decoder::new(&handles, lock) } - .map_err(|e| anyhow!("VkH264Decoder init: {e}"))?; + // The `DeviceHandles` caller contract, held for the decoder's whole lifetime + // and identical for both arms (it is the HANDLES' contract, not the codec's): + // the handles are the presenter's live instance/device, which outlives every + // session pump (the run loop tears the pump — and with it this decoder — down + // first: the exact liveness contract the FFmpeg and PyroWave backends already + // rely on over the same bundle). `video_decode` (checked above) is set only + // when the presenter enabled the Vulkan Video decode extension stack + + // synchronization2/timelineSemaphore at device creation — including the + // per-codec `VK_KHR_video_decode_h264`/`_h265`/`_av1` extensions, one for + // every codec operation the decode family advertises (`vk/setup.rs` enables + // exactly those it finds). What the decoders then re-check for themselves is + // the QUEUE FAMILY's advertised `videoCodecOperations` — the device's own + // claim about the family, which is what `native_vulkan_gate` reads too. That + // is not a proof the extension was enabled at `vkCreateDevice`; it is the + // same fact `vk/setup.rs` derived its enable list FROM, so the two agree by + // construction here and the check catches a caller that got the family wrong. + // `decode_qf`/`graphics_qf` mirror the families the presenter created queues + // for (one queue, index 0, each). + let dec = match codec { + NativeCodec::H264 => { + // SAFETY: the handle contract stated directly above. + let d = unsafe { VkH264Decoder::new(&handles, lock) } + .map_err(|e| anyhow!("VkH264Decoder init: {e}"))?; + Codec::H264(d) + } + NativeCodec::H265 => { + // The device-independent half of the shape check, first: a stream + // shape pf-vkdecode has NO picture format for (4:2:2, 12-bit) needs + // no driver to refuse it. + let wanted = h265_picture_format(stream)?; + // SAFETY: the handle contract stated directly above. + let d = unsafe { VkH265Decoder::new(&handles, lock) } + .map_err(|e| anyhow!("VkH265Decoder init: {e}"))?; + // …and the device-dependent half: does THIS driver advertise that + // format for a decode session of this profile? Same query and same + // derivation `ensure_state` would run at the first AU — only the + // timing differs, and the timing is the whole point. + let depth = stream + .bit_depth_minus8() + .expect("h265_picture_format accepted the depth"); + d.probe_stream_support(stream.chroma_format_idc, depth) + .map_err(|e| { + anyhow!( + "device cannot decode the negotiated HEVC stream shape \ + (chroma_format_idc={}, {}-bit, needs {wanted:?}): {e}", + stream.chroma_format_idc, + stream.bit_depth + ) + })?; + Codec::H265(d) + } + }; let (release_tx, release_rx) = mpsc::channel(); Ok(NativeVulkanDecoder { dec, @@ -207,6 +618,13 @@ impl NativeVulkanDecoder { /// driver-reported corrupt PREVIOUS frame — routed through the caller's shared /// streak/demotion machinery. /// + /// The one thing `Ok(None)` deliberately does NOT mean is trouble. An H.265 RASL + /// picture skipped after an open-GOP join arrives here as exactly that — the + /// decoder never turns `h265::PlanError::RaslSkipped` into a `VkDecodeError`, and + /// it clears the warning ledger on its way out, so the concealment branch below + /// cannot fire on it either. Nothing is released unshown, no re-anchor is asked + /// for, and the next AU decodes normally (module doc; pf-bitstream `h265`). + /// /// Ordering: the CURRENT AU decodes FIRST — the planner's reference state must /// advance even when a PRIOR frame's status turns out Failed, or the recovery /// IDR would land on a decoder that skipped an AU and reports a phantom @@ -228,7 +646,12 @@ impl NativeVulkanDecoder { } let corrupt = self.settle_statuses(); - if !warnings.is_empty() || corrupt > 0 { + // ONLY integrity warnings are concealment (see [`PlanWarnings`]): a + // spec-legal envelope signal — h265's `NonZeroReorder` on every SPS + // activation, h264's `Mmco5Rebase` — is an AU the planner planned in FULL, + // and dropping its frame would hitch the picture at every renegotiation. + let integrity = warnings.integrity(); + if !integrity.is_empty() || corrupt > 0 { // Concealment planned into THIS AU, or driver-reported corruption on a // PRIOR frame (the Ally X class, invisible to FFmpeg's query-less // decoder): this call's output is released unshown and the call errors, @@ -245,16 +668,19 @@ impl NativeVulkanDecoder { (RESULT_STATUS_ONLY query) — re-anchor needed" )); } - tracing::warn!( - ?warnings, - "native decode planned with concealment — dropping the frame, \ - requesting re-anchor" - ); + // The log carries EVERY warning of the AU (the spec-legal ones are + // context); the count is the concealment count, which is what the + // frame was dropped for. On H.264 the two coincide for every warning + // a punktfunk host can produce. + warnings.warn_concealment(); bail!( "AU planned with concealment ({} warning(s))", - warnings.len() + integrity.len() ); } + if !warnings.is_empty() { + warnings.warn_planned_in_full(); + } self.deliverable.extend(fresh); Ok(self.deliverable.pop_front().map(|frame| self.ship(frame))) @@ -270,46 +696,16 @@ impl NativeVulkanDecoder { generation: frame.generation, presented: false, }; - let native = NativeVkFrame { - image: frame.image.as_raw(), - plane_views: [frame.plane_views[0].as_raw(), frame.plane_views[1].as_raw()], - layer: frame.layer, - layout: if frame.layout == vk::ImageLayout::VIDEO_DECODE_DPB_KHR { - NativeVkLayout::DecodeDpb - } else { - NativeVkLayout::DecodeDst - }, - semaphore: frame.semaphore.as_raw(), - semaphore_value: frame.value, - generation: frame.generation, - width: frame.crop.width, - height: frame.crop.height, - coded_width: frame.coded_width, - coded_height: frame.coded_height, - crop_x: frame.crop.x, - crop_y: frame.crop.y, - // H.273 code points straight off the picture's ACTIVE SPS — per frame, - // never latched, because the Windows host switches an HDR desktop to - // PQ/BT.2020 IN-BAND (the Welcome still says SDR). pf-bitstream applies - // E.2.1's "unspecified" inference (2/2/2, limited) where the VUI is - // silent, and `csc_rows` resolves "unspecified" to its BT.709-limited - // SDR default — same verdicts libavcodec's CICP passthrough produced. - color: ColorDesc { - primaries: frame.colour.colour_primaries, - transfer: frame.colour.transfer_characteristics, - matrix: frame.colour.matrix_coefficients, - full_range: frame.colour.video_full_range, - }, - keyframe: frame.is_idr, - poc: frame.poc, - guard: NativeReleaseGuard::new( + let native = project_frame( + &frame, + NativeReleaseGuard::new( self.release_tx .as_ref() .expect("release_tx lives until Drop") .clone(), token, ), - }; + ); self.outstanding.push(Shipped { seq, frame, @@ -515,37 +911,65 @@ impl Drop for NativeVulkanDecoder { mod tests { use super::*; + /// A delivered frame whose every field carries a DISTINCT non-zero value. + /// + /// Deliberately not "inert handles, zeros elsewhere": [`project_frame`] is a + /// 20-field struct literal lifted out of `ship`, and the bugs it can hide are + /// field SWAPS and DROPS — `crop_x: frame.crop.y`, `semaphore_value: frame.poc + /// as u64`, a `keyframe` that stopped being carried. Against zeros every one of + /// those passes. So: no two numbers here are equal, no boolean is false, and + /// each CICP code point differs from the others. + fn decoded(format: vk::Format, layout: vk::ImageLayout, generation: u64) -> DecodedVkFrame { + DecodedVkFrame { + image: vk::Image::from_raw(0x1001), + format, + view: vk::ImageView::from_raw(0x2001), + plane_views: [ + vk::ImageView::from_raw(0x2002), + vk::ImageView::from_raw(0x2003), + ], + layer: 3, + layout, + coded_width: 1920, + coded_height: 1088, + // A non-origin crop: punktfunk hosts emit origin crops only, but x != y + // here is what makes an x/y swap in the projection visible. + crop: pf_vkdecode::DisplayCrop { + x: 8, + y: 4, + width: 1904, + height: 1072, + }, + // BT.2020 primaries / PQ transfer / a third code point for the matrix, + // so no two CICP fields share a value. + colour: pf_vkdecode::ColourDescription { + colour_primaries: 9, + transfer_characteristics: 16, + matrix_coefficients: 10, + video_full_range: true, + }, + semaphore: vk::Semaphore::from_raw(0x3001), + value: 7, + poc: 5, + is_idr: true, + query_slot: 2, + submission: 11, + picture: 6, + generation, + } + } + /// A shipped-ledger entry with inert handles — the bookkeeping under test is pure. fn shipped(seq: u64, generation: u64) -> Shipped { Shipped { seq, - frame: DecodedVkFrame { - image: vk::Image::null(), - // The H.264 envelope's picture format (this decoder is H.264-only); - // the ledger under test never reads it. - format: pf_vkdecode::NV12, - view: vk::ImageView::null(), - plane_views: [vk::ImageView::null(); 2], - layer: 0, - layout: vk::ImageLayout::VIDEO_DECODE_DST_KHR, - coded_width: 1920, - coded_height: 1088, - crop: pf_bitstream_crop(1920, 1080), - colour: pf_vkdecode::ColourDescription { - colour_primaries: 2, - transfer_characteristics: 2, - matrix_coefficients: 2, - video_full_range: false, - }, - semaphore: vk::Semaphore::null(), - value: 0, - poc: 0, - is_idr: false, - query_slot: 0, - submission: 0, - picture: 0, + // The ledger under test never reads the picture format; NV12 is what an + // H.264 session always delivers. + frame: decoded( + pf_vkdecode::NV12, + vk::ImageLayout::VIDEO_DECODE_DST_KHR, generation, - }, + ), released: false, presented: false, resolved: false, @@ -553,13 +977,161 @@ mod tests { } } - fn pf_bitstream_crop(width: u32, height: u32) -> pf_vkdecode::DisplayCrop { - pf_vkdecode::DisplayCrop { - x: 0, - y: 0, + /// Project one frame with a throwaway guard (the channel is the caller's). + fn project(frame: &DecodedVkFrame) -> NativeVkFrame { + let (tx, _rx) = mpsc::channel(); + project_frame( + frame, + NativeReleaseGuard::new( + tx, + NativeReleaseToken { + seq: 0, + generation: frame.generation, + presented: false, + }, + ), + ) + } + + /// The picture format is the STREAM's, and it must reach the presenter intact: + /// H.264 and H.265 Main deliver NV12, Main 10 delivers P010, RExt 4:4:4 delivers + /// the two-plane 4:4:4 formats. The presenter picks bit depth, MSB packing and + /// chroma siting from exactly this number, so a projection that dropped or + /// defaulted it would render a Main 10 picture with 8-bit math — decoded + /// correctly, displayed wrong, and nothing would flag it. + #[test] + fn the_projection_carries_the_pictures_own_format_whatever_the_codec() { + for format in [ + pf_vkdecode::NV12, + pf_vkdecode::P010, + pf_vkdecode::YUV444_8, + pf_vkdecode::YUV444_10, + ] { + let frame = decoded(format, vk::ImageLayout::VIDEO_DECODE_DST_KHR, 1); + assert_eq!( + project(&frame).vk_format, + crate::video::RawVkFormat(format.as_raw()), + "the presenter reads the format off the frame, never off the codec" + ); + } + } + + /// EVERY field of the projection, against a frame whose values are all distinct + /// (see [`decoded`]): the display crop is what the presenter shows, the coded + /// extent is what it must divide by (the 1088-row lesson), the crop ORIGIN is + /// what its UV-scale path assumes is (0,0), the timeline pair is what it waits, + /// the CICP quadruple is what it does colour maths with, and the decode layout is + /// what it has to restore after sampling. A swap or a drop among any of them is a + /// silently wrong picture, so the list here is deliberately exhaustive — if + /// `NativeVkFrame` grows a field, this test should stop compiling before it can + /// go unchecked. + #[test] + fn the_projection_carries_every_field_the_presenter_can_no_longer_look_up() { + let frame = decoded(pf_vkdecode::P010, vk::ImageLayout::VIDEO_DECODE_DST_KHR, 4); + let p = project(&frame); + // Destructured, not field-accessed: a NEW field on NativeVkFrame breaks this + // pattern and lands the author right here. + let NativeVkFrame { + image, + vk_format, + plane_views, + layer, + layout, + semaphore, + semaphore_value, + generation, width, height, - } + coded_width, + coded_height, + crop_x, + crop_y, + color, + keyframe, + poc, + guard: _, + } = p; + assert_eq!(image, 0x1001); + assert_eq!( + vk_format, + crate::video::RawVkFormat(pf_vkdecode::P010.as_raw()) + ); + assert_eq!( + plane_views, + [0x2002, 0x2003], + "the plane views, in order — NOT the whole-image view (0x2001)" + ); + assert_eq!(layer, 3, "the picture's array layer, not slot 0"); + assert_eq!(layout, NativeVkLayout::DecodeDst); + assert_eq!(semaphore, 0x3001); + assert_eq!( + semaphore_value, 7, + "the frame's timeline value — not its POC (5)" + ); + assert_eq!(generation, 4); + assert_eq!((width, height), (1904, 1072), "the display crop's SIZE"); + assert_eq!( + (coded_width, coded_height), + (1920, 1088), + "the allocated surface — the UV-scale denominator" + ); + assert_eq!((crop_x, crop_y), (8, 4), "the crop ORIGIN, x then y"); + assert_eq!(color.primaries, 9); + assert_eq!(color.transfer, 16); + assert_eq!(color.matrix, 10); + assert!(color.full_range); + assert!( + keyframe, + "is_idr rides through as the pump's re-anchor signal" + ); + assert_eq!(poc, 5); + + // Coincide mode: the picture IS a DPB slot, so the presenter must put the + // layer back in DPB layout after sampling. + let dpb = project(&decoded( + pf_vkdecode::NV12, + vk::ImageLayout::VIDEO_DECODE_DPB_KHR, + 4, + )); + assert_eq!(dpb.layout, NativeVkLayout::DecodeDpb); + } + + /// The construction-time shape refusal, device-independent half. A negotiated + /// shape pf-vkdecode has no picture format for must be refused where + /// `Decoder::new` still has FFmpeg-Vulkan to fall through to — NOT discovered at + /// the first AU, where the only exit is an error streak that demotes PAST that + /// rung to VAAPI/D3D11VA (and on NVIDIA/Linux, straight to software). + #[test] + fn a_stream_shape_with_no_native_picture_format_is_refused_at_construction() { + use crate::video::StreamFormat; + let f = |chroma, bit_depth| { + h265_picture_format(StreamFormat { + chroma_format_idc: chroma, + bit_depth, + }) + }; + // What the envelope DOES admit resolves, and to the right format — Main, + // Main 10 and both RExt 4:4:4 depths. + assert_eq!(f(1, 8).unwrap(), pf_vkdecode::NV12); + assert_eq!(f(1, 10).unwrap(), pf_vkdecode::P010); + assert_eq!(f(3, 8).unwrap(), pf_vkdecode::YUV444_8); + assert_eq!(f(3, 10).unwrap(), pf_vkdecode::YUV444_10); + assert_eq!( + h265_picture_format(StreamFormat::SDR_420_8).unwrap(), + pf_vkdecode::NV12, + "the default/older-host shape is the ordinary one" + ); + // 4:2:2 and monochrome are legal H.265 with no output plumbing here. + assert!(f(2, 8).is_err(), "4:2:2"); + assert!(f(0, 8).is_err(), "monochrome"); + // 12-bit has no output format either, and a depth BELOW 8 must not wrap + // around into a plausible `bit_depth_luma_minus8`. + assert!(f(1, 12).is_err(), "12-bit"); + assert!( + f(1, 0).is_err(), + "an absurd depth refuses, never underflows" + ); + assert!(f(3, 6).is_err()); } #[test] @@ -618,6 +1190,7 @@ mod tests { let (tx, rx) = mpsc::channel(); let frame = NativeVkFrame { image: 0, + vk_format: crate::video::RawVkFormat(pf_vkdecode::NV12.as_raw()), plane_views: [0; 2], layer: 0, layout: NativeVkLayout::DecodeDst, @@ -677,6 +1250,75 @@ mod tests { drop(guard); // must not panic } + /// Concealment is the INTEGRITY warnings, not "any warning at all". + /// + /// h265's `NonZeroReorder` is emitted on the AU that ACTIVATES an SPS with + /// `sps_max_num_reorder_pics > 0` — the opening IDR, and the fresh IDR at every + /// ABR resolution change. pf-bitstream documents it as spec-legal and fully + /// planned (C.5.2 bumping honours the reordering) and excludes it from its own + /// integrity set. Treating it as concealment releases that IDR UNSHOWN, errors, + /// and begs the host for a keyframe: a visible hitch at every renegotiation, on + /// a stream the planner says it planned correctly. + #[test] + fn a_spec_legal_envelope_warning_is_not_concealment() { + use pf_vkdecode::H265PlanWarning as H265; + use pf_vkdecode::PlanWarning as H264; + + // The case from the field: an SPS activation, nothing else. + let reorder = PlanWarnings::H265(vec![H265::NonZeroReorder { + max_num_reorder_pics: 1, + }]); + assert!(!reorder.is_empty(), "it IS a warning and IS logged"); + assert!( + reorder.integrity().is_empty(), + "…but it is not concealment: the frame must be shown, not dropped" + ); + + // h264's twin: an MMCO 5 was planned in full too (the plan carries the + // pre-rebase 8.2.1 values). + let mmco5 = PlanWarnings::H264(vec![H264::Mmco5Rebase]); + assert!(!mmco5.is_empty()); + assert!(mmco5.integrity().is_empty()); + + // Everything that means a reference or a slice was LOST still is — this is + // the H.264 behaviour the hardware-verified path shipped with. + for w in [ + H264::FrameNumGap { + expected: 4, + got: 7, + }, + H264::MissingReference { + context: "list0", + detail: "poc 12".into(), + }, + H264::TruncatedAu { offset: 900 }, + ] { + let warnings = PlanWarnings::H264(vec![w]); + assert_eq!(warnings.integrity().len(), 1, "damage is concealment"); + } + for w in [ + H265::MissingReference { + context: "StCurrBefore", + detail: "poc 12".into(), + }, + H265::TruncatedAu { offset: 900 }, + ] { + let warnings = PlanWarnings::H265(vec![w]); + assert_eq!(warnings.integrity().len(), 1); + } + + // Mixed AU: the damage decides, and the count the error reports is the + // damage count — the spec-legal companion rides along in the log only. + let mixed = PlanWarnings::H265(vec![ + H265::NonZeroReorder { + max_num_reorder_pics: 2, + }, + H265::TruncatedAu { offset: 12 }, + ]); + assert_eq!(mixed.len(), 2); + assert_eq!(mixed.integrity().len(), 1); + } + #[test] fn the_queue_lock_is_shared_only_when_the_families_collide() { // Same family ⇒ same VkQueue (both sides use index 0) ⇒ shared lock. diff --git a/crates/pf-client-core/src/video_vulkan.rs b/crates/pf-client-core/src/video_vulkan.rs index 3aa886e4..3b667483 100644 --- a/crates/pf-client-core/src/video_vulkan.rs +++ b/crates/pf-client-core/src/video_vulkan.rs @@ -374,7 +374,7 @@ impl VulkanDecoder { bail!("Vulkan decode output {sw:?} unsupported (NV12/P010/NV24/P410 only)"); } let vkfc = (*fc).hwctx as *const pf_ffvk::AVVulkanFramesContext; - let vk_format = (*vkfc).format[0] as i32; + let vk_format = crate::video::RawVkFormat((*vkfc).format[0] as i32); let lock_frame = (*vkfc).lock_frame.map_or(0, |f| f as usize); let unlock_frame = (*vkfc).unlock_frame.map_or(0, |f| f as usize); if lock_frame == 0 || unlock_frame == 0 { diff --git a/crates/pf-presenter/src/vk/present.rs b/crates/pf-presenter/src/vk/present.rs index 9eb16815..21e4eb7d 100644 --- a/crates/pf-presenter/src/vk/present.rs +++ b/crates/pf-presenter/src/vk/present.rs @@ -9,7 +9,7 @@ use crate::overlay::OverlayFrame; use anyhow::{bail, Context as _, Result}; use ash::vk; use ash::vk::Handle as _; -use pf_client_core::video::{NativeVkFrame, NativeVkLayout, VkVideoFrame}; +use pf_client_core::video::{NativeVkFrame, NativeVkLayout, RawVkFormat, VkVideoFrame}; impl Presenter { /// Present one frame: route `input` into the video image (staging upload or dmabuf @@ -383,8 +383,7 @@ impl Presenter { width: v.width, height: v.height, }; - let ten_bit = - f.vk_format == vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16.as_raw(); + let (depth, msb_packed) = csc_depth_packing_or_8bit(f.vk_format); // The one path that samples a surface BIGGER than the picture: FFmpeg's // pool is the coded size (1080 → 1088 rows). Scale the UVs to the visible // crop or the alignment padding — the last picture row, replicated by the @@ -397,8 +396,8 @@ impl Presenter { f.height as f32 / f.coded_height as f32, ], f.color, - if ten_bit { 10 } else { 8 }, - ten_bit, + depth, + msb_packed, ); vk_sync = Some(sync); } @@ -434,16 +433,18 @@ impl Presenter { width: v.width, height: v.height, }; - // Depth 8 / 4:2:0 because only H.264 is WIRED to the native decoder - // today, and H.264 in this program is the 8-bit 4:2:0 envelope - // (NV12) — NOT because pf-vkdecode can only produce that. An HEVC - // wiring must carry `pf_vkdecode::DecodedVkFrame::format` through - // to NativeVkFrame and pick depth + MSB packing from it, exactly - // as the FFmpeg-Vulkan arm above does from `f.vk_format`: Main 10 - // decodes to P010 and RExt 4:4:4 to the two-plane 4:4:4 formats, - // and 8-bit transfer/range math over a P010 surface (or 4:2:0 UV - // scaling over a 4:4:4 one) is the plausible-looking-and-wrong - // class. Colour rides the frame (BT.709-limited SDR default). + // Bit depth and MSB packing come from the PICTURE's own format, which + // the decoder stamps on every frame — H.264 and HEVC Main deliver + // NV12 (8-bit), Main 10 delivers P010 (10 significant bits in the + // MSBs of 16), RExt delivers the two-plane 4:4:4 pair — and which can + // change mid-stream when the host renegotiates. Nothing here assumes + // a codec: 8-bit transfer/range math over a P010 surface decodes + // correctly and displays wrong, the plausible-looking-and-wrong class + // this program refuses. Chroma siting needs no decision — the CSC + // shader's quarter-texel 4:2:0 correction self-disables when the + // chroma plane is full width, so the 4:4:4 formats are already right. + // Colour rides the frame (BT.709-limited SDR default). + let (depth, msb_packed) = csc_depth_packing_or_8bit(f.vk_format); self.record_csc( v.framebuffer, extent, @@ -452,8 +453,8 @@ impl Presenter { f.height as f32 / f.coded_height as f32, ], f.color, - 8, - false, + depth, + msb_packed, ); native_layer_barrier( &self.device, @@ -988,7 +989,7 @@ impl Presenter { bail!( "Vulkan-Video pool format {} unsupported (expected 2-plane 4:2:0 or 4:4:4, \ 8/10-bit — 3-plane layouts need a third CSC binding)", - f.vk_format + f.vk_format.0 ); }; // img[0] is creation-constant (only the sync fields need the frames lock). @@ -1049,7 +1050,7 @@ impl Presenter { /// - 3-plane 4:4:4 stays rejected: the CSC pass samples exactly two planes (luma + /// interleaved chroma); a triplanar pool needs a third binding + shader variant. No /// supported driver reports it for HEVC decode today — revisit when one does. -fn vkframe_plane_formats(raw: i32) -> Option<(vk::Format, vk::Format)> { +fn vkframe_plane_formats(raw: RawVkFormat) -> Option<(vk::Format, vk::Format)> { let eight = (vk::Format::R8_UNORM, vk::Format::R8G8_UNORM); let ten = ( vk::Format::R10X6_UNORM_PACK16, @@ -1062,7 +1063,70 @@ fn vkframe_plane_formats(raw: i32) -> Option<(vk::Format, vk::Format)> { (vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16, ten), ] .into_iter() - .find_map(|(f, planes)| (f.as_raw() == raw).then_some(planes)) + .find_map(|(f, planes)| (f.as_raw() == raw.0).then_some(planes)) +} + +/// The CSC pass's `(bit depth, MSB-packed)` pair for a decoded picture's `VkFormat`, +/// or `None` for a format this presenter has no colour math for. +/// +/// This is the whole of what the shader needs to know about the picture format, and +/// it is a property of the STREAM, never of the codec — both hardware lanes carry the +/// real format on the frame ([`VkVideoFrame::vk_format`] from FFmpeg's pool, +/// [`NativeVkFrame::vk_format`] from pf-vkdecode's) and read it here: +/// - 8-bit two-plane (NV12-layout and its 4:4:4 sibling) → depth 8, unpacked. +/// - 10-bit two-plane `3PACK16` (P010-layout and its 4:4:4 sibling) → depth 10, +/// MSB-packed: 10 significant bits live in the MSBs of 16, so a UNORM16 sample +/// reads `code·64/65535` and `csc_rows` folds in the `65535/65472` correction. +/// Rendering those with 8-bit math is not a subtle error — range expansion and the +/// PQ curve both land wrong — but it is a silent one, which is why the depth is +/// derived rather than assumed. +/// +/// Chroma subsampling deliberately does NOT appear: the CSC shader samples both +/// planes in normalized coordinates and self-disables its quarter-texel 4:2:0 siting +/// correction when the chroma plane is full width, so 4:2:0 and 4:4:4 differ only in +/// what the sampler reads. Pure, with a test pinning the table. +fn csc_depth_packing(raw: RawVkFormat) -> Option<(u8, bool)> { + [ + (vk::Format::G8_B8R8_2PLANE_420_UNORM, (8, false)), + (vk::Format::G8_B8R8_2PLANE_444_UNORM, (8, false)), + ( + vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, + (10, true), + ), + ( + vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16, + (10, true), + ), + ] + .into_iter() + .find_map(|(f, dp)| (f.as_raw() == raw.0).then_some(dp)) +} + +/// [`csc_depth_packing`] with the 8-bit fallback for a format neither lane should +/// ever hand us — FFmpeg's arm has already been through [`vkframe_plane_formats`] +/// (which rejects anything not in the same table) and pf-vkdecode refuses a picture +/// format it has no plane mapping for before a session exists. Unreachable is not +/// impossible, so it is said once PER FORMAT rather than silently guessed forever. +/// +/// Per format, not once per process: a session can renegotiate its picture format +/// mid-stream (the ABR/HDR flips this program exists around), so a single latch +/// would let the first unmapped format silence every later, DIFFERENT one — and the +/// second one is the interesting one, because the pair says the gap is systematic. +fn csc_depth_packing_or_8bit(raw: RawVkFormat) -> (u8, bool) { + csc_depth_packing(raw).unwrap_or_else(|| { + use std::sync::Mutex; + static WARNED: Mutex> = Mutex::new(Vec::new()); + let mut seen = WARNED.lock().unwrap_or_else(|e| e.into_inner()); + if !seen.contains(&raw) { + seen.push(raw); + tracing::warn!( + vk_format = raw.0, + "decoded picture in a format the CSC pass has no depth mapping for — \ + rendering it as 8-bit, which is wrong if it is not" + ); + } + (8, false) + }) } /// Flatten the 3×vec4 rows for the push-constant block. @@ -1140,7 +1204,7 @@ mod tests { vk::Format::R10X6G10X6_UNORM_2PACK16, )); // 2-plane 4:2:0, both depths — the classic pair. - let f = |fmt: vk::Format| vkframe_plane_formats(fmt.as_raw()); + let f = |fmt: vk::Format| vkframe_plane_formats(RawVkFormat(fmt.as_raw())); assert_eq!(f(vk::Format::G8_B8R8_2PLANE_420_UNORM), eight); assert_eq!( f(vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16), @@ -1159,7 +1223,83 @@ mod tests { assert_eq!(f(vk::Format::G8_B8R8_2PLANE_422_UNORM), None); assert_eq!(f(vk::Format::G16_B16R16_2PLANE_444_UNORM), None); // Garbage never maps. - assert_eq!(vkframe_plane_formats(0), None); - assert_eq!(vkframe_plane_formats(-1), None); + assert_eq!(vkframe_plane_formats(RawVkFormat(0)), None); + assert_eq!(vkframe_plane_formats(RawVkFormat(-1)), None); + } + + /// The colour-math half of the same decision: what bit depth and packing the CSC + /// pass runs for a decoded picture's format. Both hardware lanes read it off the + /// frame — an HEVC Main 10 stream reaches the native decoder as P010 and the + /// FFmpeg one as the same format, and rendering either with 8-bit range/transfer + /// math is wrong in a way only a side-by-side would catch. + #[test] + fn csc_depth_and_packing_follow_the_pictures_format() { + let d = |fmt: vk::Format| csc_depth_packing(RawVkFormat(fmt.as_raw())); + // 8-bit: H.264, HEVC Main, and the 4:4:4 RExt 8-bit sibling. + assert_eq!(d(vk::Format::G8_B8R8_2PLANE_420_UNORM), Some((8, false))); + assert_eq!(d(vk::Format::G8_B8R8_2PLANE_444_UNORM), Some((8, false))); + // 10-bit, MSB-packed into 16: HEVC Main 10 and its 4:4:4 sibling. The packing + // flag is what recovers exact `code/1023` from a UNORM16 sample. + assert_eq!( + d(vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16), + Some((10, true)) + ); + assert_eq!( + d(vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16), + Some((10, true)) + ); + // Formats the presenter cannot sample at all (they never survive + // `vkframe_plane_formats`, and pf-vkdecode never produces them) have no + // mapping rather than a plausible default. + assert_eq!(d(vk::Format::G8_B8_R8_3PLANE_444_UNORM), None); + assert_eq!(d(vk::Format::G16_B16R16_2PLANE_444_UNORM), None); + assert_eq!(csc_depth_packing(RawVkFormat(0)), None); + assert_eq!(csc_depth_packing(RawVkFormat(-1)), None); + // …and the fallback says 8-bit for those rather than panicking, because a + // wrong-looking picture beats a dead session. + assert_eq!(csc_depth_packing_or_8bit(RawVkFormat(0)), (8, false)); + // The FFmpeg lane's own closure: every pool format it admits must have + // colour math. This half is the table checked against itself, which is + // exactly right HERE — `vkframe_plane_formats` IS that lane's producer (a + // format it rejects never reaches the CSC pass). + for fmt in [ + vk::Format::G8_B8R8_2PLANE_420_UNORM, + vk::Format::G8_B8R8_2PLANE_444_UNORM, + vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, + vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16, + ] { + let raw = RawVkFormat(fmt.as_raw()); + assert!(vkframe_plane_formats(raw).is_some(), "{fmt:?}"); + assert!(csc_depth_packing(raw).is_some(), "{fmt:?}"); + } + } + + /// The NATIVE lane's closure, and the one the table above cannot state: its + /// producer is pf-vkdecode, whose output-format vocabulary this presenter has no + /// dependency on — so the check is against + /// [`pf_client_core::video::native_picture_formats`], which forwards + /// `pf_vkdecode::OUTPUT_FORMATS` verbatim. + /// + /// Without it, pf-vkdecode growing a fifth output format (12-bit RExt) would + /// build images fine, reach `csc_depth_packing_or_8bit`, render 10 or 12 bits as + /// 8 behind one warn line — and every test here would stay green, because they + /// only ever asked the FFmpeg lane's table about itself. Note there is NO + /// converse assertion: pf-vkdecode is not obliged to produce every format the + /// presenter can sample. + #[test] + fn every_format_the_native_decoder_can_deliver_has_colour_math_here() { + let produced = pf_client_core::video::native_picture_formats(); + assert!(!produced.is_empty(), "the vocabulary must not be empty"); + for raw in produced { + assert!( + csc_depth_packing(raw).is_some(), + "pf-vkdecode delivers vk_format {} and the CSC pass has no depth \ + mapping for it — it would render as 8-bit", + raw.0 + ); + // …and the sampler contract too: pf-vkdecode makes the per-plane views + // itself, but the two tables must agree on what a plane pair means. + assert!(vkframe_plane_formats(raw).is_some(), "vk_format {}", raw.0); + } } } diff --git a/crates/pf-vkdecode/src/caps.rs b/crates/pf-vkdecode/src/caps.rs index edb8b299..4a51f2f1 100644 --- a/crates/pf-vkdecode/src/caps.rs +++ b/crates/pf-vkdecode/src/caps.rs @@ -28,6 +28,17 @@ pub const YUV444_8: vk::Format = vk::Format::G8_B8R8_2PLANE_444_UNORM; /// 10-bit 4:4:4 two-plane: H.265 RExt 4:4:4 10-bit, where the device advertises it. pub const YUV444_10: vk::Format = vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16; +/// EVERY picture format a pf-vkdecode session can deliver — this crate's whole +/// output vocabulary, in one place. +/// +/// It exists so a CONSUMER's per-format table (the presenter's CSC bit-depth and +/// MSB-packing map) can be pinned against the PRODUCER rather than against a +/// hand-copied list of its own: a fifth format added here (12-bit RExt, say) breaks +/// the consumer's test instead of silently rendering through that consumer's +/// fallback. [`plane_formats`] and [`crate::caps_h265::output_format_for`] are both +/// tested to agree with it, so the vocabulary can only grow in one edit. +pub const OUTPUT_FORMATS: [vk::Format; 4] = [NV12, P010, YUV444_8, YUV444_10]; + /// The `R*`/`R*G*` per-plane view formats the presenter's sampler path needs for /// one picture format, or `None` for a format this crate has no plane mapping for. /// @@ -749,6 +760,36 @@ mod tests { } } + /// [`OUTPUT_FORMATS`] is the vocabulary a CONSUMER pins its own per-format + /// table against, so it has to be the whole of what this crate can deliver — + /// no more (a format listed here but unmappable would fail a pool build) and + /// no less (a format produced but unlisted is exactly the silent + /// wrong-colour-math case the listing exists to stop). + #[test] + fn the_output_format_vocabulary_is_the_whole_of_what_this_crate_delivers() { + for format in OUTPUT_FORMATS { + assert!( + plane_formats(format).is_some(), + "{format:?} is advertised as an output but has no plane views" + ); + } + // Every (chroma, depth) pair the H.265 envelope admits resolves INTO the + // vocabulary — the one producer that picks a format from stream facts. + for chroma in 0u8..=4 { + for depth in 0u8..=4 { + if let Some(f) = crate::caps_h265::output_format_for(chroma, depth) { + assert!( + OUTPUT_FORMATS.contains(&f), + "output_format_for({chroma}, {depth}) = {f:?} is outside \ + OUTPUT_FORMATS" + ); + } + } + } + // H.264 is the 8-bit 4:2:0 envelope — its one format is in there too. + assert!(OUTPUT_FORMATS.contains(&NV12)); + } + #[test] fn a_coincide_device_derives_coincide_with_one_shared_format() { let caps = derive_caps(&radv_like()).unwrap(); diff --git a/crates/pf-vkdecode/src/caps_h265.rs b/crates/pf-vkdecode/src/caps_h265.rs index 8779e898..901f6414 100644 --- a/crates/pf-vkdecode/src/caps_h265.rs +++ b/crates/pf-vkdecode/src/caps_h265.rs @@ -109,6 +109,39 @@ impl H265ProfileKey { }) } + /// The key for a stream whose (chroma format, bit depth) the SESSION already + /// negotiated but whose SPS has not arrived yet — the construction-time probe's + /// entry point ([`crate::VkH265Decoder::probe_stream_support`]). + /// + /// The profile idc is the one thing the negotiation does not carry, so it is + /// derived from the pair: 4:2:0 8-bit → Main, 4:2:0 10-bit → Main 10, 4:4:4 → + /// Format Range Extensions (4:4:4 is only expressible in RExt, so that leg is + /// exact — and it is the leg the probe exists for). A stream that turns out to + /// carry a DIFFERENT profile idc for the same pair (RExt 4:2:0, say) simply + /// re-queries under its real key at the first AU: [`Self::from_stream`] stays + /// the authority once the SPS is in hand, and this one never widens what that + /// gate admits — every combination it cannot express is refused here too. + pub fn from_negotiated( + chroma_format_idc: u8, + bit_depth_luma_minus8: u8, + ) -> Result { + let general_profile_idc = match (chroma_format_idc, bit_depth_luma_minus8) { + (1, 0) => 1, + (1, 2) => 2, + (3, _) => 4, + // Everything else is outside the envelope; hand it to `from_stream` + // with a profile that cannot rescue it so ONE gate produces the error. + _ => 4, + }; + Self::from_stream( + general_profile_idc, + chroma_format_idc, + false, + bit_depth_luma_minus8, + bit_depth_luma_minus8, + ) + } + /// The picture format a session on this profile decodes to, or `None` for a /// combination outside the envelope (unreachable off [`Self::from_stream`], /// which already gated it). @@ -401,6 +434,58 @@ mod tests { assert_eq!(output_format_for(2, 0), None, "4:2:2 has no output format"); } + /// The negotiated-facts constructor: the session knows the chroma format and + /// bit depth from the host's Welcome long before the first SPS, and that is + /// enough to pick the profile a punktfunk host encodes the pair with — which + /// is what lets the client PROBE the device before it commits to the native + /// decoder rung. + #[test] + fn the_negotiated_pair_picks_the_profile_a_host_encodes_it_with() { + let main = H265ProfileKey::from_negotiated(1, 0).unwrap(); + assert_eq!( + main, + H265ProfileKey::from_stream(1, 1, false, 0, 0).unwrap() + ); + assert_eq!(main.output_format(), Some(NV12)); + + let main10 = H265ProfileKey::from_negotiated(1, 2).unwrap(); + assert_eq!( + main10, + H265ProfileKey::from_stream(2, 1, false, 2, 2).unwrap() + ); + assert_eq!(main10.output_format(), Some(P010)); + + // 4:4:4 is only expressible in RExt, so this leg is exact — and it is the + // one the probe exists for (a 4:4:4 session on a device with no 4:4:4 + // decode format used to burn the ladder mid-stream). + let rext8 = H265ProfileKey::from_negotiated(3, 0).unwrap(); + assert_eq!( + rext8, + H265ProfileKey::from_stream(4, 3, false, 0, 0).unwrap() + ); + assert_eq!(rext8.output_format(), Some(YUV444_8)); + let rext10 = H265ProfileKey::from_negotiated(3, 2).unwrap(); + assert_eq!(rext10.output_format(), Some(YUV444_10)); + + // It never admits what `from_stream` refuses: outside-envelope pairs come + // back typed, so the probe REFUSES rather than guessing a profile. + assert_eq!( + H265ProfileKey::from_negotiated(2, 0).unwrap_err(), + H265ParamsError::UnsupportedChromaFormat(2) + ); + assert_eq!( + H265ProfileKey::from_negotiated(0, 0).unwrap_err(), + H265ParamsError::UnsupportedChromaFormat(0) + ); + assert_eq!( + H265ProfileKey::from_negotiated(1, 4).unwrap_err(), + H265ParamsError::UnsupportedBitDepth { + luma_minus8: 4, + chroma_minus8: 4 + } + ); + } + #[test] fn stream_facts_outside_the_envelope_are_refused_by_the_profile_builder() { assert_eq!( diff --git a/crates/pf-vkdecode/src/decoder_h265.rs b/crates/pf-vkdecode/src/decoder_h265.rs index 7e497069..41b01cd4 100644 --- a/crates/pf-vkdecode/src/decoder_h265.rs +++ b/crates/pf-vkdecode/src/decoder_h265.rs @@ -218,10 +218,14 @@ impl VkH265Decoder { /// and features, truthful queue families) — held for this decoder's whole /// lifetime, not just this call. The device must additionally have been /// created with `VK_KHR_video_decode_h265` enabled; that part of the contract - /// is CHECKED below rather than trusted, because it is the one the client - /// wiring is most likely to get wrong (the presenter creates its device with - /// `VK_KHR_video_decode_h264` alone today) and getting it wrong is undefined - /// behaviour at session creation rather than an error. + /// is checked below AS FAR AS IT CAN BE — the check reads the decode queue + /// family's advertised `videoCodecOperations`, which is the + /// device's own claim about the family, not proof that the client enabled the + /// extension at `vkCreateDevice`. (punktfunk's presenter enables h264 + h265 + + /// av1, filtered by what the device supports — `pf-presenter/src/vk/setup.rs` + /// — so the two coincide there.) Getting it wrong is undefined behaviour at + /// session creation rather than an error, which is why the family check runs + /// before anything is queried or created. pub unsafe fn new( handles: &DeviceHandles, lock: Box, @@ -250,6 +254,43 @@ impl VkH265Decoder { }) } + /// Ask the device, BEFORE a single AU is fed, whether it can decode a stream of + /// the negotiated (chroma format, bit depth) shape — the construction-time half + /// of what the lazy `ensure_state` path would otherwise only discover at the + /// first SPS. + /// + /// Why it exists: the session's picture format is the STREAM's, and a device + /// that advertises H.265 decode need not advertise a picture format for every + /// shape of it — 4:4:4 RExt is absent everywhere but NVIDIA, and 10-bit is + /// absent on some older silicon. Discovering that lazily makes the refusal a + /// mid-stream ERROR STREAK, which demotes past the FFmpeg rungs to + /// VAAPI/D3D11VA/software; discovering it here makes it a construction failure, + /// which the client's ladder answers by falling through to the next rung with + /// the session's hardware decode intact. Same query, same derivation, same + /// [`crate::CapsError`] — only the timing differs. + /// + /// The negotiated facts are a HINT (the in-band SPS is authoritative), so this + /// is deliberately not a promise that decode will succeed: the level ceiling and + /// an SPS that disagrees with the Welcome still surface at the first AU. What it + /// does guarantee is that a shape the device provably cannot host never gets a + /// session built for it. + pub fn probe_stream_support( + &self, + chroma_format_idc: u8, + bit_depth_luma_minus8: u8, + ) -> Result<(), VkDecodeError> { + let key = H265ProfileKey::from_negotiated(chroma_format_idc, bit_depth_luma_minus8)?; + let wanted = key + .output_format() + .expect("from_negotiated gated the chroma/depth combination"); + // SAFETY: the constructor's `DeviceHandles` contract holds for this + // decoder's whole lifetime, so the physical device is live — the same + // proof `ensure_state`'s identical call carries. + let raw = unsafe { query_h265_caps(&self.dev, key) }.map_err(VkDecodeError::from)?; + derive_caps_h265(&raw, wanted)?; + Ok(()) + } + /// Decode one access unit. Returns the next display-ready frame, if the /// planner declared one. /// diff --git a/crates/pf-vkdecode/src/lib.rs b/crates/pf-vkdecode/src/lib.rs index 4e73b8f7..43aa66e4 100644 --- a/crates/pf-vkdecode/src/lib.rs +++ b/crates/pf-vkdecode/src/lib.rs @@ -101,6 +101,15 @@ pub use pf_bitstream::h264::ColourDescription; pub use pf_bitstream::h264::DisplayCrop; /// [`VkH264Decoder::take_warnings`]'s warning type. pub use pf_bitstream::h264::PlanWarning; +/// [`VkH265Decoder::take_warnings`]'s warning type — the H.265 twin of +/// [`PlanWarning`], renamed rather than shadowed because the two enums are +/// genuinely different (H.264 has `FrameNumGap`/`Mmco5Rebase`, H.265 has +/// `NonZeroReorder`) and a consumer dispatching per codec must be able to name +/// BOTH. Without it the client could only render warnings as strings — and it has +/// to BRANCH on them: `NonZeroReorder` and `Mmco5Rebase` are spec-legal facts the +/// planner planned in full, not concealment, and dropping their frames would cost +/// a visible hitch at every SPS activation. +pub use pf_bitstream::h265::PlanWarning as H265PlanWarning; pub use caps::derive_caps; pub use caps::plane_formats; @@ -110,6 +119,7 @@ pub use caps::MaxLevelIdc; pub use caps::RawH264Caps; pub use caps::VideoFormat; pub use caps::NV12; +pub use caps::OUTPUT_FORMATS; pub use caps::P010; pub use caps::YUV444_10; pub use caps::YUV444_8;