diff --git a/Cargo.lock b/Cargo.lock index 9f719168..7043fb23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3215,6 +3215,16 @@ dependencies = [ "x11rb", ] +[[package]] +name = "pf-vkdecode" +version = "0.24.0" +dependencies = [ + "ash", + "cros-codecs", + "pf-bitstream", + "tracing", +] + [[package]] name = "pf-win-display" version = "0.24.0" diff --git a/Cargo.toml b/Cargo.toml index 003b9b97..309eab01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "crates/pf-capture", "crates/pf-inject", "crates/pf-vdisplay", + "crates/pf-vkdecode", "crates/pyrowave-sys", "crates/libvpl-sys", "clients/probe", diff --git a/crates/pf-bitstream/src/h264.rs b/crates/pf-bitstream/src/h264.rs index d4a9e6e1..1813f8ba 100644 --- a/crates/pf-bitstream/src/h264.rs +++ b/crates/pf-bitstream/src/h264.rs @@ -127,7 +127,14 @@ pub struct SlicePlan { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct RefPic { pub id: PicId, - pub poc: i32, + /// The stored picture's 8.2.1 field order counts. Equal for a progressive frame + /// UNLESS the PPS set `bottom_field_pic_order_in_frame_present_flag` and the + /// slice carried a nonzero `delta_pic_order_cnt_bottom` — backend picparams + /// formats want the pair, and collapsing to one value would fabricate the bottom + /// count. After an MMCO 5 these are the picture's REBASED values (8.2.5.4.5), + /// which is what later AUs reference it by — see [`PlanWarning::Mmco5Rebase`]. + pub top_field_order_cnt: i32, + pub bottom_field_order_cnt: i32, pub is_long_term: bool, /// `frame_num` for short-term references, `LongTermFrameIdx` for long-term ones — /// the pair DXVA and Vulkan both key reference pictures by. @@ -160,6 +167,14 @@ pub enum PlanWarning { /// or a slice belonging to another picture (mis-split AU). The plan covers only /// the slices before the cut; `offset` is the byte position of the cut in the AU. TruncatedAu { offset: usize }, + /// The AU carried an MMCO 5 (8.2.5.4.5): the DPB was drained and the CURRENT + /// picture's stored frame_num/POC were rebased to zero AFTER its plan was + /// captured. Spec-legal and fully planned — the [`PicturePlan`] holds the + /// pre-rebase 8.2.1 values a decoder submits with, while later AUs reference the + /// picture by its rebased values ([`RefPic`] carries the stored pair). punktfunk + /// hosts never emit MMCO 5, so this warning is the field signal if that + /// assumption ever breaks. + Mmco5Rebase, } /// The AU cannot be planned at all. @@ -510,6 +525,16 @@ impl H264Planner { "separate colour plane coding (separate_colour_plane_flag == 1)", )); } + // A.3.1 caps the DPB at 16 frames; the only route past the cap is the VUI's + // max_dec_frame_buffering, an unbounded ue(v) the vendored parser reads + // uncapped. No hardware decoder implements a deeper DPB — a larger value is a + // corrupt (or hostile) VUI, not a feature request — and backends size real + // slot pools from this number, so it is gated here, at SPS activation. + if sps.max_dpb_frames() > 16 { + return Err(PlanError::OutsideEnvelope( + "DPB deeper than 16 frames (max_dec_frame_buffering)", + )); + } Ok(()) } @@ -1086,7 +1111,8 @@ impl H264Planner { slots.push(Some(( RefPic { id, - poc: pic.pic_order_cnt, + top_field_order_cnt: pic.top_field_order_cnt, + bottom_field_order_cnt: pic.bottom_field_order_cnt, is_long_term, frame_num_or_lt_idx, }, @@ -1119,7 +1145,8 @@ impl H264Planner { if let Some((substitute, frame_num)) = prev_existing.or(first_existing) { out.push(RefPic { id: substitute.id, - poc: substitute.poc, + top_field_order_cnt: substitute.top_field_order_cnt, + bottom_field_order_cnt: substitute.bottom_field_order_cnt, is_long_term: false, frame_num_or_lt_idx: frame_num, }); @@ -1479,6 +1506,7 @@ impl H264Planner { self.prev_pic_info.fill(&pic); if pic.has_mmco_5 { + warnings.push(PlanWarning::Mmco5Rebase); // C.4.5.3 "Bumping process" // The bumping process is invoked in the following cases: // Clause 3: @@ -1566,6 +1594,7 @@ mod tests { use cros_codecs::codec::h264::parser::PpsBuilder; use cros_codecs::codec::h264::parser::Profile; use cros_codecs::codec::h264::parser::SpsBuilder; + use cros_codecs::codec::h264::parser::VuiParams; use cros_codecs::codec::h264::synthesizer::Synthesizer; use super::*; @@ -1704,9 +1733,11 @@ mod tests { let ids1: Vec = slice.ref_list1.iter().map(|r| r.id).collect(); assert_ne!(ids0, ids1, "list1 must not be list0's ordering"); - // 8.2.4.2.3: list0 leads with the past, list1 with the future. - assert!(slice.ref_list0[0].poc < plan.picture.pic_order_cnt); - assert!(slice.ref_list1[0].poc > plan.picture.pic_order_cnt); + // 8.2.4.2.3: list0 leads with the past, list1 with the future + // (a frame's PicOrderCnt is the min of its field order counts). + let poc = |r: &RefPic| r.top_field_order_cnt.min(r.bottom_field_order_cnt); + assert!(poc(&slice.ref_list0[0]) < plan.picture.pic_order_cnt); + assert!(poc(&slice.ref_list1[0]) > plan.picture.pic_order_cnt); } } @@ -2077,6 +2108,84 @@ mod tests { )); } + #[test] + fn a_dpb_deeper_than_16_frames_is_rejected_as_outside_the_envelope() { + // The one route past the A.3.1 16-frame cap: the VUI bitstream restriction's + // max_dec_frame_buffering, an unbounded ue(v) that overrides the level-derived + // size in `Sps::max_dpb_frames`. The builder has no VUI-restriction setter, so + // the Sps is constructed directly (its fields are public). + let sps = Sps { + profile_idc: Profile::Main as u8, + level_idc: Level::L4, + frame_mbs_only_flag: true, + direct_8x8_inference_flag: true, + max_num_ref_frames: 4, + vui_parameters_present_flag: true, + vui_parameters: VuiParams { + bitstream_restriction_flag: true, + max_dec_frame_buffering: 17, + ..Default::default() + }, + ..Default::default() + }; + let mut au = Vec::new(); + Synthesizer::<'_, Sps, _>::synthesize(3, &sps, &mut au, true).unwrap(); + + let err = H264Planner::new().plan_au(&au).unwrap_err(); + assert!( + matches!(err, PlanError::OutsideEnvelope(what) if what.contains("DPB")), + "{err:?}" + ); + } + + /// MMCO 5 writer: op 5 takes NO argument (Table 7-9), so the generic + /// [`write_p_slice`] — whose supported ops all take exactly one — cannot author + /// it. + fn write_p_slice_mmco5(frame_num: u32, poc_lsb: u32) -> Vec { + let mut buf = Vec::new(); + { + let mut w = NaluWriter::new(&mut buf, true); + w.write_header(1, NaluType::Slice as u8).unwrap(); + w.write_ue(0u32).unwrap(); // first_mb_in_slice + w.write_ue(0u32).unwrap(); // slice_type: P + w.write_ue(0u32).unwrap(); // pic_parameter_set_id + w.write_f(4, frame_num).unwrap(); // frame_num, u(4) + w.write_f(4, poc_lsb).unwrap(); // pic_order_cnt_lsb, u(4) + w.write_f(1, 1u32).unwrap(); // num_ref_idx_active_override_flag + w.write_ue(0u32).unwrap(); // num_ref_idx_l0_active_minus1 + w.write_f(1, 0u32).unwrap(); // ref_pic_list_modification_flag_l0 + w.write_f(1, 1u32).unwrap(); // adaptive_ref_pic_marking_mode_flag + w.write_ue(5u32).unwrap(); // memory_management_control_operation 5 + w.write_ue(0u32).unwrap(); // memory_management_control_operation end + w.write_se(0i32).unwrap(); // slice_qp_delta + w.write_f(1, 1u32).unwrap(); // rbsp stop bit + while !w.aligned() { + w.write_f(1, 0u32).unwrap(); + } + } + buf + } + + #[test] + fn an_mmco_5_is_planned_with_a_rebase_warning_not_rejected() { + let (sps, pps) = authored_sps_pps(); + let mut au0 = param_set_au(&sps, &pps); + au0.extend(write_idr_slice()); + let au1 = write_p_slice_mmco5(1, 2); + + let mut planner = H264Planner::new(); + let p0 = planner.plan_au(&au0).unwrap(); + let p1 = planner.plan_au(&au1).unwrap(); + + assert!(p1.warnings.contains(&PlanWarning::Mmco5Rebase)); + // The plan carries the pre-rebase 8.2.1 values a decoder submits with; the + // zeroed frame_num/POC exist only in the STORED picture later AUs reference. + assert_eq!(p1.picture.frame_num, 1); + assert_eq!(p1.picture.pic_order_cnt, 2); + // And the op's C.4.5.3 clause-3 drain ran: the IDR is display-ready. + assert!(p1.dpb.outputs.contains(&p0.dpb.stored.unwrap())); + } + #[test] fn a_separate_colour_plane_sps_is_rejected_as_outside_the_envelope() { // SpsBuilder has no separate_colour_plane setter; construct the Sps directly diff --git a/crates/pf-vkdecode/Cargo.toml b/crates/pf-vkdecode/Cargo.toml new file mode 100644 index 00000000..e1625166 --- /dev/null +++ b/crates/pf-vkdecode/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "pf-vkdecode" +description = "Native Vulkan Video H.264 decode for the clients (M2): StdVideo parameter-set/picture conversion and DPB slot management over pf-bitstream's AuPlans — the CPU-testable half; session, memory and command recording follow in WP-B (design/client-native-decode.md §3.2)" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[dependencies] +ash = "0.38" +# Direct dependency on the vendored parser crate, not just pf-bitstream: the parameter-set +# conversion consumes the parser's `Sps`/`Pps` types wholesale (pf-bitstream re-exports only +# `Level`/`SliceHeader`), and the same path means the one crate instance the workspace already +# builds — no duplicate types. +cros-codecs = { path = "../pf-bitstream/vendor/cros-codecs" } +pf-bitstream = { path = "../pf-bitstream" } +tracing = "0.1" + +[lints] +workspace = true diff --git a/crates/pf-vkdecode/src/lib.rs b/crates/pf-vkdecode/src/lib.rs new file mode 100644 index 00000000..9b59e356 --- /dev/null +++ b/crates/pf-vkdecode/src/lib.rs @@ -0,0 +1,42 @@ +//! Native Vulkan Video H.264 decode for the clients — M2 of the native-decode program +//! (design/client-native-decode.md §3.2). +//! +//! This crate sits between [`pf_bitstream`]'s per-AU planning and Vulkan Video +//! submission. WP-A (this round) is the CPU-testable half, and everything in it runs +//! without a GPU: +//! +//! - [`params`]: the vendored parser's `Sps`/`Pps` converted into the +//! `StdVideoH264*ParameterSet` structs session parameters are created from, behind +//! owning wrappers ([`OwnedStdSps`]/[`OwnedStdPps`]) because the Std structs embed +//! raw pointers. +//! - [`slots`]: [`SlotMap`], the hardware DPB slot ledger keyed by +//! [`pf_bitstream::h264::PicId`]. pf-bitstream's DPB decides what lives and dies; +//! this map only translates ids to slot indices and refuses to guess. +//! - [`pic`]: [`plan_to_vk`], one [`pf_bitstream::h264::AuPlan`] converted into the +//! `StdVideoDecodeH264PictureInfo`/`StdVideoDecodeH264ReferenceInfo` set plus slice +//! offsets and slot bindings a `vkCmdDecodeVideoKHR` call wants. +//! +//! WP-B adds the other half: VkVideoSessionKHR/session-parameters objects, DPB image +//! memory, command recording and result queries. Nothing here touches a VkDevice. +//! +//! Unsafe posture: unlike pf-bitstream (which forbids unsafe outright), this crate +//! cannot — the `ash::vk::native` bindgen structs are zero-initialized the way the +//! encode side does it (`pf-encode/src/enc/linux/vk_build.rs`), and WP-B brings Vulkan +//! FFI. Every unsafe block therefore carries a written `// SAFETY:` proof, enforced: +#![deny(clippy::undocumented_unsafe_blocks)] + +pub mod params; +pub mod pic; +pub mod slots; + +pub use params::pps_to_std; +pub use params::sps_to_std; +pub use params::OwnedStdPps; +pub use params::OwnedStdSps; +pub use params::ParamsError; +pub use pic::plan_to_vk; +pub use pic::DecodePlanVk; +pub use pic::PlanToVkError; +pub use pic::VkRef; +pub use slots::SlotError; +pub use slots::SlotMap; diff --git a/crates/pf-vkdecode/src/params.rs b/crates/pf-vkdecode/src/params.rs new file mode 100644 index 00000000..9a5fe84a --- /dev/null +++ b/crates/pf-vkdecode/src/params.rs @@ -0,0 +1,717 @@ +//! Parameter-set conversion: the vendored parser's [`Sps`]/[`Pps`] into the +//! `StdVideoH264*ParameterSet` structs a Vulkan Video session-parameters object is +//! created from (WP-B's `vkCreateVideoSessionParametersKHR`). +//! +//! The Std structs embed raw pointers (`pOffsetForRefFrame`, `pScalingLists`, +//! `pSequenceParameterSetVui`), so conversion returns OWNING wrappers instead of bare +//! structs — see [`OwnedStdSps`] for the aliasing/lifetime contract. +//! +//! VUI is deliberately not converted: a DECODE session consumes no VUI (it shapes +//! display, not reconstruction), so `vui_parameters_present_flag` stays 0 and +//! `pSequenceParameterSetVui` stays null. Colour handling rides +//! [`pf_bitstream::h264::PicturePlan`] into the presenter instead, exactly as the +//! FFmpeg-based path did. + +use ash::vk::native as hh; +use cros_codecs::codec::h264::parser::Level; +pub use cros_codecs::codec::h264::parser::Pps; +pub use cros_codecs::codec::h264::parser::Sps; + +/// A parameter set that cannot be represented as a StdVideo struct. All of these are +/// outside the punktfunk decode envelope (8-bit 4:2:0 streams from encoders we +/// control), so hitting one is a stream-integrity failure, not a feature gap — +/// reject-with-error rather than submit a half-truth to a driver. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ParamsError { + /// `profile_idc` has no `StdVideoH264ProfileIdc` code point. Vulkan defines + /// Baseline (66), Main (77), High (100) and High 4:4:4 Predictive (244); + /// Extended/High10/High422 land here. + UnmappableProfileIdc(u8), + /// `chroma_format_idc` past 3 — not legal H.264 to begin with. + InvalidChromaFormatIdc(u8), + /// `pic_order_cnt_type` past 2 — not legal H.264 to begin with. + InvalidPocType(u8), + /// `weighted_bipred_idc` of 3: representable in the two-bit field, invalid per + /// 7.4.2.2, and no `StdVideoH264WeightedBipredIdc` code point exists for it. + InvalidWeightedBipredIdc(u8), + /// FMO (`num_slice_groups_minus1 > 0`): `StdVideoH264PictureParameterSet` has no + /// slice-group fields at all — Vulkan Video cannot express it. + SliceGroups(u32), +} + +impl std::fmt::Display for ParamsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ParamsError::UnmappableProfileIdc(idc) => { + write!( + f, + "profile_idc {idc} has no StdVideoH264ProfileIdc code point" + ) + } + ParamsError::InvalidChromaFormatIdc(idc) => { + write!(f, "invalid chroma_format_idc {idc}") + } + ParamsError::InvalidPocType(t) => write!(f, "invalid pic_order_cnt_type {t}"), + ParamsError::InvalidWeightedBipredIdc(idc) => { + write!(f, "invalid weighted_bipred_idc {idc}") + } + ParamsError::SliceGroups(n) => { + write!( + f, + "FMO ({} slice groups) is not expressible in Vulkan Video", + n + 1 + ) + } + } + } +} + +impl std::error::Error for ParamsError {} + +/// The converted SPS plus the heap allocations its embedded pointers target. +/// +/// `StdVideoH264SequenceParameterSet` points at data it does not contain: the +/// POC-type-1 offset array (`pOffsetForRefFrame`) and the scaling lists +/// (`pScalingLists`). This wrapper owns that data, and the ownership design is the +/// contract WP-B builds on: +/// +/// - The backing is boxed, so the wrapper may be MOVED freely: moving it relocates +/// the `Box` handles (pointer values), never the heap blocks the Std struct's +/// pointers hold the addresses of. +/// - [`Self::std`] hands the struct out by shared reference. The struct is `Copy`; a +/// copy taken out of the wrapper still points INTO the wrapper's backing and must +/// not outlive it. The intended use is passing the reference straight into +/// `vkCreateVideoSessionParametersKHR`, which copies all parameter data before +/// returning — keeping the wrapper alive across that call is the whole obligation. +/// - Nothing exposes mutation of the backing, so for the wrapper's lifetime the +/// pointed-to data is immutable and the `*const` aliasing rules hold trivially. +/// - Deliberately NOT `Clone`: a derived clone would duplicate the pointer VALUES but +/// not the backing, silently tying the clone's validity to the original's lifetime. +/// Re-convert from the `Sps` instead — conversion is cheap and pure. +#[derive(Debug)] +pub struct OwnedStdSps { + std: hh::StdVideoH264SequenceParameterSet, + /// `pOffsetForRefFrame`'s target (POC type 1 only, else `None`/null). + _offset_backing: Option>, + /// `pScalingLists`' target (`seq_scaling_matrix_present_flag` only, else null). + _scaling_backing: Option>, +} + +impl OwnedStdSps { + /// The Std struct, valid for as long as `self` lives (see the type-level + /// contract; do not let a `Copy` of it outlive the wrapper). + pub fn std(&self) -> &hh::StdVideoH264SequenceParameterSet { + &self.std + } +} + +/// The converted PPS plus the scaling-list allocation its `pScalingLists` targets. +/// Same ownership contract as [`OwnedStdSps`], with the one pointer. +#[derive(Debug)] +pub struct OwnedStdPps { + std: hh::StdVideoH264PictureParameterSet, + _scaling_backing: Option>, +} + +impl OwnedStdPps { + /// The Std struct, valid for as long as `self` lives (see [`OwnedStdSps`]). + pub fn std(&self) -> &hh::StdVideoH264PictureParameterSet { + &self.std + } +} + +/// H.264 `level_idc` (value-coded: 10 ⇒ 1.0) to Vulkan's index-coded +/// `StdVideoH264LevelIdc`. +const fn level_to_std(level: Level) -> hh::StdVideoH264LevelIdc { + match level { + Level::L1 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_1_0, + // Vulkan has no 1b code point. 1b is signalled on the wire as level_idc 11 + // plus constraint_set3_flag — the flag is mapped, so 1.1 is the faithful cap. + Level::L1B => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_1_1, + Level::L1_1 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_1_1, + Level::L1_2 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_1_2, + Level::L1_3 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_1_3, + Level::L2_0 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_2_0, + Level::L2_1 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_2_1, + Level::L2_2 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_2_2, + Level::L3 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_3_0, + Level::L3_1 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_3_1, + Level::L3_2 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_3_2, + Level::L4 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_4_0, + Level::L4_1 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_4_1, + Level::L4_2 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_4_2, + Level::L5 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_5_0, + Level::L5_1 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_5_1, + Level::L5_2 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_5_2, + Level::L6 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_6_0, + Level::L6_1 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_6_1, + Level::L6_2 => hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_6_2, + } +} + +/// Pack the parser's scaling-list arrays into the Std layout. +/// +/// The vendored parser has already run 7.3.2.1.1.1 plus the Table 7-2 default and +/// fallback rules, so the arrays hold the fully RESOLVED lists. Every resolved list is +/// therefore declared present verbatim (`scaling_list_present_mask` set, +/// `use_default_scaling_matrix_mask` 0) and the driver applies no further inference — +/// simpler and equivalent to re-encoding which lists came from the bitstream. +/// +/// `num_8x8` is the count of 8x8 lists the parser actually resolved: 2 for 4:2:0/4:2:2 +/// (Y intra/inter), 6 for 4:4:4, and 0 for a PPS without `transform_8x8_mode_flag` +/// (whose 8x8 arrays are untouched zeros and must not be declared present). +fn scaling_lists_to_std( + lists_4x4: &[[u8; 16]; 6], + lists_8x8: &[[u8; 64]; 6], + num_8x8: u16, +) -> hh::StdVideoH264ScalingLists { + // SAFETY: StdVideoH264ScalingLists is a plain-C bindgen struct of two u16 masks + // and two byte arrays; the all-zero bit pattern is a valid value for every field. + let mut std: hh::StdVideoH264ScalingLists = unsafe { std::mem::zeroed() }; + std.scaling_list_present_mask = 0x3F | (((1u16 << num_8x8) - 1) << 6); + std.use_default_scaling_matrix_mask = 0; + std.ScalingList4x4 = *lists_4x4; + // All six 8x8 arrays are copied even when only two are declared present; the + // driver ignores entries whose mask bit is clear. + std.ScalingList8x8 = *lists_8x8; + std +} + +/// Convert one SPS into the Std struct (owning wrapper), mapping every field the +/// H.264 decode profile consumes. VUI is skipped by design (module docs). +pub fn sps_to_std(sps: &Sps) -> Result { + // StdVideoH264ProfileIdc code points equal the profile_idc values they name, so + // recognised ones pass through; everything else has no representation. + let profile_idc = match u32::from(sps.profile_idc) { + p @ (66 | 77 | 100 | 244) => p, + _ => return Err(ParamsError::UnmappableProfileIdc(sps.profile_idc)), + }; + if sps.chroma_format_idc > 3 { + return Err(ParamsError::InvalidChromaFormatIdc(sps.chroma_format_idc)); + } + if sps.pic_order_cnt_type > 2 { + return Err(ParamsError::InvalidPocType(sps.pic_order_cnt_type)); + } + + // SAFETY: StdVideoH264SequenceParameterSet is a plain-C bindgen struct of + // integers, a bitfield word and const pointers; all-zero is a valid value for + // every field (null for the pointers) and is the "everything absent" baseline the + // field writes below build on. Same idiom as pf-encode's vk_build.rs. + let mut std: hh::StdVideoH264SequenceParameterSet = unsafe { std::mem::zeroed() }; + + std.flags + .set_constraint_set0_flag(u32::from(sps.constraint_set0_flag)); + std.flags + .set_constraint_set1_flag(u32::from(sps.constraint_set1_flag)); + std.flags + .set_constraint_set2_flag(u32::from(sps.constraint_set2_flag)); + std.flags + .set_constraint_set3_flag(u32::from(sps.constraint_set3_flag)); + std.flags + .set_constraint_set4_flag(u32::from(sps.constraint_set4_flag)); + std.flags + .set_constraint_set5_flag(u32::from(sps.constraint_set5_flag)); + std.flags + .set_direct_8x8_inference_flag(u32::from(sps.direct_8x8_inference_flag)); + std.flags + .set_mb_adaptive_frame_field_flag(u32::from(sps.mb_adaptive_frame_field_flag)); + // 1 under the punktfunk envelope (pf-bitstream rejects interlaced SPSes), but the + // conversion itself is faithful, not envelope-coupled. + std.flags + .set_frame_mbs_only_flag(u32::from(sps.frame_mbs_only_flag)); + std.flags + .set_delta_pic_order_always_zero_flag(u32::from(sps.delta_pic_order_always_zero_flag)); + std.flags + .set_separate_colour_plane_flag(u32::from(sps.separate_colour_plane_flag)); + std.flags + .set_gaps_in_frame_num_value_allowed_flag(u32::from( + sps.gaps_in_frame_num_value_allowed_flag, + )); + std.flags + .set_qpprime_y_zero_transform_bypass_flag(u32::from( + sps.qpprime_y_zero_transform_bypass_flag, + )); + std.flags + .set_frame_cropping_flag(u32::from(sps.frame_cropping_flag)); + std.flags + .set_seq_scaling_matrix_present_flag(u32::from(sps.seq_scaling_matrix_present_flag)); + // vui_parameters_present_flag stays 0: decode sessions consume no VUI (module docs). + + std.profile_idc = profile_idc; + std.level_idc = level_to_std(sps.level_idc); + // Chroma format code points equal the chroma_format_idc values (0..3). + std.chroma_format_idc = u32::from(sps.chroma_format_idc); + std.seq_parameter_set_id = sps.seq_parameter_set_id; + std.bit_depth_luma_minus8 = sps.bit_depth_luma_minus8; + std.bit_depth_chroma_minus8 = sps.bit_depth_chroma_minus8; + std.log2_max_frame_num_minus4 = sps.log2_max_frame_num_minus4; + // POC type code points equal the pic_order_cnt_type values (0..2). + std.pic_order_cnt_type = u32::from(sps.pic_order_cnt_type); + std.offset_for_non_ref_pic = sps.offset_for_non_ref_pic; + std.offset_for_top_to_bottom_field = sps.offset_for_top_to_bottom_field; + std.log2_max_pic_order_cnt_lsb_minus4 = sps.log2_max_pic_order_cnt_lsb_minus4; + std.max_num_ref_frames = sps.max_num_ref_frames; + std.pic_width_in_mbs_minus1 = u32::from(sps.pic_width_in_mbs_minus1); + std.pic_height_in_map_units_minus1 = u32::from(sps.pic_height_in_map_units_minus1); + std.frame_crop_left_offset = sps.frame_crop_left_offset; + std.frame_crop_right_offset = sps.frame_crop_right_offset; + std.frame_crop_top_offset = sps.frame_crop_top_offset; + std.frame_crop_bottom_offset = sps.frame_crop_bottom_offset; + + // POC type 1's offset array: exactly num_ref_frames_in_pic_order_cnt_cycle + // entries, boxed so the pointer survives moves of the wrapper. The COUNT field + // and the pointer derive from this one condition so they can never disagree — a + // stale cycle count on a type-0/2 SPS must not become a nonzero count over a + // null array (both stay zeroed instead). + let offset_backing = + (sps.pic_order_cnt_type == 1 && sps.num_ref_frames_in_pic_order_cnt_cycle > 0).then(|| { + let cycle = usize::from(sps.num_ref_frames_in_pic_order_cnt_cycle); + Box::<[i32]>::from(&sps.offset_for_ref_frame[..cycle]) + }); + if let Some(backing) = &offset_backing { + std.num_ref_frames_in_pic_order_cnt_cycle = sps.num_ref_frames_in_pic_order_cnt_cycle; + std.pOffsetForRefFrame = backing.as_ptr(); + } + + let scaling_backing = sps.seq_scaling_matrix_present_flag.then(|| { + let num_8x8 = if sps.chroma_format_idc == 3 { 6 } else { 2 }; + Box::new(scaling_lists_to_std( + &sps.scaling_lists_4x4, + &sps.scaling_lists_8x8, + num_8x8, + )) + }); + if let Some(backing) = &scaling_backing { + std.pScalingLists = &**backing; + } + + Ok(OwnedStdSps { + std, + _offset_backing: offset_backing, + _scaling_backing: scaling_backing, + }) +} + +/// Convert one PPS into the Std struct (owning wrapper), mapping every field the +/// H.264 decode profile consumes. +/// +/// `num_slice_groups_minus1` has no Std field at all; a PPS carrying FMO is rejected +/// rather than converted into a struct that silently claims there is none. +pub fn pps_to_std(pps: &Pps) -> Result { + if pps.num_slice_groups_minus1 != 0 { + return Err(ParamsError::SliceGroups(pps.num_slice_groups_minus1)); + } + if pps.weighted_bipred_idc > 2 { + return Err(ParamsError::InvalidWeightedBipredIdc( + pps.weighted_bipred_idc, + )); + } + + // SAFETY: StdVideoH264PictureParameterSet is a plain-C bindgen struct of + // integers, a bitfield word and one const pointer; all-zero is a valid value for + // every field (null for the pointer) and is the baseline the writes below fill. + let mut std: hh::StdVideoH264PictureParameterSet = unsafe { std::mem::zeroed() }; + + std.flags + .set_transform_8x8_mode_flag(u32::from(pps.transform_8x8_mode_flag)); + std.flags + .set_redundant_pic_cnt_present_flag(u32::from(pps.redundant_pic_cnt_present_flag)); + std.flags + .set_constrained_intra_pred_flag(u32::from(pps.constrained_intra_pred_flag)); + std.flags + .set_deblocking_filter_control_present_flag(u32::from( + pps.deblocking_filter_control_present_flag, + )); + std.flags + .set_weighted_pred_flag(u32::from(pps.weighted_pred_flag)); + std.flags + .set_bottom_field_pic_order_in_frame_present_flag(u32::from( + pps.bottom_field_pic_order_in_frame_present_flag, + )); + std.flags + .set_entropy_coding_mode_flag(u32::from(pps.entropy_coding_mode_flag)); + std.flags + .set_pic_scaling_matrix_present_flag(u32::from(pps.pic_scaling_matrix_present_flag)); + + std.seq_parameter_set_id = pps.seq_parameter_set_id; + std.pic_parameter_set_id = pps.pic_parameter_set_id; + std.num_ref_idx_l0_default_active_minus1 = pps.num_ref_idx_l0_default_active_minus1; + std.num_ref_idx_l1_default_active_minus1 = pps.num_ref_idx_l1_default_active_minus1; + // Code points equal the weighted_bipred_idc values (0..2), validated above. + std.weighted_bipred_idc = u32::from(pps.weighted_bipred_idc); + std.pic_init_qp_minus26 = pps.pic_init_qp_minus26; + std.pic_init_qs_minus26 = pps.pic_init_qs_minus26; + std.chroma_qp_index_offset = pps.chroma_qp_index_offset; + std.second_chroma_qp_index_offset = pps.second_chroma_qp_index_offset; + + let scaling_backing = pps.pic_scaling_matrix_present_flag.then(|| { + // The parser resolves a PPS's 8x8 lists only under transform_8x8_mode_flag + // (7.3.2.2 reads them only then); without it the arrays are untouched zeros + // and must not be declared present. + let num_8x8 = match (pps.transform_8x8_mode_flag, pps.sps.chroma_format_idc == 3) { + (false, _) => 0, + (true, false) => 2, + (true, true) => 6, + }; + Box::new(scaling_lists_to_std( + &pps.scaling_lists_4x4, + &pps.scaling_lists_8x8, + num_8x8, + )) + }); + if let Some(backing) = &scaling_backing { + std.pScalingLists = &**backing; + } + + Ok(OwnedStdPps { + std, + _scaling_backing: scaling_backing, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// An SPS exercising every mapped field with distinct values. The sixteen flags + /// follow the Std bitfield order and strictly ALTERNATE false/true, so a swap of + /// any two adjacent flag mappings fails; `vui_parameters_present_flag` is true at + /// the SOURCE precisely because the conversion must NOT copy it (VUI skip). + fn full_sps() -> Sps { + Sps { + seq_parameter_set_id: 3, + profile_idc: 100, + // Flags, in Std bit order 0..15: F T F T F T F T F T F T F T F (T). + constraint_set1_flag: true, + constraint_set3_flag: true, + constraint_set5_flag: true, + mb_adaptive_frame_field_flag: true, + delta_pic_order_always_zero_flag: true, + gaps_in_frame_num_value_allowed_flag: true, + frame_cropping_flag: true, + vui_parameters_present_flag: true, + // constraint_set0/2/4, direct_8x8_inference, frame_mbs_only, + // separate_colour_plane, qpprime_y_zero_transform_bypass and + // seq_scaling_matrix_present stay false via ..Default. + level_idc: Level::L4_1, + chroma_format_idc: 1, + bit_depth_luma_minus8: 2, + bit_depth_chroma_minus8: 3, + log2_max_frame_num_minus4: 5, + pic_order_cnt_type: 0, + log2_max_pic_order_cnt_lsb_minus4: 6, + offset_for_non_ref_pic: -7, + offset_for_top_to_bottom_field: 3, + max_num_ref_frames: 4, + pic_width_in_mbs_minus1: 119, + pic_height_in_map_units_minus1: 67, + frame_crop_left_offset: 1, + frame_crop_right_offset: 2, + frame_crop_top_offset: 3, + frame_crop_bottom_offset: 4, + ..Default::default() + } + } + + /// A PPS over `sps` exercising every mapped field with distinct values. The + /// eight flags follow the Std bitfield order and strictly ALTERNATE true/false, + /// so a swap of any two adjacent flag mappings fails. + fn full_pps(sps: Sps) -> Pps { + Pps { + pic_parameter_set_id: 5, + seq_parameter_set_id: 3, + // Flags, in Std bit order 0..7: T F T F T F T F. + transform_8x8_mode_flag: true, + redundant_pic_cnt_present_flag: false, + constrained_intra_pred_flag: true, + deblocking_filter_control_present_flag: false, + weighted_pred_flag: true, + bottom_field_pic_order_in_frame_present_flag: false, + entropy_coding_mode_flag: true, + pic_scaling_matrix_present_flag: false, + num_slice_groups_minus1: 0, + num_ref_idx_l0_default_active_minus1: 2, + num_ref_idx_l1_default_active_minus1: 1, + weighted_bipred_idc: 2, + pic_init_qp_minus26: -3, + pic_init_qs_minus26: 4, + chroma_qp_index_offset: -2, + scaling_lists_4x4: [[0; 16]; 6], + scaling_lists_8x8: [[0; 64]; 6], + second_chroma_qp_index_offset: 6, + sps: std::rc::Rc::new(sps), + } + } + + #[test] + fn every_mapped_sps_field_and_flag_round_trips_exactly() { + let sps = full_sps(); + let owned = sps_to_std(&sps).unwrap(); + let std = owned.std(); + + // The strictly alternating pattern of the fixture, bit for bit. + assert_eq!(std.flags.constraint_set0_flag(), 0); + assert_eq!(std.flags.constraint_set1_flag(), 1); + assert_eq!(std.flags.constraint_set2_flag(), 0); + assert_eq!(std.flags.constraint_set3_flag(), 1); + assert_eq!(std.flags.constraint_set4_flag(), 0); + assert_eq!(std.flags.constraint_set5_flag(), 1); + assert_eq!(std.flags.direct_8x8_inference_flag(), 0); + assert_eq!(std.flags.mb_adaptive_frame_field_flag(), 1); + assert_eq!(std.flags.frame_mbs_only_flag(), 0); + assert_eq!(std.flags.delta_pic_order_always_zero_flag(), 1); + assert_eq!(std.flags.separate_colour_plane_flag(), 0); + assert_eq!(std.flags.gaps_in_frame_num_value_allowed_flag(), 1); + assert_eq!(std.flags.qpprime_y_zero_transform_bypass_flag(), 0); + assert_eq!(std.flags.frame_cropping_flag(), 1); + assert_eq!(std.flags.seq_scaling_matrix_present_flag(), 0); + assert_eq!( + std.flags.vui_parameters_present_flag(), + 0, + "true at the source, skipped by design" + ); + + assert_eq!( + std.profile_idc, + hh::StdVideoH264ProfileIdc_STD_VIDEO_H264_PROFILE_IDC_HIGH + ); + assert_eq!( + std.level_idc, + hh::StdVideoH264LevelIdc_STD_VIDEO_H264_LEVEL_IDC_4_1 + ); + assert_eq!( + std.chroma_format_idc, + hh::StdVideoH264ChromaFormatIdc_STD_VIDEO_H264_CHROMA_FORMAT_IDC_420 + ); + assert_eq!(std.seq_parameter_set_id, 3); + assert_eq!(std.bit_depth_luma_minus8, 2); + assert_eq!(std.bit_depth_chroma_minus8, 3, "distinct from luma"); + assert_eq!(std.log2_max_frame_num_minus4, 5); + assert_eq!( + std.pic_order_cnt_type, + hh::StdVideoH264PocType_STD_VIDEO_H264_POC_TYPE_0 + ); + assert_eq!(std.offset_for_non_ref_pic, -7); + assert_eq!(std.offset_for_top_to_bottom_field, 3); + assert_eq!(std.log2_max_pic_order_cnt_lsb_minus4, 6); + assert_eq!(std.num_ref_frames_in_pic_order_cnt_cycle, 0); + assert_eq!(std.max_num_ref_frames, 4); + assert_eq!(std.pic_width_in_mbs_minus1, 119); + assert_eq!(std.pic_height_in_map_units_minus1, 67); + assert_eq!(std.frame_crop_left_offset, 1); + assert_eq!(std.frame_crop_right_offset, 2); + assert_eq!(std.frame_crop_top_offset, 3); + assert_eq!(std.frame_crop_bottom_offset, 4); + + assert!( + std.pOffsetForRefFrame.is_null(), + "POC type 0 carries no offset array" + ); + assert!(std.pScalingLists.is_null()); + assert!(std.pSequenceParameterSetVui.is_null()); + } + + #[test] + fn poc_type_1_offsets_are_owned_and_survive_moving_the_wrapper() { + let mut sps = full_sps(); + sps.pic_order_cnt_type = 1; + sps.num_ref_frames_in_pic_order_cnt_cycle = 3; + sps.offset_for_ref_frame[0] = 2; + sps.offset_for_ref_frame[1] = -1; + sps.offset_for_ref_frame[2] = 4; + + // Box the wrapper AFTER conversion: a move that relocates the wrapper itself + // must not invalidate the pointer, because the backing is heap-pinned. + let owned = Box::new(sps_to_std(&sps).unwrap()); + let std = owned.std(); + assert_eq!( + std.pic_order_cnt_type, + hh::StdVideoH264PocType_STD_VIDEO_H264_POC_TYPE_1 + ); + assert_eq!(std.num_ref_frames_in_pic_order_cnt_cycle, 3); + assert!(!std.pOffsetForRefFrame.is_null()); + // SAFETY: pOffsetForRefFrame points into `owned`'s boxed backing of exactly + // num_ref_frames_in_pic_order_cnt_cycle i32s, alive for this whole scope. + let offsets = unsafe { std::slice::from_raw_parts(std.pOffsetForRefFrame, 3) }; + assert_eq!(offsets, [2, -1, 4]); + } + + #[test] + fn sps_scaling_lists_convert_when_present_and_stay_absent_when_not() { + let mut sps = full_sps(); + assert!(sps_to_std(&sps).unwrap().std().pScalingLists.is_null()); + + sps.seq_scaling_matrix_present_flag = true; + // Each list gets a DISTINCT fill byte: a permutation of lists, or an + // intra/inter reinterleave, cannot pass. + sps.scaling_lists_4x4 = std::array::from_fn(|i| [10 + i as u8; 16]); + sps.scaling_lists_8x8 = std::array::from_fn(|i| [20 + i as u8; 64]); + let owned = sps_to_std(&sps).unwrap(); + let std = owned.std(); + assert_eq!(std.flags.seq_scaling_matrix_present_flag(), 1); + assert!(!std.pScalingLists.is_null()); + // SAFETY: pScalingLists points at `owned`'s boxed StdVideoH264ScalingLists, + // alive for this whole scope. + let lists = unsafe { &*std.pScalingLists }; + // 4:2:0: bits 0-5 (the six 4x4 lists) + bits 6-7 (the two resolved 8x8 + // lists), none deferred to driver-side defaults (the parser already + // resolved them). + assert_eq!(lists.scaling_list_present_mask, 0xFF); + assert_eq!(lists.use_default_scaling_matrix_mask, 0); + for i in 0..6 { + assert_eq!(lists.ScalingList4x4[i], [10 + i as u8; 16], "4x4 list {i}"); + assert_eq!(lists.ScalingList8x8[i], [20 + i as u8; 64], "8x8 list {i}"); + } + + // 4:4:4 resolves all six 8x8 lists. + sps.chroma_format_idc = 3; + let owned = sps_to_std(&sps).unwrap(); + // SAFETY: as above — the pointer targets `owned`'s boxed backing. + let lists = unsafe { &*owned.std().pScalingLists }; + assert_eq!(lists.scaling_list_present_mask, 0xFFF); + } + + #[test] + fn every_mapped_pps_field_and_flag_round_trips_exactly() { + let pps = full_pps(full_sps()); + let owned = pps_to_std(&pps).unwrap(); + let std = owned.std(); + + // The strictly alternating pattern of the fixture, bit for bit. + assert_eq!(std.flags.transform_8x8_mode_flag(), 1); + assert_eq!(std.flags.redundant_pic_cnt_present_flag(), 0); + assert_eq!(std.flags.constrained_intra_pred_flag(), 1); + assert_eq!(std.flags.deblocking_filter_control_present_flag(), 0); + assert_eq!(std.flags.weighted_pred_flag(), 1); + assert_eq!(std.flags.bottom_field_pic_order_in_frame_present_flag(), 0); + assert_eq!(std.flags.entropy_coding_mode_flag(), 1); + assert_eq!(std.flags.pic_scaling_matrix_present_flag(), 0); + + assert_eq!(std.seq_parameter_set_id, 3); + assert_eq!(std.pic_parameter_set_id, 5); + assert_eq!(std.num_ref_idx_l0_default_active_minus1, 2); + assert_eq!(std.num_ref_idx_l1_default_active_minus1, 1); + assert_eq!( + std.weighted_bipred_idc, + hh::StdVideoH264WeightedBipredIdc_STD_VIDEO_H264_WEIGHTED_BIPRED_IDC_IMPLICIT + ); + assert_eq!(std.pic_init_qp_minus26, -3); + assert_eq!(std.pic_init_qs_minus26, 4); + assert_eq!(std.chroma_qp_index_offset, -2); + assert_eq!(std.second_chroma_qp_index_offset, 6); + assert!(std.pScalingLists.is_null()); + } + + #[test] + fn pps_scaling_lists_declare_8x8_present_only_under_transform_8x8_mode() { + let mut pps = full_pps(full_sps()); + pps.pic_scaling_matrix_present_flag = true; + // Distinct fill bytes per list, as in the SPS test. + pps.scaling_lists_4x4 = std::array::from_fn(|i| [30 + i as u8; 16]); + pps.scaling_lists_8x8 = std::array::from_fn(|i| [40 + i as u8; 64]); + + let owned = pps_to_std(&pps).unwrap(); + // SAFETY: pScalingLists points at `owned`'s boxed backing, alive here. + let lists = unsafe { &*owned.std().pScalingLists }; + assert_eq!(lists.scaling_list_present_mask, 0xFF); + assert_eq!(lists.use_default_scaling_matrix_mask, 0); + for i in 0..6 { + assert_eq!(lists.ScalingList4x4[i], [30 + i as u8; 16], "4x4 list {i}"); + assert_eq!(lists.ScalingList8x8[i], [40 + i as u8; 64], "8x8 list {i}"); + } + + // Without transform_8x8_mode the parser never resolved the 8x8 arrays: only + // the six 4x4 lists may be declared present. + pps.transform_8x8_mode_flag = false; + let owned = pps_to_std(&pps).unwrap(); + // SAFETY: as above — the pointer targets `owned`'s boxed backing. + let lists = unsafe { &*owned.std().pScalingLists }; + assert_eq!(lists.scaling_list_present_mask, 0x3F); + assert_eq!(lists.use_default_scaling_matrix_mask, 0); + } + + #[test] + fn a_stale_cycle_count_on_a_type_0_sps_converts_to_zero_offsets() { + let mut sps = full_sps(); + sps.pic_order_cnt_type = 0; + // A stale/corrupt count with no POC-type-1 semantics behind it: the Std + // struct must not claim a cycle over a null array. + sps.num_ref_frames_in_pic_order_cnt_cycle = 5; + let owned = sps_to_std(&sps).unwrap(); + assert_eq!( + owned.std().num_ref_frames_in_pic_order_cnt_cycle, + 0, + "count and pointer derive from one condition" + ); + assert!(owned.std().pOffsetForRefFrame.is_null()); + } + + #[test] + fn the_25fps_vectors_own_parameter_sets_convert_cleanly() { + use std::io::Cursor; + + use cros_codecs::codec::h264::parser::Nalu; + use cros_codecs::codec::h264::parser::NaluType; + use cros_codecs::codec::h264::parser::Parser; + + // The same vendored vector pf-bitstream's tests plan, same relative path. + const TEST_25FPS: &[u8] = include_bytes!( + "../../pf-bitstream/vendor/cros-codecs/src/codec/h264/test_data/test-25fps.h264" + ); + + let mut cursor = Cursor::new(TEST_25FPS); + let mut parser = Parser::default(); + let (mut sps_seen, mut pps_seen) = (false, false); + while let Ok(nalu) = Nalu::next(&mut cursor) { + match nalu.header.type_ { + NaluType::Sps if !sps_seen => { + let sps = parser.parse_sps(&nalu).expect("the vector's SPS parses"); + let owned = sps_to_std(sps).expect("the vector's SPS converts"); + let std = owned.std(); + // The vector's own goldens: 320x240 progressive Main profile. + assert_eq!( + std.profile_idc, + hh::StdVideoH264ProfileIdc_STD_VIDEO_H264_PROFILE_IDC_MAIN + ); + assert_eq!((std.pic_width_in_mbs_minus1 + 1) * 16, 320); + assert_eq!((std.pic_height_in_map_units_minus1 + 1) * 16, 240); + assert_eq!(std.flags.frame_mbs_only_flag(), 1); + assert_eq!(std.flags.vui_parameters_present_flag(), 0); + sps_seen = true; + } + NaluType::Pps if !pps_seen => { + let pps = parser.parse_pps(&nalu).expect("the vector's PPS parses"); + let owned = pps_to_std(pps).expect("the vector's PPS converts"); + assert_eq!(owned.std().pic_parameter_set_id, 0); + pps_seen = true; + } + _ => {} + } + if sps_seen && pps_seen { + break; + } + } + assert!(sps_seen && pps_seen, "the vector opens with SPS + PPS"); + } + + #[test] + fn unrepresentable_parameter_sets_are_rejected_not_approximated() { + let mut sps = full_sps(); + sps.profile_idc = 110; // High10: no StdVideoH264ProfileIdc code point. + assert_eq!( + sps_to_std(&sps).unwrap_err(), + ParamsError::UnmappableProfileIdc(110) + ); + + let mut pps = full_pps(full_sps()); + pps.num_slice_groups_minus1 = 1; + assert_eq!(pps_to_std(&pps).unwrap_err(), ParamsError::SliceGroups(1)); + + let mut pps = full_pps(full_sps()); + pps.weighted_bipred_idc = 3; + assert_eq!( + pps_to_std(&pps).unwrap_err(), + ParamsError::InvalidWeightedBipredIdc(3) + ); + } +} diff --git a/crates/pf-vkdecode/src/pic.rs b/crates/pf-vkdecode/src/pic.rs new file mode 100644 index 00000000..5e088513 --- /dev/null +++ b/crates/pf-vkdecode/src/pic.rs @@ -0,0 +1,760 @@ +//! Per-AU conversion: one [`AuPlan`] into the `StdVideoDecodeH264*` structs, slice +//! offsets and DPB slot bindings a `vkCmdDecodeVideoKHR` call is built from (WP-B). +//! +//! Progressive envelope: pf-bitstream's planner rejects interlaced streams before a +//! plan exists, so every field/bottom FLAG here is written 0. The top/bottom +//! PicOrderCnt pairs are still real pairs — a progressive frame's bottom count +//! differs from its top whenever the PPS carries +//! `bottom_field_pic_order_in_frame_present_flag` — and ride through from +//! pf-bitstream verbatim. + +use ash::vk::native as hh; +use pf_bitstream::h264::AuPlan; +use pf_bitstream::h264::PicId; +use pf_bitstream::h264::RefPic; +use tracing::trace; + +use crate::slots::SlotError; +use crate::slots::SlotMap; + +/// One active reference of the AU: its DPB slot, its Std reference info, and the +/// planner id it resolves (kept so the backend can map the slot to its image). +#[derive(Debug, Clone)] +pub struct VkRef { + pub slot: u8, + pub std: hh::StdVideoDecodeH264ReferenceInfo, + pub id: PicId, +} + +/// Everything CPU-derivable of one AU's decode submission. WP-B adds the live +/// objects: bitstream buffer, DPB images, session and command recording. +#[derive(Debug, Clone)] +pub struct DecodePlanVk { + pub std_pic: hh::StdVideoDecodeH264PictureInfo, + /// Byte offset of each slice NALU in the submitted AU, START CODE INCLUDED — + /// Vulkan's `pSliceOffsets` points at start codes within the bitstream buffer, + /// and punktfunk submits the AU exactly as planned. + pub slice_offsets: Vec, + /// The slot the decoded picture activates (`pSetupReferenceSlot`). + pub setup_slot: u8, + /// Reference info for the setup slot: the picture's own FrameNum/POC, with the + /// long-term flag already set when this very AU marks itself long-term (IDR + /// `long_term_reference_flag` or MMCO 6). + pub setup_ref: hh::StdVideoDecodeH264ReferenceInfo, + /// The planner id of the decoded picture (`AuPlan.dpb.stored`) — the backend + /// keys its image bookkeeping by it. + pub setup_id: PicId, + /// Whether the decoded picture is a reference. When `false` the setup slot + /// exists for the decode itself (plus any remaining DPB residency the planner + /// grants the picture) and must never be bound as a reference for later AUs — + /// it may even have been released already, via this very AU's `removed`, when + /// the picture bypassed the DPB. + pub setup_is_reference: bool, + /// The unique referenced pictures across all slices, in first-appearance order. + pub refs: Vec, +} + +/// Conversion failures. Stream damage never lands here — pf-bitstream degrades it to +/// [`pf_bitstream::h264::PlanWarning`]s upstream; these are caller/session bugs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PlanToVkError { + /// The plan holds no slices; there is nothing to submit. + NoSlices, + /// The plan's `DpbUpdate.stored` is `None`. `plan_au` always stores; only + /// `flush()` produces such updates, and those go to [`SlotMap::apply`] directly. + NoStoredId, + /// A reference list entry's id holds no slot: an earlier plan of this stream + /// never went through this [`SlotMap`]. + UnresolvedReference(PicId), + Slot(SlotError), + /// A slice offset exceeds `u32` (Vulkan submits offsets as `u32`). + OffsetOverflow(usize), + /// The map was built for a different DPB depth than this plan's + /// `max_dpb_frames` — an SPS renegotiation resized the DPB. The session (WP-C) + /// must rebuild the video session and its [`SlotMap`]; converting against the + /// stale map would hand out slot indices the session's image pool does not have. + CapacityMismatch { + required: usize, + capacity: usize, + }, +} + +impl std::fmt::Display for PlanToVkError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PlanToVkError::NoSlices => write!(f, "the plan holds no slices"), + PlanToVkError::NoStoredId => { + write!( + f, + "the plan stores no picture (flush updates go to SlotMap::apply)" + ) + } + PlanToVkError::UnresolvedReference(id) => { + write!(f, "referenced picture {id} holds no DPB slot in this map") + } + PlanToVkError::Slot(err) => write!(f, "slot assignment failed: {err}"), + PlanToVkError::OffsetOverflow(offset) => { + write!(f, "slice offset {offset} exceeds u32") + } + PlanToVkError::CapacityMismatch { required, capacity } => { + write!( + f, + "the plan needs {required} slots but the map holds {capacity} — \ + an SPS renegotiation resized the DPB; rebuild session and map" + ) + } + } + } +} + +impl std::error::Error for PlanToVkError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + PlanToVkError::Slot(err) => Some(err), + _ => None, + } + } +} + +impl From for PlanToVkError { + fn from(err: SlotError) -> Self { + PlanToVkError::Slot(err) + } +} + +/// One [`RefPic`] as Std reference info. +fn ref_info(rp: &RefPic) -> hh::StdVideoDecodeH264ReferenceInfo { + // SAFETY: StdVideoDecodeH264ReferenceInfo is a plain-C bindgen struct of a + // bitfield word and integers; all-zero is a valid value for every field. + let mut std: hh::StdVideoDecodeH264ReferenceInfo = unsafe { std::mem::zeroed() }; + std.flags + .set_used_for_long_term_reference(u32::from(rp.is_long_term)); + // top/bottom_field_flag stay 0 and is_non_existing stays 0: progressive envelope, + // and pf-bitstream never emits a gap placeholder as an id (it substitutes and + // warns instead). + // + // FrameNum carries exactly the pair-key the Std struct wants: frame_num for + // short-term references, LongTermFrameIdx for long-term ones. + std.FrameNum = rp.frame_num_or_lt_idx; + // The stored picture's real 8.2.1 pair: top != bottom whenever the PPS carried + // bottom_field_pic_order_in_frame_present_flag and the slice a nonzero + // delta_pic_order_cnt_bottom — even for progressive frames. + std.PicOrderCnt = [rp.top_field_order_cnt, rp.bottom_field_order_cnt]; + std +} + +/// Convert one planned AU, driving `slots` through the AU's slot lifecycle. +/// +/// `sps_id` is the id of the active SPS: an [`AuPlan`] names the active PPS (each +/// slice header carries `pic_parameter_set_id`) but not the SPS that PPS references — +/// the caller resolves it through its parameter-set table (in WP-B, +/// [`crate::OwnedStdPps`]'s `seq_parameter_set_id` keyed by the first slice's PPS id). +/// +/// Atomicity contract: every fallible step runs before any mutation of `slots`, so +/// an error leaves the map exactly as it was. In order: +/// 1. capacity is validated against the plan's `max_dpb_frames` (read-only); +/// 2. references resolve against the PRE-removal state (read-only) — this AU's own +/// end-of-picture marking (8.2.5) can evict a picture its slices legitimately +/// reference, e.g. the sliding window dropping the oldest short-term reference, +/// so `removed` must not be applied before the lists are mapped; +/// 3. slice offsets are validated (read-only); +/// 4. `removed` is applied — removals were real regardless of this AU's fate — and +/// the setup slot is assigned last (its failures are caller bugs; nothing is ever +/// half-applied). Released slots become assignable to later pictures; keeping the +/// underlying images alive until in-flight decodes complete is WP-B's +/// synchronization, not this map's. +pub fn plan_to_vk( + plan: &AuPlan, + slots: &mut SlotMap, + sps_id: u8, +) -> Result { + let first_slice = plan.slices.first().ok_or(PlanToVkError::NoSlices)?; + let setup_id = plan.dpb.stored.ok_or(PlanToVkError::NoStoredId)?; + + // The map must match THIS plan's DPB depth; a mismatch means an SPS + // renegotiation resized the DPB and the session must be rebuilt (WP-C). + let required = plan.picture.max_dpb_frames + 1; + if slots.capacity() != required { + return Err(PlanToVkError::CapacityMismatch { + required, + capacity: slots.capacity(), + }); + } + + // Unique referenced pictures across every slice's two lists, first appearance + // first. Per-slice list ORDER (ref_idx mapping) lives in the SlicePlans; this Vec + // is the AU-level slot binding set Vulkan wants (each slot listed once). + let mut refs: Vec = Vec::new(); + for slice in &plan.slices { + for rp in slice.ref_list0.iter().chain(&slice.ref_list1) { + if refs.iter().any(|existing| existing.id == rp.id) { + continue; + } + let slot = slots + .slot_of(rp.id) + .ok_or(PlanToVkError::UnresolvedReference(rp.id))?; + refs.push(VkRef { + slot, + std: ref_info(rp), + id: rp.id, + }); + } + } + + let pic = &plan.picture; + + // SAFETY: StdVideoDecodeH264PictureInfo is a plain-C bindgen struct of a bitfield + // word and integers; all-zero is a valid value for every field. + let mut std_pic: hh::StdVideoDecodeH264PictureInfo = unsafe { std::mem::zeroed() }; + let is_intra = plan + .slices + .iter() + .all(|slice| slice.header.slice_type.is_i() || slice.header.slice_type.is_si()); + std_pic.flags.set_is_intra(u32::from(is_intra)); + std_pic.flags.set_is_reference(u32::from(pic.is_reference)); + std_pic.flags.set_IdrPicFlag(u32::from(pic.is_idr)); + // field_pic_flag / bottom_field_flag / complementary_field_pair stay 0 by the + // progressive envelope (module docs). + std_pic.seq_parameter_set_id = sps_id; + std_pic.pic_parameter_set_id = first_slice.header.pic_parameter_set_id; + std_pic.frame_num = pic.frame_num; + std_pic.idr_pic_id = if pic.is_idr { + first_slice.header.idr_pic_id + } else { + 0 + }; + std_pic.PicOrderCnt = [pic.top_field_order_cnt, pic.bottom_field_order_cnt]; + + // The setup slot's reference info: this picture's own identity. When the AU marks + // ITSELF long-term — an IDR's long_term_reference_flag, or an MMCO 6 assigning an + // index (8.2.5) — the slot activates as a long-term reference keyed by + // LongTermFrameIdx, mirroring how ref_info keys long-term entries. + // + // ACTIVATION-vs-REFERENCE asymmetry under MMCO 5 (spec-legal, deliberately NOT + // rejected): these are the picture's 8.2.1 values as decoded, but an MMCO 5 in + // this same AU rebases the STORED frame_num/POC to zero after decoding + // (8.2.5.4.5), so later AUs reference this slot by the rebased pair (RefPic + // carries the stored values). punktfunk hosts never emit MMCO 5; + // pf_bitstream::h264::PlanWarning::Mmco5Rebase flags any occurrence so field + // logs tell us if that assumption ever breaks. + // SAFETY: as above — all-zero is a valid StdVideoDecodeH264ReferenceInfo. + let mut setup_ref: hh::StdVideoDecodeH264ReferenceInfo = unsafe { std::mem::zeroed() }; + setup_ref.PicOrderCnt = [pic.top_field_order_cnt, pic.bottom_field_order_cnt]; + let marking = &first_slice.header.dec_ref_pic_marking; + let self_lt_idx = if pic.is_idr { + marking.long_term_reference_flag.then_some(0u32) + } else if marking.adaptive_ref_pic_marking_mode_flag { + marking + .inner + .iter() + .find(|op| op.memory_management_control_operation == 6) + .map(|op| op.long_term_frame_idx) + } else { + None + }; + match self_lt_idx { + Some(idx) => { + setup_ref.flags.set_used_for_long_term_reference(1); + // Same saturation as pf-bitstream's frame_num_or_lt_idx: the spec bounds + // the ue(v)-coded index at 15, the parser does not. + setup_ref.FrameNum = u16::try_from(idx).unwrap_or(u16::MAX); + } + None => setup_ref.FrameNum = pic.frame_num, + } + + let mut slice_offsets = Vec::with_capacity(plan.slices.len()); + for slice in &plan.slices { + // SlicePlan.data starts at the slice NALU's start code — exactly the offset + // Vulkan wants (struct docs). + slice_offsets.push( + u32::try_from(slice.data.start) + .map_err(|_| PlanToVkError::OffsetOverflow(slice.data.start))?, + ); + } + + // Mutations LAST, after every fallible step above (fn docs). Removals first — + // they were real regardless of this AU's fate — then the setup assignment. + // + // The AU's own picture can itself appear in `removed`: a non-reference picture + // with no free frame buffer bypasses the DPB and is stored-and-evicted within + // one plan. Its slot must still exist for the decode itself, so it is assigned + // here and released right after — see `DecodePlanVk::setup_is_reference`. + let setup_evicted = plan.dpb.removed.contains(&setup_id); + for &id in &plan.dpb.removed { + if id == setup_id { + continue; + } + if !slots.release(id) { + // Tolerated but never silent: reachable only when the caller skipped + // feeding an AU's plan through this map. + trace!(id, "DpbUpdate removed an id this SlotMap never assigned"); + } + } + let setup_slot = slots.assign(setup_id)?; + if setup_evicted { + slots.release(setup_id); + } + + Ok(DecodePlanVk { + std_pic, + slice_offsets, + setup_slot, + setup_ref, + setup_id, + setup_is_reference: pic.is_reference, + refs, + }) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::io::Cursor; + use std::rc::Rc; + + use cros_codecs::codec::h264::nalu_writer::NaluWriter; + use cros_codecs::codec::h264::parser::Nalu; + use cros_codecs::codec::h264::parser::NaluType; + use cros_codecs::codec::h264::parser::Pps; + use cros_codecs::codec::h264::parser::PpsBuilder; + use cros_codecs::codec::h264::parser::Profile; + use cros_codecs::codec::h264::parser::Sps; + use cros_codecs::codec::h264::parser::SpsBuilder; + use cros_codecs::codec::h264::synthesizer::Synthesizer; + use pf_bitstream::h264::H264Planner; + use pf_bitstream::h264::Level; + + use super::*; + + // The same vendored vector pf-bitstream's tests plan (its goldens: 250 AUs, 500 + // slices), included from the same path rather than copied. + const TEST_25FPS: &[u8] = include_bytes!( + "../../pf-bitstream/vendor/cros-codecs/src/codec/h264/test_data/test-25fps.h264" + ); + + /// Test-only AU splitter, mirroring pf-bitstream's `split_into_aus` helper (which + /// is `#[cfg(test)]`-private there): a new AU starts at a non-slice NALU following + /// slices, or at a slice with `first_mb_in_slice == 0` (whose ue(v) encoding makes + /// the first RBSP bit 1) when the current AU already has slices. + fn split_into_aus(stream: &[u8]) -> Vec<&[u8]> { + let mut aus = Vec::new(); + let mut cursor = Cursor::new(stream); + let mut au_start = 0usize; + let mut au_has_slice = false; + + while let Ok(nalu) = Nalu::next(&mut cursor) { + let nalu_offset = cursor.position() as usize; + let start = nalu_offset - nalu.offset; + let is_slice = matches!(nalu.header.type_, NaluType::Slice | NaluType::SliceIdr); + let first_mb_zero = + is_slice && stream.get(nalu_offset + 1).is_some_and(|b| b & 0x80 != 0); + + if au_has_slice && (!is_slice || first_mb_zero) { + aus.push(&stream[au_start..start]); + au_start = start; + au_has_slice = false; + } + au_has_slice |= is_slice; + } + aus.push(&stream[au_start..]); + aus + } + + #[test] + fn the_full_25fps_vector_converts_with_stable_slots_and_start_code_offsets() { + let aus = split_into_aus(TEST_25FPS); + let mut planner = H264Planner::new(); + let mut slots: Option = None; + // PicId -> the slot it was assigned; entries leave only on `removed`. + let mut held: BTreeMap = BTreeMap::new(); + let mut converted = 0usize; + + for au in &aus { + let plan = planner.plan_au(au).expect("the clean vector plans"); + let slots = slots.get_or_insert_with(|| SlotMap::new(plan.picture.max_dpb_frames)); + let vk = plan_to_vk(&plan, slots, 0).expect("the clean vector converts"); + converted += 1; + + // Slot stability: every reference resolves to the slot its picture was + // assigned when IT was decoded, and no ref shares the setup slot. + for r in &vk.refs { + assert_eq!( + held.get(&r.id), + Some(&r.slot), + "a referenced picture's slot changed while it was referenced" + ); + assert_ne!(r.slot, vk.setup_slot, "a reference aliases the setup slot"); + } + + // Slice offsets: one per slice, each at a start-code boundary of the AU, + // and exactly where the plan said the slice begins. + assert_eq!(vk.slice_offsets.len(), plan.slices.len()); + for (offset, slice) in vk.slice_offsets.iter().zip(&plan.slices) { + let offset = *offset as usize; + assert_eq!(offset, slice.data.start); + let at = &au[offset..]; + assert!( + at.starts_with(&[0, 0, 1]) || at.starts_with(&[0, 0, 0, 1]), + "slice offset {offset} does not sit on a start code" + ); + } + + assert_eq!(vk.std_pic.frame_num, plan.picture.frame_num); + assert_eq!( + u32::from(plan.picture.is_idr), + vk.std_pic.flags.IdrPicFlag() + ); + assert_eq!( + vk.setup_ref.PicOrderCnt[0], + plan.picture.top_field_order_cnt + ); + + // Mirror the map's bookkeeping: record the new picture, drop the removed. + let stored = plan.dpb.stored.unwrap(); + assert_eq!(vk.setup_id, stored); + assert_eq!(vk.setup_is_reference, plan.picture.is_reference); + held.insert(stored, vk.setup_slot); + for id in &plan.dpb.removed { + held.remove(id); + } + + // held() must mirror the plan-driven bookkeeping exactly, every AU. + let ledger: BTreeMap = slots.held().map(|(slot, id)| (id, slot)).collect(); + assert_eq!(ledger, held); + } + + assert_eq!(converted, 250, "the vector's own golden"); + + // Teardown: the flush update releases every remaining slot. + let mut slots = slots.unwrap(); + slots.apply(&planner.flush()); + assert_eq!(slots.active(), 0); + } + + /// Byte-level authoring, mirroring pf-bitstream's MMCO/LTR test (its helpers are + /// `#[cfg(test)]`-private): parameter sets via the vendored builders + + /// synthesizer, slice headers hand-written with the vendored `NaluWriter`. The + /// planner only reads headers, so no slice data follows the rbsp stop bit. + fn authored_sps_pps() -> (Rc, Rc) { + let sps = SpsBuilder::new() + .seq_parameter_set_id(0) + .profile_idc(Profile::Main) + .level_idc(Level::L4) + .frame_mbs_only_flag(true) + .direct_8x8_inference_flag(true) + .max_num_ref_frames(4) + .log2_max_frame_num_minus4(0) + .pic_order_cnt_type(0) + .log2_max_pic_order_cnt_lsb_minus4(0) + .resolution(64, 64) + .build(); + let pps = PpsBuilder::new(Rc::clone(&sps)) + .pic_parameter_set_id(0) + .pic_init_qp(26) + .build(); + (sps, pps) + } + + /// `bottom_delta` writes `delta_pic_order_cnt_bottom` — only legal when the PPS + /// the slice references sets `bottom_field_pic_order_in_frame_present_flag` + /// (the parser reads the field iff the flag is set, so writer and PPS must + /// agree). + fn write_idr_slice(bottom_delta: Option) -> Vec { + let mut buf = Vec::new(); + { + let mut w = NaluWriter::new(&mut buf, true); + w.write_header(3, NaluType::SliceIdr as u8).unwrap(); + w.write_ue(0u32).unwrap(); // first_mb_in_slice + w.write_ue(2u32).unwrap(); // slice_type: I + w.write_ue(0u32).unwrap(); // pic_parameter_set_id + w.write_f(4, 0u32).unwrap(); // frame_num, u(4): log2_max_frame_num_minus4 = 0 + w.write_ue(7u32).unwrap(); // idr_pic_id + w.write_f(4, 0u32).unwrap(); // pic_order_cnt_lsb, u(4) + if let Some(delta) = bottom_delta { + w.write_se(delta).unwrap(); // delta_pic_order_cnt_bottom + } + w.write_f(1, 0u32).unwrap(); // no_output_of_prior_pics_flag + w.write_f(1, 0u32).unwrap(); // long_term_reference_flag + w.write_se(0i32).unwrap(); // slice_qp_delta + w.write_f(1, 1u32).unwrap(); // rbsp stop bit + while !w.aligned() { + w.write_f(1, 0u32).unwrap(); + } + } + buf + } + + /// One P slice NALU. `mmco_ops` = `None` for sliding-window marking, `Some(ops)` + /// for adaptive marking with `(operation, single-argument)` pairs (ops 2/4/6 all + /// take exactly one) — the writer appends the terminating op 0. `bottom_delta` + /// as in [`write_idr_slice`]. + fn write_p_slice( + frame_num: u32, + poc_lsb: u32, + bottom_delta: Option, + num_ref_idx_l0_active: u32, + mmco_ops: Option<&[(u32, u32)]>, + ) -> Vec { + let mut buf = Vec::new(); + { + let mut w = NaluWriter::new(&mut buf, true); + w.write_header(1, NaluType::Slice as u8).unwrap(); + w.write_ue(0u32).unwrap(); // first_mb_in_slice + w.write_ue(0u32).unwrap(); // slice_type: P + w.write_ue(0u32).unwrap(); // pic_parameter_set_id + w.write_f(4, frame_num).unwrap(); // frame_num, u(4) + w.write_f(4, poc_lsb).unwrap(); // pic_order_cnt_lsb, u(4) + if let Some(delta) = bottom_delta { + w.write_se(delta).unwrap(); // delta_pic_order_cnt_bottom + } + w.write_f(1, 1u32).unwrap(); // num_ref_idx_active_override_flag + w.write_ue(num_ref_idx_l0_active - 1).unwrap(); + w.write_f(1, 0u32).unwrap(); // ref_pic_list_modification_flag_l0 + match mmco_ops { + None => w.write_f(1, 0u32).map(|_| ()).unwrap(), + Some(ops) => { + w.write_f(1, 1u32).unwrap(); // adaptive_ref_pic_marking_mode_flag + for (op, arg) in ops { + w.write_ue(*op).unwrap(); + w.write_ue(*arg).unwrap(); + } + w.write_ue(0u32).unwrap(); // memory_management_control_operation end + } + } + w.write_se(0i32).unwrap(); // slice_qp_delta + w.write_f(1, 1u32).unwrap(); // rbsp stop bit + while !w.aligned() { + w.write_f(1, 0u32).unwrap(); + } + } + buf + } + + #[test] + fn an_mmco_self_marking_sets_the_setup_lt_flag_and_later_refs_carry_it() { + let (sps, pps) = authored_sps_pps(); + let mut au0 = Vec::new(); + Synthesizer::<'_, Sps, _>::synthesize(3, &sps, &mut au0, true).unwrap(); + Synthesizer::<'_, Pps, _>::synthesize(3, &pps, &mut au0, true).unwrap(); + au0.extend(write_idr_slice(None)); + // AU1 marks itself long-term: MMCO 4 admits long-term index 0, MMCO 6 + // assigns it to the current picture. + let au1 = write_p_slice(1, 2, None, 1, Some(&[(4, 1), (6, 0)])); + let au2 = write_p_slice(2, 4, None, 2, None); + + let mut planner = H264Planner::new(); + let p0 = planner.plan_au(&au0).unwrap(); + let mut slots = SlotMap::new(p0.picture.max_dpb_frames); + + let idr_id = p0.dpb.stored.unwrap(); + let vk0 = plan_to_vk(&p0, &mut slots, 0).unwrap(); + assert_eq!(vk0.std_pic.flags.IdrPicFlag(), 1); + assert_eq!(vk0.std_pic.flags.is_intra(), 1); + assert_eq!(vk0.std_pic.idr_pic_id, 7, "from the authored slice header"); + assert_eq!(vk0.std_pic.PicOrderCnt, [0, 0]); + assert_eq!(vk0.setup_ref.flags.used_for_long_term_reference(), 0); + assert_eq!(vk0.setup_id, idr_id); + assert!(vk0.setup_is_reference, "an IDR is a reference"); + assert!(vk0.refs.is_empty()); + + // AU1: the setup slot activates LONG-TERM (its own MMCO 6), keyed by + // LongTermFrameIdx 0, not by frame_num 1. + let p1 = planner.plan_au(&au1).unwrap(); + let lt_id = p1.dpb.stored.unwrap(); + let vk1 = plan_to_vk(&p1, &mut slots, 0).unwrap(); + assert_eq!(vk1.std_pic.flags.is_intra(), 0); + assert_eq!(vk1.std_pic.PicOrderCnt, [2, 2]); + assert_eq!(vk1.setup_ref.flags.used_for_long_term_reference(), 1); + assert_eq!(vk1.setup_ref.FrameNum, 0, "LongTermFrameIdx, not frame_num"); + assert_eq!(vk1.refs.len(), 1); + assert_eq!(vk1.refs[0].id, idr_id); + assert_eq!(vk1.refs[0].std.flags.used_for_long_term_reference(), 0); + + // AU2 references both: the IDR short-term (keyed by frame_num) and AU1 + // long-term (keyed by LongTermFrameIdx), each on its stable slot. + let p2 = planner.plan_au(&au2).unwrap(); + let vk2 = plan_to_vk(&p2, &mut slots, 0).unwrap(); + let by_id: BTreeMap = vk2.refs.iter().map(|r| (r.id, r)).collect(); + let idr_ref = by_id[&idr_id]; + assert_eq!(idr_ref.std.flags.used_for_long_term_reference(), 0); + assert_eq!(idr_ref.std.FrameNum, 0); + assert_eq!(idr_ref.slot, vk0.setup_slot); + let lt_ref = by_id[<_id]; + assert_eq!(lt_ref.std.flags.used_for_long_term_reference(), 1); + assert_eq!(lt_ref.std.FrameNum, 0, "LongTermFrameIdx, not frame_num"); + assert_eq!( + lt_ref.std.PicOrderCnt, + [2, 2], + "the stored top/bottom pair (equal here: no delta_pic_order_cnt_bottom)" + ); + assert_eq!(lt_ref.slot, vk1.setup_slot); + assert_ne!(vk2.setup_slot, idr_ref.slot); + assert_ne!(vk2.setup_slot, lt_ref.slot); + } + + #[test] + fn a_failed_conversion_leaves_the_slot_map_untouched_and_the_session_recovers() { + let (sps, pps) = authored_sps_pps(); + let mut au0 = Vec::new(); + Synthesizer::<'_, Sps, _>::synthesize(3, &sps, &mut au0, true).unwrap(); + Synthesizer::<'_, Pps, _>::synthesize(3, &pps, &mut au0, true).unwrap(); + au0.extend(write_idr_slice(None)); + let au1 = write_p_slice(1, 2, None, 1, None); + + let mut planner = H264Planner::new(); + let p0 = planner.plan_au(&au0).unwrap(); + let idr_id = p0.dpb.stored.unwrap(); + let p1 = planner.plan_au(&au1).unwrap(); + + // A right-sized map that never saw AU0, holding one unrelated slot: the + // reference must fail loudly, not resolve to a fabricated slot. + let mut fresh = SlotMap::new(p1.picture.max_dpb_frames); + fresh.assign(999).unwrap(); + assert_eq!( + plan_to_vk(&p1, &mut fresh, 0).unwrap_err(), + PlanToVkError::UnresolvedReference(idr_id) + ); + + // Atomicity: the failed conversion mutated nothing. + assert_eq!(fresh.active(), 1); + assert_eq!(fresh.held().collect::>(), vec![(0, 999)]); + + // And the session recovers: the next valid AU (an IDR restart) still + // converts on the same map. Its `removed` names ids this map never assigned + // (planned before the map existed) — tolerated by design. + let p2 = planner.plan_au(&write_idr_slice(None)).unwrap(); + let vk2 = plan_to_vk(&p2, &mut fresh, 0).unwrap(); + assert_eq!(vk2.setup_slot, 1, "the lowest free slot after the held one"); + assert_eq!(fresh.active(), 2); + } + + #[test] + fn an_sps_switch_that_resizes_the_dpb_is_a_capacity_mismatch_not_a_guess() { + // Two SPSes whose only planning-relevant difference is DPB depth: Level 1 at + // 320x240 gives MaxDpbMbs 396 / 300 = 1 frame; Level 4 gives 16. + let authored = |level: Level, max_refs: u8| -> (Rc, Rc) { + let sps = SpsBuilder::new() + .seq_parameter_set_id(0) + .profile_idc(Profile::Main) + .level_idc(level) + .frame_mbs_only_flag(true) + .direct_8x8_inference_flag(true) + .max_num_ref_frames(max_refs) + .log2_max_frame_num_minus4(0) + .pic_order_cnt_type(0) + .log2_max_pic_order_cnt_lsb_minus4(0) + .resolution(320, 240) + .build(); + let pps = PpsBuilder::new(Rc::clone(&sps)) + .pic_parameter_set_id(0) + .pic_init_qp(26) + .build(); + (sps, pps) + }; + let (sps_a, pps_a) = authored(Level::L1, 1); + let (sps_b, pps_b) = authored(Level::L4, 4); + + let mut au0 = Vec::new(); + Synthesizer::<'_, Sps, _>::synthesize(3, &sps_a, &mut au0, true).unwrap(); + Synthesizer::<'_, Pps, _>::synthesize(3, &pps_a, &mut au0, true).unwrap(); + au0.extend(write_idr_slice(None)); + let mut au1 = Vec::new(); + Synthesizer::<'_, Sps, _>::synthesize(3, &sps_b, &mut au1, true).unwrap(); + Synthesizer::<'_, Pps, _>::synthesize(3, &pps_b, &mut au1, true).unwrap(); + au1.extend(write_idr_slice(None)); + + let mut planner = H264Planner::new(); + let p0 = planner.plan_au(&au0).unwrap(); + assert_eq!(p0.picture.max_dpb_frames, 1); + let mut slots = SlotMap::new(p0.picture.max_dpb_frames); + plan_to_vk(&p0, &mut slots, 0).unwrap(); + + // The renegotiated stream needs a deeper DPB than this map was built for: + // refuse, so the session (WP-C) rebuilds session + map instead of handing + // out slots the image pool does not have. + let p1 = planner.plan_au(&au1).unwrap(); + assert_eq!(p1.picture.max_dpb_frames, 16); + assert_eq!( + plan_to_vk(&p1, &mut slots, 0).unwrap_err(), + PlanToVkError::CapacityMismatch { + required: 17, + capacity: 2 + } + ); + } + + #[test] + fn a_nonzero_delta_bottom_reaches_setup_and_reference_poc_pairs_distinctly() { + // A PPS with bottom_field_pic_order_in_frame_present_flag: progressive + // frames then carry delta_pic_order_cnt_bottom and bottom != top. The + // builder has no setter for the flag, so the Pps is constructed directly + // (its fields are public) and synthesized from there. + let sps = SpsBuilder::new() + .seq_parameter_set_id(0) + .profile_idc(Profile::Main) + .level_idc(Level::L4) + .frame_mbs_only_flag(true) + .direct_8x8_inference_flag(true) + .max_num_ref_frames(4) + .log2_max_frame_num_minus4(0) + .pic_order_cnt_type(0) + .log2_max_pic_order_cnt_lsb_minus4(0) + .resolution(64, 64) + .build(); + let pps = Pps { + pic_parameter_set_id: 0, + seq_parameter_set_id: 0, + entropy_coding_mode_flag: false, + bottom_field_pic_order_in_frame_present_flag: true, + num_slice_groups_minus1: 0, + num_ref_idx_l0_default_active_minus1: 0, + num_ref_idx_l1_default_active_minus1: 0, + weighted_pred_flag: false, + weighted_bipred_idc: 0, + pic_init_qp_minus26: 0, + pic_init_qs_minus26: 0, + chroma_qp_index_offset: 0, + deblocking_filter_control_present_flag: false, + constrained_intra_pred_flag: false, + redundant_pic_cnt_present_flag: false, + transform_8x8_mode_flag: false, + pic_scaling_matrix_present_flag: false, + scaling_lists_4x4: [[0; 16]; 6], + scaling_lists_8x8: [[0; 64]; 6], + second_chroma_qp_index_offset: 0, + sps: Rc::clone(&sps), + }; + + let mut au0 = Vec::new(); + Synthesizer::<'_, Sps, _>::synthesize(3, &sps, &mut au0, true).unwrap(); + Synthesizer::<'_, Pps, _>::synthesize(3, &pps, &mut au0, true).unwrap(); + au0.extend(write_idr_slice(Some(2))); // top 0, bottom 0 + 2 + let au1 = write_p_slice(1, 4, Some(1), 1, None); // top 4, bottom 4 + 1 + + let mut planner = H264Planner::new(); + let p0 = planner.plan_au(&au0).unwrap(); + assert_eq!( + ( + p0.picture.top_field_order_cnt, + p0.picture.bottom_field_order_cnt + ), + (0, 2) + ); + let mut slots = SlotMap::new(p0.picture.max_dpb_frames); + let vk0 = plan_to_vk(&p0, &mut slots, 0).unwrap(); + // BOTH orders, both structs: a top/bottom swap or a collapse to one value + // must fail here. + assert_eq!(vk0.std_pic.PicOrderCnt, [0, 2]); + assert_eq!(vk0.setup_ref.PicOrderCnt, [0, 2]); + + let p1 = planner.plan_au(&au1).unwrap(); + let vk1 = plan_to_vk(&p1, &mut slots, 0).unwrap(); + assert_eq!(vk1.std_pic.PicOrderCnt, [4, 5]); + assert_eq!(vk1.setup_ref.PicOrderCnt, [4, 5]); + // The reference carries the STORED pair of the IDR — through RefPic, not a + // fabricated bottom. + assert_eq!(vk1.refs.len(), 1); + assert_eq!(vk1.refs[0].id, p0.dpb.stored.unwrap()); + assert_eq!(vk1.refs[0].std.PicOrderCnt, [0, 2]); + } +} diff --git a/crates/pf-vkdecode/src/slots.rs b/crates/pf-vkdecode/src/slots.rs new file mode 100644 index 00000000..4e3cc0a9 --- /dev/null +++ b/crates/pf-vkdecode/src/slots.rs @@ -0,0 +1,279 @@ +//! The hardware DPB slot ledger: [`pf_bitstream::h264::PicId`]s mapped to the slot +//! indices a Vulkan Video session binds DPB images by. +//! +//! Division of labour: pf-bitstream's DPB runs the 8.2.5/C.4.5.3 processes and +//! DECIDES which pictures live and die — this map only translates its verdicts into +//! stable slot indices. It therefore never evicts on its own: running out of slots is +//! an error ([`SlotError::Full`]), because it can only mean removals were missed, and +//! a silent eviction would hide that bug behind corrupted output. + +use pf_bitstream::h264::DpbUpdate; +use pf_bitstream::h264::PicId; +use tracing::trace; + +/// The H.264 slot ceiling: 16 reference frames plus the picture being decoded. +const MAX_SLOTS: usize = 17; + +/// What went wrong with a slot operation. Both variants are caller bugs, not stream +/// conditions — pf-bitstream degrades stream damage to warnings long before here. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SlotError { + /// No free slot. The map is sized to `max_dpb_frames + 1`, which the planner's + /// DPB never exceeds; overflow means this map missed `removed` entries. + Full { capacity: usize }, + /// The id already holds a slot; ids are per-picture and never re-assigned. + AlreadyAssigned { id: PicId, slot: u8 }, +} + +impl std::fmt::Display for SlotError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SlotError::Full { capacity } => { + write!( + f, + "all {capacity} DPB slots are held — removals were missed" + ) + } + SlotError::AlreadyAssigned { id, slot } => { + write!(f, "picture {id} already holds slot {slot}") + } + } + } +} + +impl std::error::Error for SlotError {} + +/// The slot ledger. One per decode session; feed it every [`DpbUpdate`] in decode +/// order (via [`Self::apply`] or `plan_to_vk`, which applies internally). +/// +/// Invariants (unit-tested): +/// - a [`PicId`] keeps its slot from [`Self::assign`] until [`Self::release`]; +/// - a slot is reused only after its holder is released; +/// - assigning past capacity errors instead of evicting. +#[derive(Debug, Clone)] +pub struct SlotMap { + /// `slots[i]` holds the id bound to slot `i`, `None` while the slot is free. + slots: Vec>, +} + +impl SlotMap { + /// Sized from [`pf_bitstream::h264::PicturePlan::max_dpb_frames`] plus one for + /// the picture being decoded (its setup slot coexists with a full reference + /// window). + /// + /// pf-bitstream's envelope gate rejects any SPS asking for a DPB deeper than the + /// spec's 16 frames before a plan exists, so a larger request here is a caller + /// bug — debug-asserted, never silently clamped (a clamp would turn the bug into + /// silent evictions later). + pub fn new(max_dpb_frames: usize) -> Self { + debug_assert!( + max_dpb_frames < MAX_SLOTS, + "a {max_dpb_frames}-frame DPB exceeds the H.264 ceiling pf-bitstream's \ + envelope gate enforces" + ); + Self { + slots: vec![None; max_dpb_frames + 1], + } + } + + /// Total slot count (fixed at construction). + pub fn capacity(&self) -> usize { + self.slots.len() + } + + /// Slots currently held. + pub fn active(&self) -> usize { + self.slots.iter().filter(|slot| slot.is_some()).count() + } + + /// The held slots as `(slot, id)` pairs, in slot order — WP-B walks this to + /// build `VkVideoReferenceSlotInfoKHR` bindings and to map slots back to their + /// images. + pub fn held(&self) -> impl Iterator + '_ { + self.slots + .iter() + .enumerate() + // The envelope-gated capacity (<= 17) keeps every index within u8. + .filter_map(|(index, slot)| slot.map(|id| (index as u8, id))) + } + + /// Bind `id` to the lowest free slot. + pub fn assign(&mut self, id: PicId) -> Result { + if let Some(slot) = self.slot_of(id) { + return Err(SlotError::AlreadyAssigned { id, slot }); + } + let free = self + .slots + .iter() + .position(Option::is_none) + .ok_or(SlotError::Full { + capacity: self.slots.len(), + })?; + self.slots[free] = Some(id); + // The envelope-gated capacity (<= 17) keeps every index within u8. + Ok(free as u8) + } + + /// The slot `id` holds, if any. + pub fn slot_of(&self, id: PicId) -> Option { + self.slots + .iter() + .position(|slot| *slot == Some(id)) + // The envelope-gated capacity (<= 17) keeps every index within u8. + .map(|index| index as u8) + } + + /// Free `id`'s slot. Returns whether the id held one. + /// + /// Slot lifetime is DPB RESIDENCY: a picture holds its slot for exactly as long + /// as the planner's DPB holds the picture — as a reference OR as a decoded + /// picture awaiting output — and that residency ends only when a + /// [`DpbUpdate::removed`] entry reports it. This method is that report's + /// primitive: `plan_to_vk` and [`Self::apply`] call it with the planner's + /// `removed` ids and nothing else may release a slot. + /// + /// Releasing is CPU-side bookkeeping (the slot becomes assignable to a later + /// picture); keeping the released slot's IMAGE out of reuse until in-flight + /// decodes complete is the backend's synchronization, not this ledger's. + pub fn release(&mut self, id: PicId) -> bool { + match self.slots.iter().position(|slot| *slot == Some(id)) { + Some(index) => { + self.slots[index] = None; + true + } + None => false, + } + } + + /// Apply one [`DpbUpdate`]: release every `removed` id. + /// + /// `outputs` is deliberately ignored: output-readiness is display sequencing, + /// not the end of DPB residency — a display-ready picture can still be a + /// reference (its slot stays), and only its later `removed` entry frees the + /// slot. + pub fn apply(&mut self, update: &DpbUpdate) { + for &id in &update.removed { + if !self.release(id) { + // Tolerated but never silent: reachable only when the caller skipped + // feeding an AU's plan through this map. + trace!(id, "DpbUpdate removed an id this SlotMap never assigned"); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_pic_id_keeps_its_slot_until_released_and_the_slot_is_then_reusable() { + let mut slots = SlotMap::new(3); // capacity 4 + let s0 = slots.assign(10).unwrap(); + let s1 = slots.assign(11).unwrap(); + assert_ne!(s0, s1); + + // Stable across unrelated churn. + assert_eq!(slots.slot_of(10), Some(s0)); + slots.release(11); + assert_eq!(slots.slot_of(10), Some(s0)); + assert_eq!(slots.slot_of(11), None); + + // The freed slot is reusable; the held one is not. + let s2 = slots.assign(12).unwrap(); + assert_eq!(s2, s1, "the lowest free slot is the released one"); + assert_eq!(slots.slot_of(10), Some(s0)); + assert_eq!(slots.active(), 2); + } + + #[test] + fn assigning_past_capacity_is_an_error_never_a_silent_eviction() { + let mut slots = SlotMap::new(1); // capacity 2 + slots.assign(1).unwrap(); + slots.assign(2).unwrap(); + assert_eq!(slots.assign(3), Err(SlotError::Full { capacity: 2 })); + // The failed assign evicted nothing. + assert_eq!(slots.slot_of(1), Some(0)); + assert_eq!(slots.slot_of(2), Some(1)); + } + + #[test] + fn re_assigning_a_held_id_is_an_error_not_a_move() { + let mut slots = SlotMap::new(2); + let s = slots.assign(7).unwrap(); + assert_eq!( + slots.assign(7), + Err(SlotError::AlreadyAssigned { id: 7, slot: s }) + ); + assert_eq!(slots.active(), 1); + } + + #[test] + fn capacity_is_dpb_frames_plus_one_for_the_setup_slot() { + assert_eq!(SlotMap::new(16).capacity(), 17); + assert_eq!(SlotMap::new(4).capacity(), 5); + } + + #[test] + #[should_panic(expected = "envelope")] + fn a_dpb_past_the_h264_ceiling_is_a_debug_panic_not_a_clamp() { + // pf-bitstream's envelope gate makes this unreachable from a real stream; + // reaching it means a caller bypassed the planner. + let _ = SlotMap::new(17); + } + + #[test] + fn held_lists_slot_id_pairs_in_slot_order() { + let mut slots = SlotMap::new(3); + slots.assign(10).unwrap(); + slots.assign(11).unwrap(); + slots.assign(12).unwrap(); + slots.release(11); + assert_eq!(slots.held().collect::>(), vec![(0, 10), (2, 12)]); + } + + #[test] + fn apply_releases_removed_ids_and_ignores_outputs() { + let mut slots = SlotMap::new(3); + slots.assign(1).unwrap(); + slots.assign(2).unwrap(); + slots.apply(&DpbUpdate { + stored: None, + outputs: vec![1], // display-ready, still a reference: must keep its slot + removed: vec![2], + }); + assert_eq!(slots.slot_of(1), Some(0)); + assert_eq!(slots.slot_of(2), None); + } + + #[test] + fn a_hundred_synthetic_dpb_updates_churn_without_aliasing_a_slot() { + // A sliding window of 4 references over 100 pictures: each id's slot must + // stay fixed while it lives, and no two live ids may ever share a slot. + let mut slots = SlotMap::new(4); + let mut recorded: Vec<(PicId, u8)> = Vec::new(); + for id in 0u64..100 { + let slot = slots.assign(id).unwrap(); + assert!( + recorded.iter().all(|&(_, held)| held != slot), + "assign handed out a slot a live picture still holds" + ); + recorded.push((id, slot)); + + let removed = if id >= 4 { vec![id - 4] } else { Vec::new() }; + slots.apply(&DpbUpdate { + stored: Some(id), + outputs: vec![id], + removed: removed.clone(), + }); + for gone in removed { + recorded.retain(|&(held_id, _)| held_id != gone); + } + // Every live picture still holds exactly the slot it was assigned. + for &(live, slot) in &recorded { + assert_eq!(slots.slot_of(live), Some(slot)); + } + assert!(slots.active() <= 5); + } + } +}