From 8578141d43e5e38b167b20377423459ddc706a6b Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Sat, 25 Jul 2026 15:11:20 +0200 Subject: [PATCH] fix(encode/vulkan): AV1 at unaligned modes was violating two VUIDs on every frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by running the smokes at 1920x1080 under the validation layers while chasing the three items left over from WP4.2. AV1 forbids the encode source's `codedExtent` differing from the sequence header without `FRAME_SIZE_OVERRIDE` (VUID-vkCmdEncodeVideoKHR-flags-10324), or from the reference slots without `MOTION_VECTOR_SCALING` (`-10325`). RADV PHOENIX advertises NEITHER — and RGB-direct is the default on EFC hosts with true-extent the default at unaligned modes, so plain 1080p AV1 was tripping both on every frame: source 1920x1080 against an app-aligned 1920x1088 header and DPB. Measured 16 violations per 8-frame run; the CSC path had none. The fix is to make all three agree at the RENDER size rather than the aligned one — an unpadded coded size is valid on this hardware, so the coded frame simply IS the visible frame: - the AV1 sequence header follows the render size when true-extent is active (joining native NV12, which already authors true-size headers for the same reason); - the DPB setup and reference slots carry `src_extent` instead of the aligned `ext2d`. `src_extent` already collapses to `ext2d` whenever true-extent is off, so every other configuration is untouched; - `render_and_frame_size_different` now compares render against the DECLARED source extent instead of the aligned size, or true-extent would have claimed a mismatch that no longer exists. Fixing only the header is not enough and is actively misleading: it clears -10324 and immediately exposes -10325, because the mismatch has moved to the reference slots rather than gone. Both had to move together. This keeps the EFC fast path. Two alternatives were implemented, measured and rejected on the way here: falling back to the compute CSC costs the zero-copy the B2 work existed to deliver, and routing to the padded-copy staging trades these two VUIDs for VUID-VkImageCreateInfo-pNext-06811 — `pad_img`'s extra TRANSFER_SRC usage is not in the profile-advertised set, measured 8x per session on HEVC-padded too, so that is a pre-existing defect of the padded path and not somewhere to route a default session. HEVC is deliberately untouched: it has no equivalent constraint (its crop rides the conformance window), it measures zero violations, and its aligned-SPS path is the validated one. On RADV PHOENIX (780M, Mesa 26.0.4) the AV1 stream now decodes as coded 1920x1080 / render 1920x1080 — genuinely unpadded, where before the alignment rows were encoded and cropped back out. The CSC path still reports coded 1920x1088 / render 1920x1080, which is correct for it. All four `vulkan_smoke*` pass at 256x256 and 1920x1080. Two validation errors remain on the RGB-direct path and are NOT ours, now with evidence rather than assumption: `VUID-VkImageViewCreateInfo-image-08336` uses the PROFILE-BLIND format query, so it cannot see that RGB conversion legalises BGRA as an encode source — the profile-aware query used by -06811 accepts the very same image; and `VUID-VkQueryPoolCreateInfo-pNext-pNext` rejects `VkVideoEncodeProfileRgbConversionInfoVALVE`, which the VALVE extension REQUIRES for profile identity, and the layer diagnoses itself as "a struct from an extension added to a later version of the Vulkan header". Verified: canonical Linux gate (docker linux/amd64) L1-L4 green; on-glass on RADV PHOENIX under `VK_LOADER_LAYERS_ENABLE='*validation*'`, with the bitstreams read back through libdav1d/trace_headers. --- .../pf-encode/src/enc/linux/vk_av1_encode.rs | 9 ++++++ .../pf-encode/src/enc/linux/vulkan_video.rs | 29 ++++++++++++++++--- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/crates/pf-encode/src/enc/linux/vk_av1_encode.rs b/crates/pf-encode/src/enc/linux/vk_av1_encode.rs index c4bdddb5..55ba59dc 100644 --- a/crates/pf-encode/src/enc/linux/vk_av1_encode.rs +++ b/crates/pf-encode/src/enc/linux/vk_av1_encode.rs @@ -47,6 +47,15 @@ pub const PRIMARY_REF_NONE: u8 = 7; /// `VK_VIDEO_ENCODE_AV1_SUPERBLOCK_SIZE_128_BIT_KHR` (bit 1 of the superblock-size flags). pub const SUPERBLOCK_SIZE_128: u32 = 0x2; +// `VkVideoEncodeAV1CapabilityFlagBitsKHR` — the two that decide whether the encode source may be a +// different size from the declared frame. Both absent on RADV PHOENIX. +/// Without this, the source's `codedExtent` MUST equal the sequence header's +/// `max_frame_{width,height}_minus_1 + 1` (`VUID-vkCmdEncodeVideoKHR-flags-10324`). +pub const CAPABILITY_FRAME_SIZE_OVERRIDE: u32 = 0x0000_0008; +/// Without this, EVERY reference slot's `codedExtent` MUST equal the source's +/// (`VUID-vkCmdEncodeVideoKHR-flags-10325`). +pub const CAPABILITY_MOTION_VECTOR_SCALING: u32 = 0x0000_0010; + // `VkVideoEncodeAV1PredictionModeKHR` pub const PREDICTION_MODE_INTRA_ONLY: i32 = 0; pub const PREDICTION_MODE_SINGLE_REFERENCE: i32 = 1; diff --git a/crates/pf-encode/src/enc/linux/vulkan_video.rs b/crates/pf-encode/src/enc/linux/vulkan_video.rs index 2afbee3b..e90ba1e9 100644 --- a/crates/pf-encode/src/enc/linux/vulkan_video.rs +++ b/crates/pf-encode/src/enc/linux/vulkan_video.rs @@ -1053,7 +1053,23 @@ impl VulkanVideoEncoder { // matching true-size codedExtent, programs the VCN with firmware padding so the source // is never read past its extent. The CSC/RGB paths keep the app-aligned convention // (their sources genuinely cover the aligned extent). - let (hdr_w, hdr_h) = if native_nv12 { (rw, rh) } else { (w, h) }; + // AV1 + RGB true-extent joins the true-size camp for the same reason: true-extent shrinks + // `srcPictureResource.codedExtent` to the render size, and AV1 forbids the sequence header + // (`flags-10324`) or the reference slots (`flags-10325`) disagreeing with the source unless + // the GPU advertises FRAME_SIZE_OVERRIDE / MOTION_VECTOR_SCALING — RADV PHOENIX advertises + // neither. Rather than give up the EFC fast path, make all three agree at the RENDER size: + // the coded frame genuinely IS the visible size (an unpadded coded size is valid on this + // hardware), so nothing codes alignment padding at all and the reference slots follow the + // source extent below. + // + // HEVC deliberately stays on the app-aligned convention: it has no equivalent constraint + // (its crop rides the conformance window) and that path is the validated one. + let (hdr_w, hdr_h) = + if native_nv12 || (av1 && rgb_cfg.as_ref().is_some_and(|c| c.true_extent)) { + (rw, rh) + } else { + (w, h) + }; let (params, header, frame_prefix) = if av1 { build_parameters_av1( &device, @@ -3040,7 +3056,7 @@ impl VulkanVideoEncoder { // at any mode that needed 64x16 alignment the decoder fell back to the CODED size and the // client displayed our alignment padding — e.g. 1080p arriving as 1088 rows, the bottom 8 // being duplicated edge pixels. - if self.render_w != self.width || self.render_h != self.height { + if self.render_w != src_extent.width || self.render_h != src_extent.height { pic_flags.set_render_and_frame_size_different(1); } let mut std_pic: av1::StdVideoEncodeAV1PictureInfo = std::mem::zeroed(); @@ -3107,8 +3123,13 @@ impl VulkanVideoEncoder { }; // ---- setup (reconstruct into) + reference (read from) DPB slots ---- + // DPB slots carry the SOURCE extent, not the aligned one. Without + // MOTION_VECTOR_SCALING every reference slot's `codedExtent` must equal the source's + // (`VUID-vkCmdEncodeVideoKHR-flags-10325`), and in true-extent mode the source is the + // render size. `src_extent` already collapses to `ext2d` whenever true-extent is off, so + // this is a no-op for every other configuration. let setup_res = vk::VideoPictureResourceInfoKHR::default() - .coded_extent(ext2d) + .coded_extent(src_extent) .image_view_binding(self.dpb_views[setup_idx]); let mut setup_ref_std: av1::StdVideoEncodeAV1ReferenceInfo = std::mem::zeroed(); setup_ref_std.frame_type = std_pic.frame_type; @@ -3128,7 +3149,7 @@ impl VulkanVideoEncoder { begin_setup.p_next = &setup_dpb as *const _ as *const c_void; let ref_res = vk::VideoPictureResourceInfoKHR::default() - .coded_extent(ext2d) + .coded_extent(src_extent) .image_view_binding(self.dpb_views[ref_slot]); let mut ref_ref_std: av1::StdVideoEncodeAV1ReferenceInfo = std::mem::zeroed(); ref_ref_std.frame_type = if self.slot_poc[ref_slot] == 0 {