feat(encode/vulkan): Vulkan Video encodes 10-bit, so AMD/Intel HDR keeps the good path

The Vulkan Video backend was 8-bit for no structural reason — the API has
`VK_VIDEO_COMPONENT_BIT_DEPTH_10_BIT` and `PROFILE_IDC_MAIN_10` in the very
fields this pinned to 8 and MAIN, and AMD VCN and Intel both encode Main10.
It was six hardcoded sites, and the cost of leaving them was paid twice over:
an HDR session had to take libav VAAPI, losing real RFI loss recovery AND the
compute CSC's cursor blend — which on gamescope is the only way the pointer
reaches the stream at all, since gamescope has no embedded-cursor mode.

An HDR session now opens a Main10 profile with 10-bit component depths, a
`G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16` picture + DPB, and an SPS carrying
`bit_depth_*_minus8 = 2` with the BT.2020/PQ CICP triplet instead of BT.709.

`rgb2yuv10.comp` is the CSC's twin, and the two interesting parts of it are:

* it is a PURE 3x3 matrix. The samples arrive already PQ-encoded (gamescope
  composites into the PQ container), so BT.2020 NCL applies to the code values
  as they are — there is no transfer function to apply here and applying one
  would be wrong;
* the scratch planes are `R16`/`RG16`, not the picture's plane formats. The
  10-bit ycbcr plane formats are not storage-image formats, so the shader
  writes the value into the HIGH bits by hand (`code10 << 6`, hence the
  `64/65535` factor and not `1/1023`) into planes that are merely
  SIZE-compatible with the picture's — which is all `vkCmdCopyImage` requires.

Scope and safety:

* HEVC only. AV1 10-bit encode has far thinner driver coverage, and a session
  open is not the place to gamble on it — those stay on VAAPI, as does a device
  that fails the Main10 profile query inside the open (the pre-existing "failed
  Vulkan open falls back to VAAPI" net, no new probe needed).
* HDR pins the compute-CSC arm over the EFC RGB-direct one, which the EFC could
  not serve anyway: its fixed-function conversion is 8-bit BT.709 narrow with
  no knob for BT.2020.
* `open_inner` binds `hdr` to the parameter-set HEADER bytes, so the depth flag
  is `ten_bit` there — the one name collision this change had to route around.

crates/pf-encode/src/enc/linux/rgb2yuv10.comp

@@ -0,0 +1,70 @@
+#version 450
+// The 10-bit HDR twin of `rgb2yuv.comp`: packed 2:10:10:10 RGB -> 10-bit 4:2:0 (BT.2020 NCL,
+// limited range), for an HEVC Main10 session off a gamescope HDR capture.
+//
+// The source samples are ALREADY PQ-encoded BT.2020 R'G'B' (gamescope composites into the PQ
+// container; see packaging/gamescope), so this is a pure 3x3 matrix on the code values — there is
+// no transfer function to apply here, and applying one would be wrong. That is the whole
+// difference from the SDR shader besides the coefficients and the store width.
+//
+// STORE LAYOUT. The scratch planes are `r16`/`rg16` and get `vkCmdCopyImage`d into the
+// `G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16` picture, whose plane texels put the 10-bit value in
+// the HIGH bits of a 16-bit word (`X6` = 6 unused low bits). So a written normalized value `f`
+// lands as `round(f * 65535)` and must equal `code10 << 6` — hence the `CODE10` factor below,
+// NOT `1/1023`. Getting this wrong is a ~1.6% luminance error: invisible on a test pattern,
+// visible as crushed blacks on real content.
+//
+// Rebuild: glslangValidator -V rgb2yuv10.comp -o rgb2yuv10.spv   (vendored beside this file)
+layout(local_size_x = 8, local_size_y = 8) in;
+layout(binding = 0) uniform sampler2D rgb;                  // packed 2:10:10:10 input (sampled)
+layout(binding = 1, r16)  uniform writeonly image2D yImg;   // full-res Y  (10-bit in the high bits)
+layout(binding = 2, rg16) uniform writeonly image2D uvImg;  // half-res UV (interleaved)
+layout(binding = 3) uniform sampler2D cursorTex;            // straight-alpha RGBA cursor (top-left)
+
+layout(push_constant) uniform Push {
+    ivec2 curOrigin;   // top-left of the cursor in frame pixels (position - hotspot)
+    ivec2 curSize;     // cursor w,h in pixels; x <= 0 => disabled
+} pc;
+
+// 10-bit code value -> the normalized float that stores it in the high bits of a 16-bit texel.
+const float CODE10 = 64.0 / 65535.0;
+
+// BT.2020 non-constant-luminance, limited range, expressed directly in 10-bit code values:
+//   Y' = 64  + 876*(0.2627R + 0.6780G + 0.0593B)
+//   Cb = 512 + 896*(B' - Y')/1.8814
+//   Cr = 512 + 896*(R' - Y')/1.4746
+float lumaY(vec3 c) { return 64.0 + 230.1252*c.r + 593.9280*c.g + 51.9468*c.b; }
+
+// Blend the cursor over `col` at frame pixel `p`, when `p` falls inside the cursor rectangle.
+// The cursor bitmap is 8-bit sRGB and the frame is PQ, so this is a display-referred
+// approximation — the same one the CPU path (`pw_cursor.rs::composite_cursor_rgb10`) and the
+// CUDA path (`cursor_blend.comp` MODE 3/4) already make. Fine for a pointer.
+vec3 withCursor(ivec2 p, vec3 col) {
+    if (pc.curSize.x <= 0) return col;
+    ivec2 cp = p - pc.curOrigin;
+    if (cp.x < 0 || cp.y < 0 || cp.x >= pc.curSize.x || cp.y >= pc.curSize.y) return col;
+    vec4 c = texelFetch(cursorTex, cp, 0);
+    return mix(col, c.rgb, c.a);
+}
+
+// Source may be SMALLER than the coded (16-aligned) Y plane — clamp every fetch to the source
+// edge so the alignment-padding rows duplicate the last real row (see the SDR shader).
+void main() {
+    ivec2 sz = imageSize(yImg);
+    ivec2 rmax = textureSize(rgb, 0) - 1;
+    ivec2 uvc = ivec2(gl_GlobalInvocationID.xy);
+    ivec2 p = uvc * 2;
+    if (p.x >= sz.x || p.y >= sz.y) return;
+    vec3 c00 = withCursor(p,               texelFetch(rgb, min(p,               rmax), 0).rgb);
+    vec3 c10 = withCursor(p + ivec2(1, 0), texelFetch(rgb, min(p + ivec2(1, 0), rmax), 0).rgb);
+    vec3 c01 = withCursor(p + ivec2(0, 1), texelFetch(rgb, min(p + ivec2(0, 1), rmax), 0).rgb);
+    vec3 c11 = withCursor(p + ivec2(1, 1), texelFetch(rgb, min(p + ivec2(1, 1), rmax), 0).rgb);
+    imageStore(yImg, p,               vec4(lumaY(c00) * CODE10, 0, 0, 1));
+    imageStore(yImg, p + ivec2(1, 0), vec4(lumaY(c10) * CODE10, 0, 0, 1));
+    imageStore(yImg, p + ivec2(0, 1), vec4(lumaY(c01) * CODE10, 0, 0, 1));
+    imageStore(yImg, p + ivec2(1, 1), vec4(lumaY(c11) * CODE10, 0, 0, 1));
+    vec3 a = (c00 + c10 + c01 + c11) * 0.25;
+    float U = 512.0 - 125.1085*a.r - 322.8915*a.g + 448.0000*a.b;
+    float V = 512.0 + 448.0000*a.r - 411.9680*a.g -  36.0320*a.b;
+    imageStore(uvImg, uvc, vec4(U * CODE10, V * CODE10, 0, 1));
+}

crates/pf-encode/src/enc/linux/vk_build.rs

@@ -225,6 +225,7 @@ pub(super) unsafe fn make_frame(
     with_ts: bool,
     csc: bool,
     pad_fmt: Option<vk::Format>,
+    hdr: bool,
     f: &mut Frame,
 ) -> Result<()> {
     // "no cursor uploaded yet" sentinel — a real serial may be 0 (see `prep_cursor`).
@@ -263,6 +264,7 @@ pub(super) unsafe fn make_frame(
             csc_dsl,
             csc_pool,
             sampler,
+            hdr,
             f,
         )?;
     }
@@ -291,13 +293,15 @@ unsafe fn make_frame_csc(
     csc_dsl: vk::DescriptorSetLayout,
     csc_pool: vk::DescriptorPool,
     sampler: vk::Sampler,
+    hdr: bool,
     f: &mut Frame,
 ) -> Result<()> {
-    // NV12 encode-src (filled by the CSC copy) — concurrent compute+encode.
+    // 4:2:0 encode-src (filled by the CSC copy) — concurrent compute+encode.
+    let pic = yuv_format(hdr);
     (f.nv12_src, f.nv12_mem) = make_video_image(
         device,
         mem_props,
-        NV12,
+        pic,
         w,
         h,
         1,
@@ -305,12 +309,22 @@ unsafe fn make_frame_csc(
         profile_list,
         fams,
     )?;
-    f.nv12_view = make_view(device, f.nv12_src, NV12, 0)?;
-    // CSC scratch (Y R8 full-res, UV RG8 half-res).
+    f.nv12_view = make_view(device, f.nv12_src, pic, 0)?;
+    // CSC scratch: Y full-res + UV half-res, in a single-plane format the shader can declare as a
+    // storage image AND that is SIZE-COMPATIBLE with the picture's planes (`vkCmdCopyImage`
+    // between differing formats requires equal texel-block size). 8-bit: R8/RG8 vs the NV12
+    // planes' 1/2 bytes. 10-bit: R16/RG16 vs the 3PACK16 planes' 2/4 bytes — the 10-bit ycbcr
+    // plane formats themselves are not storage-image formats, which is why the scratch is 16-bit
+    // and `rgb2yuv10.comp` writes the value into the high bits by hand.
+    let (y_fmt, uv_fmt) = if hdr {
+        (vk::Format::R16_UNORM, vk::Format::R16G16_UNORM)
+    } else {
+        (vk::Format::R8_UNORM, vk::Format::R8G8_UNORM)
+    };
     (f.y_img, f.y_mem, f.y_view) = make_plain_image(
         device,
         mem_props,
-        vk::Format::R8_UNORM,
+        y_fmt,
         w,
         h,
         vk::ImageUsageFlags::STORAGE | vk::ImageUsageFlags::TRANSFER_SRC,
@@ -318,7 +332,7 @@ unsafe fn make_frame_csc(
     (f.uv_img, f.uv_mem, f.uv_view) = make_plain_image(
         device,
         mem_props,
-        vk::Format::R8G8_UNORM,
+        uv_fmt,
         w / 2,
         h / 2,
         vk::ImageUsageFlags::STORAGE | vk::ImageUsageFlags::TRANSFER_SRC,
@@ -479,12 +493,20 @@ pub(super) unsafe fn build_parameters_h265(
     rw: u32,
     rh: u32,
     quality_level: u32,
+    // 10-bit HDR session: Main10 + BT.2020/PQ colour signalling. Must agree with the video
+    // profile the session was CREATED with (`open_inner`'s `ten_bit`) — a Main SPS on a Main10
+    // session is a bitstream that says one thing and carries another.
+    ten_bit: bool,
 ) -> Result<(vk::VideoSessionParametersKHR, Vec<u8>)> {
     use ash::vk::native as hh;
     let mut ptl: hh::StdVideoH265ProfileTierLevel = std::mem::zeroed();
     ptl.flags.set_general_progressive_source_flag(1);
     ptl.flags.set_general_frame_only_constraint_flag(1);
-    ptl.general_profile_idc = hh::StdVideoH265ProfileIdc_STD_VIDEO_H265_PROFILE_IDC_MAIN;
+    ptl.general_profile_idc = if ten_bit {
+        hh::StdVideoH265ProfileIdc_STD_VIDEO_H265_PROFILE_IDC_MAIN_10
+    } else {
+        hh::StdVideoH265ProfileIdc_STD_VIDEO_H265_PROFILE_IDC_MAIN
+    };
     ptl.general_level_idc = hh::StdVideoH265LevelIdc_STD_VIDEO_H265_LEVEL_IDC_6_0;
 
     let mut dpbm: hh::StdVideoH265DecPicBufMgr = std::mem::zeroed();
@@ -505,6 +527,11 @@ pub(super) unsafe fn build_parameters_h265(
     sps.pic_width_in_luma_samples = w;
     sps.pic_height_in_luma_samples = h;
     sps.log2_max_pic_order_cnt_lsb_minus4 = 4;
+    // Main10's `bit_depth_*_minus8 = 2`. Zeroed (= 8-bit) for Main, as before.
+    if ten_bit {
+        sps.bit_depth_luma_minus8 = 2;
+        sps.bit_depth_chroma_minus8 = 2;
+    }
     sps.log2_diff_max_min_luma_coding_block_size = 3;
     sps.log2_diff_max_min_luma_transform_block_size = 3;
     sps.max_transform_hierarchy_depth_inter = 4;
@@ -517,25 +544,26 @@ pub(super) unsafe fn build_parameters_h265(
         sps.conf_win_bottom_offset = (h - rh) / 2; // 4:2:0 SubHeightC = 2
     }
 
-    // Colour signalling. This backend's CSC (`rgb2yuv.comp`) is BT.709 LIMITED 8-bit and nothing
-    // else — `open_amd_intel` routes every HDR session to VAAPI precisely because this path
-    // hardcodes it — so the SPS can state it as a constant. Without the VUI the stream is
-    // "unspecified" and each decoder applies its own default: the punktfunk clients fall back to
-    // BT.709 (`pf_client_core::video_color::csc_rows`), but vendor TV decoders guess from
-    // RESOLUTION — an LG webOS panel reads a 4K SDR stream as BT.2020 and renders it visibly
-    // washed out. Every sibling backend (NVENC `nvenc_core.rs`, VAAPI, QSV, the Windows libav
-    // path) already signals this triplet; this one was the hole.
+    // Colour signalling, exactly what this session's CSC produced: `rgb2yuv.comp` is BT.709
+    // limited 8-bit, `rgb2yuv10.comp` is BT.2020 NCL limited 10-bit with a PQ transfer (the
+    // samples arrive PQ-encoded from the compositor and the matrix does not touch the transfer).
+    // Without the VUI the stream is "unspecified" and each decoder applies its own default: the
+    // punktfunk clients fall back to BT.709 (`pf_client_core::video_color::csc_rows`), but vendor
+    // TV decoders guess from RESOLUTION — an LG webOS panel reads a 4K SDR stream as BT.2020 and
+    // renders it visibly washed out.
     //
     // `vui` must outlive `create_video_session_parameters_khr` below — it does, `sps_arr` only
     // copies the pointer and both live to the end of this function.
     let mut vui: hh::StdVideoH265SequenceParameterSetVui = std::mem::zeroed();
     vui.flags.set_video_signal_type_present_flag(1);
-    vui.flags.set_video_full_range_flag(0); // limited/studio swing (16-235 luma)
+    vui.flags.set_video_full_range_flag(0); // limited/studio swing
     vui.flags.set_colour_description_present_flag(1);
     vui.video_format = 5; // unspecified — the CICP triplet below is what matters
-    vui.colour_primaries = 1; // BT.709
-    vui.transfer_characteristics = 1; // BT.709
-    vui.matrix_coeffs = 1; // BT.709
+                          // CICP code points: 1 = BT.709, 9 = BT.2020 primaries / BT.2020 NCL matrix, 16 = SMPTE 2084.
+    let (prim, trc, mat) = if ten_bit { (9, 16, 9) } else { (1, 1, 1) };
+    vui.colour_primaries = prim;
+    vui.transfer_characteristics = trc;
+    vui.matrix_coeffs = mat;
     sps.flags.set_vui_parameters_present_flag(1);
     sps.pSequenceParameterSetVui = &vui;
 

crates/pf-encode/src/enc/linux/vk_util.rs

@@ -50,9 +50,23 @@ pub(crate) fn fourcc_to_vk(fourcc: u32) -> Option<vk::Format> {
     const XB24: u32 = 0x3432_4258; // XBGR8888
     const AB24: u32 = 0x3432_4241; // ABGR8888
     const NV12: u32 = 0x3231_564e; // DRM_FORMAT_NV12
+                                   // The 10-bit HDR capture formats. DRM packs these as a little-endian 32-bit word — XR30 is
+                                   // x:R:G:B with B in bits 0-9 — which is exactly Vulkan's `A2R10G10B10_UNORM_PACK32` bit
+                                   // layout, so the mapping is an identity on the word and not a byte swizzle like the 8-bit
+                                   // pair above.
+                                   //
+                                   // ⚠ Only `A2B10G10R10_UNORM_PACK32` is a Vulkan-mandatory SAMPLED format; `A2R10G10B10` is
+                                   // optional (widely supported on RADV/ANV, and we only ever `texelFetch` it — no filtering).
+                                   // Both are offered to the producer, so which one a session lands on is the producer's pick;
+                                   // if a device ever rejects the XR30 import, the fix is to drop that format from the capture
+                                   // offer rather than to convert here.
+    const XR30: u32 = 0x3033_5258; // DRM_FORMAT_XRGB2101010
+    const XB30: u32 = 0x3033_4258; // DRM_FORMAT_XBGR2101010
     match fourcc {
         XR24 | AR24 => Some(vk::Format::B8G8R8A8_UNORM),
         XB24 | AB24 => Some(vk::Format::R8G8B8A8_UNORM),
+        XR30 => Some(vk::Format::A2R10G10B10_UNORM_PACK32),
+        XB30 => Some(vk::Format::A2B10G10R10_UNORM_PACK32),
         NV12 => Some(vk::Format::G8_B8R8_2PLANE_420_UNORM),
         _ => None,
     }
@@ -62,6 +76,10 @@ pub(crate) fn pixel_to_vk(fmt: PixelFormat) -> Option<vk::Format> {
     match fmt {
         PixelFormat::Bgrx | PixelFormat::Bgra => Some(vk::Format::B8G8R8A8_UNORM),
         PixelFormat::Rgbx | PixelFormat::Rgba => Some(vk::Format::R8G8B8A8_UNORM),
+        // The packed 10-bit PQ/BT.2020 capture formats (an HDR gamescope output). Sampling one
+        // yields the PQ code values normalized to [0,1] — which is what `rgb2yuv10.comp` wants.
+        PixelFormat::X2Rgb10 => Some(vk::Format::A2R10G10B10_UNORM_PACK32),
+        PixelFormat::X2Bgr10 => Some(vk::Format::A2B10G10R10_UNORM_PACK32),
         _ => None,
     }
 }

crates/pf-encode/src/enc/linux/vulkan_video.rs

@@ -4,8 +4,10 @@
 //! slot (no IDR): HEVC via an explicit short-term RPS, AV1 via `ref_frame_idx` + a
 //! `primary_ref_frame = NONE` recovery anchor that also breaks the CDF chain.
 //!
-//! Capture delivers packed RGB (dmabuf/CPU); this backend imports it, runs an on-GPU RGB→NV12
-//! BT.709 compute CSC, then encodes. Proven end-to-end in `punktfunk-planning/design/vkenc-probe-harness`.
+//! Capture delivers packed RGB (dmabuf/CPU); this backend imports it, runs an on-GPU RGB→4:2:0
+//! compute CSC, then encodes — 8-bit BT.709 for an SDR session (`rgb2yuv.comp`), 10-bit BT.2020
+//! for an HDR one (`rgb2yuv10.comp` + an HEVC Main10 session; a 10-bit AV1 session is routed to
+//! VAAPI instead). Proven end-to-end in `punktfunk-planning/design/vkenc-probe-harness`.
 //! Opt-in via `PUNKTFUNK_VULKAN_ENCODE`; gated to HEVC/AV1 + a device that advertises the encode op.
 //! The AV1 encode structs our pinned `ash 0.38` predates are vendored in `vk_av1_encode.rs`.
 #![allow(clippy::too_many_arguments)]
@@ -23,12 +25,31 @@ use std::ffi::c_void;
 use std::os::fd::AsRawFd;
 
 const NV12: vk::Format = vk::Format::G8_B8R8_2PLANE_420_UNORM;
+/// The 10-bit 4:2:0 picture/DPB format an HDR (HEVC Main10) session encodes from. `3PACK16`
+/// stores each 10-bit sample in the HIGH bits of a 16-bit word — see `rgb2yuv10.comp`, whose
+/// scratch planes are the size-compatible `R16`/`RG16` this gets `vkCmdCopyImage`d from.
+const P010: vk::Format = vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16;
+
+/// The session's 4:2:0 picture + DPB format for its bit depth. One function so the session
+/// create-info, the DPB image, its views and the per-frame encode source cannot drift apart —
+/// they must all name the SAME format or session creation fails.
+const fn yuv_format(hdr: bool) -> vk::Format {
+    if hdr {
+        P010
+    } else {
+        NV12
+    }
+}
 /// Max resident dmabuf imports (comfortably above any PipeWire pool depth; imports alias existing
 /// buffers so this holds handles, not new allocations).
 const IMPORT_CACHE_CAP: usize = 16;
 // Prebuilt SPIR-V for the RGB→NV12 BT.709 compute CSC. Source is `rgb2yuv.comp` beside this file;
 // regenerate with `glslangValidator -V rgb2yuv.comp -o rgb2yuv.spv` after editing the shader.
 const CSC_SPV: &[u8] = include_bytes!("rgb2yuv.spv");
+/// The 10-bit HDR twin (`rgb2yuv10.comp`): packed 2:10:10:10 PQ/BT.2020 RGB → 10-bit 4:2:0,
+/// BT.2020 NCL limited. Separate module rather than a spec constant because a shader's storage
+/// image FORMAT (`r8`/`rg8` vs `r16`/`rg16`) is part of its layout, not specializable.
+const CSC10_SPV: &[u8] = include_bytes!("rgb2yuv10.spv");
 /// Fixed cursor-overlay texture size (px). Larger than any real pointer; the actual `w×h` uploads
 /// into the top-left and the shader's push constant bounds sampling, so one allocation fits every
 /// cursor and no per-size recreation is needed. See the CSC shader's `cursorTex`/push constant.
@@ -463,6 +484,9 @@ struct Frame {
     uv_img: vk::Image,
     uv_mem: vk::DeviceMemory,
     uv_view: vk::ImageView,
+    /// The CSC's output picture and this frame's encode source: NV12, or the 10-bit
+    /// `P010`/3PACK16 twin on an HDR session ([`yuv_format`]). The name predates the second
+    /// depth; every use goes through the format the session was created with.
     nv12_src: vk::Image,
     nv12_mem: vk::DeviceMemory,
     nv12_view: vk::ImageView,
@@ -669,16 +693,30 @@ impl VulkanVideoEncoder {
         cursor_blend: bool,
     ) -> Result<Self> {
         let native_nv12 = format == PixelFormat::Nv12;
-        // A cursor-blend session must keep the compute-CSC path — the only arm with the cursor
-        // blend — so it outranks an explicit RGB-direct pin (EFC cannot composite; the
-        // negotiation promised the client a composited pointer).
-        if cursor_blend && rgb_request() == Some(true) {
-            tracing::info!(
-                "PUNKTFUNK_VULKAN_RGB_DIRECT=1 ignored for this session — it composites the \
-                 pointer, which the EFC front-end cannot; using the compute-CSC path"
+        // HDR: a packed 10-bit PQ/BT.2020 capture (a gamescope output off our `pipewire-hdr`
+        // build) encodes HEVC Main10 through the 10-bit compute CSC. HEVC only — AV1 10-bit
+        // encode has far thinner driver coverage, and `open_amd_intel` keeps those sessions on
+        // libav VAAPI rather than gambling a session open on it.
+        let hdr = format.is_hdr_rgb10();
+        if hdr && codec != Codec::H265 {
+            bail!(
+                "vulkan-encode: 10-bit HDR is HEVC Main10 only here (got {codec:?}) — the \
+                 dispatcher should have routed this session to VAAPI"
             );
         }
-        let want_rgb = !native_nv12 && !cursor_blend && rgb_request().unwrap_or(true);
+        // A cursor-blend session must keep the compute-CSC path — the only arm with the cursor
+        // blend — so it outranks an explicit RGB-direct pin (EFC cannot composite; the
+        // negotiation promised the client a composited pointer). HDR pins the same arm for a
+        // different reason: the EFC's fixed-function conversion is 8-bit BT.709 narrow with no
+        // knob for BT.2020, so it cannot express this session's colourimetry at all.
+        if (cursor_blend || hdr) && rgb_request() == Some(true) {
+            tracing::info!(
+                hdr,
+                "PUNKTFUNK_VULKAN_RGB_DIRECT=1 ignored for this session — the EFC front-end can \
+                 neither composite a pointer nor convert BT.2020 10-bit; using the compute-CSC path"
+            );
+        }
+        let want_rgb = !native_nv12 && !cursor_blend && !hdr && rgb_request().unwrap_or(true);
         Self::open_opts_inner(
             codec,
             width,
@@ -687,6 +725,7 @@ impl VulkanVideoEncoder {
             bitrate_bps,
             want_rgb,
             native_nv12,
+            hdr,
         )
     }
 
@@ -704,9 +743,19 @@ impl VulkanVideoEncoder {
         bitrate_bps: u64,
         want_rgb: bool,
     ) -> Result<Self> {
-        Self::open_opts_inner(codec, width, height, fps, bitrate_bps, want_rgb, false)
+        Self::open_opts_inner(
+            codec,
+            width,
+            height,
+            fps,
+            bitrate_bps,
+            want_rgb,
+            false,
+            false,
+        )
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn open_opts_inner(
         codec: Codec,
         width: u32,
@@ -715,6 +764,7 @@ impl VulkanVideoEncoder {
         bitrate_bps: u64,
         want_rgb: bool,
         native_nv12: bool,
+        hdr: bool,
     ) -> Result<Self> {
         if !matches!(codec, Codec::H265 | Codec::Av1) {
             bail!("vulkan-encode backend supports HEVC + AV1 only (got {codec:?})");
@@ -737,6 +787,7 @@ impl VulkanVideoEncoder {
                 bitrate_bps.max(1_000_000),
                 want_rgb,
                 native_nv12,
+                hdr,
             )
         }
     }
@@ -752,6 +803,8 @@ impl VulkanVideoEncoder {
         bitrate: u64,
         want_rgb: bool,
         native_nv12: bool,
+        // NOT `hdr`: this fn already binds that name to the parameter-set HEADER bytes below.
+        ten_bit: bool,
     ) -> Result<Self> {
         use super::vk_av1_encode as av1b;
         use super::vk_valve_rgb as vrgb;
@@ -871,8 +924,12 @@ impl VulkanVideoEncoder {
             p_next: std::ptr::null(),
             perform_encode_rgb_conversion: vk::TRUE,
         };
-        let mut h265_profile = vk::VideoEncodeH265ProfileInfoKHR::default()
-            .std_profile_idc(vk::native::StdVideoH265ProfileIdc_STD_VIDEO_H265_PROFILE_IDC_MAIN);
+        let mut h265_profile =
+            vk::VideoEncodeH265ProfileInfoKHR::default().std_profile_idc(if ten_bit {
+                vk::native::StdVideoH265ProfileIdc_STD_VIDEO_H265_PROFILE_IDC_MAIN_10
+            } else {
+                vk::native::StdVideoH265ProfileIdc_STD_VIDEO_H265_PROFILE_IDC_MAIN
+            });
         let mut av1_profile = av1b::VideoEncodeAV1ProfileInfoKHR {
             s_type: av1b::stype(av1b::ST_PROFILE_INFO),
             p_next: std::ptr::null(),
@@ -885,11 +942,20 @@ impl VulkanVideoEncoder {
         if rgb_cfg.is_some() {
             usage.p_next = &rgb_info as *const _ as *const c_void;
         }
+        // A device that cannot encode 10-bit fails `get_physical_device_video_capabilities`
+        // below with VIDEO_PROFILE_FORMAT_NOT_SUPPORTED, which fails the open — and a failed
+        // Vulkan open falls back to libav VAAPI in `open_amd_intel`. That is the whole 10-bit
+        // capability gate: no separate probe, and no way to reach a half-configured session.
+        let depth = if ten_bit {
+            vk::VideoComponentBitDepthFlagsKHR::TYPE_10
+        } else {
+            vk::VideoComponentBitDepthFlagsKHR::TYPE_8
+        };
         let mut profile = vk::VideoProfileInfoKHR::default()
             .video_codec_operation(codec_op)
             .chroma_subsampling(vk::VideoChromaSubsamplingFlagsKHR::TYPE_420)
-            .luma_bit_depth(vk::VideoComponentBitDepthFlagsKHR::TYPE_8)
-            .chroma_bit_depth(vk::VideoComponentBitDepthFlagsKHR::TYPE_8);
+            .luma_bit_depth(depth)
+            .chroma_bit_depth(depth);
         if av1 {
             av1_profile.p_next = &usage as *const _ as *const c_void;
             profile.p_next = &av1_profile as *const _ as *const c_void;
@@ -1120,13 +1186,13 @@ impl VulkanVideoEncoder {
             .picture_format(if rgb_cfg.is_some() {
                 vk::Format::B8G8R8A8_UNORM
             } else {
-                NV12
+                yuv_format(ten_bit)
             })
             .max_coded_extent(vk::Extent2D {
                 width: w,
                 height: h,
             })
-            .reference_picture_format(NV12)
+            .reference_picture_format(yuv_format(ten_bit))
             .max_dpb_slots(DPB_SLOTS + 1)
             .max_active_reference_pictures(1)
             .std_header_version(&std_hdr);
@@ -1236,6 +1302,7 @@ impl VulkanVideoEncoder {
                 rw,
                 rh,
                 quality_level,
+                ten_bit,
             )?;
             (p, hdr, Vec::new())
         };
@@ -1247,7 +1314,7 @@ impl VulkanVideoEncoder {
         let (dpb_image, dpb_mem) = make_video_image(
             &device,
             &mem_props,
-            NV12,
+            yuv_format(ten_bit),
             w,
             h,
             DPB_SLOTS,
@@ -1260,7 +1327,7 @@ impl VulkanVideoEncoder {
         for slot in 0..DPB_SLOTS {
             guard
                 .dpb_views
-                .push(make_view(&device, dpb_image, NV12, slot)?);
+                .push(make_view(&device, dpb_image, yuv_format(ten_bit), slot)?);
         }
 
         // NV12 encode-src, CSC scratch (Y/UV), bitstream, query and command buffers are all per
@@ -1281,7 +1348,11 @@ impl VulkanVideoEncoder {
             None,
         )?;
         guard.sampler = sampler;
-        let spv = ash::util::read_spv(&mut std::io::Cursor::new(CSC_SPV))?;
+        let spv = ash::util::read_spv(&mut std::io::Cursor::new(if ten_bit {
+            CSC10_SPV
+        } else {
+            CSC_SPV
+        }))?;
         let shader =
             device.create_shader_module(&vk::ShaderModuleCreateInfo::default().code(&spv), None)?;
         guard.shader = shader;
@@ -1396,6 +1467,7 @@ impl VulkanVideoEncoder {
                     .as_ref()
                     .is_some_and(|c| c.padded)
                     .then_some(vk::Format::B8G8R8A8_UNORM),
+                ten_bit,
                 guard.frames.last_mut().expect("frame just pushed"),
             )?;
         }

crates/pf-encode/src/lib.rs

@@ -357,24 +357,25 @@ fn open_video_backend_linux(
     // Linux binary serves any GPU; `PUNKTFUNK_ENCODER` forces a specific backend (and surfaces
     // its errors crisply instead of silently trying the other).
     // AMD/Intel opener. Default = libav VAAPI. With `--features vulkan-encode` +
-    // PUNKTFUNK_VULKAN_ENCODE, an HEVC session instead opens the raw Vulkan Video backend (real
-    // RFI loss recovery the VAAPI path can't express); a failed open falls back to VAAPI so the
-    // stream never dies over the new path. `format`/`bit_depth`/`chroma` only matter to VAAPI —
-    // the Vulkan backend imports the dmabuf and does its own 8-bit 4:2:0 CSC.
+    // PUNKTFUNK_VULKAN_ENCODE, an HEVC/AV1 session instead opens the raw Vulkan Video backend
+    // (real RFI loss recovery the VAAPI path can't express); a failed open falls back to VAAPI so
+    // the stream never dies over the new path. `format`/`bit_depth`/`chroma` only matter to VAAPI
+    // — the Vulkan backend imports the dmabuf and does its own CSC.
     let open_amd_intel = || -> Result<(Box<dyn Encoder>, &'static str)> {
-        // An HDR session (10-bit + a PQ/BT.2020 capture format) must skip the Vulkan Video
-        // backend — it hardcodes an 8-bit 4:2:0 BT.709 CSC — and take the libav VAAPI path,
-        // which has the P010/Main10/PQ wiring. SDR sessions keep the Vulkan default.
+        // HDR (10-bit + a PQ/BT.2020 capture format) takes the Vulkan backend for **HEVC**: its
+        // compute CSC has a 10-bit BT.2020 variant and the session opens Main10. That keeps the
+        // two things an HDR session would otherwise lose here — real RFI recovery and the
+        // compute-CSC cursor blend, which is the only way a gamescope pointer reaches the stream
+        // (gamescope has no embedded-cursor mode to fall back to).
         //
-        // Two things ride this switch, and both are the accepted cost of AMD/Intel HDR until
-        // Vulkan Video learns 10-bit: the Vulkan backend's real RFI loss recovery, and its
-        // compute-CSC **cursor blend**. A gamescope HDR session therefore streams without the
-        // host-composited XFixes pointer (gamescope has no embedded-cursor mode to fall back
-        // to) — `open_video`'s `blends_cursor` backstop logs it per session.
+        // AV1 10-bit stays on libav VAAPI: encode-side driver coverage for it is far thinner than
+        // HEVC Main10, and this is not the place to gamble a session open. A device that cannot
+        // do Main10 either fails `get_physical_device_video_capabilities` inside the open and
+        // lands on the VAAPI fallback below — the same net as any other unsupported config.
         #[cfg(feature = "vulkan-encode")]
         if matches!(codec, Codec::H265 | Codec::Av1)
             && vulkan_encode_enabled()
-            && !(bit_depth == 10 && format.is_hdr_rgb10())
+            && !(bit_depth == 10 && format.is_hdr_rgb10() && codec != Codec::H265)
         {
             match vulkan_video::VulkanVideoEncoder::open(
                 codec,
@@ -1045,8 +1046,14 @@ pub fn linux_hdr_cuda_ok() -> bool {
 ///
 /// `cuda_planned` is the caller's prediction of a CUDA capture payload (NVIDIA + zero-copy — the
 /// prediction `SessionPlan` already makes); `ten_bit` the negotiated depth. Both shift the
-/// dispatch: a CPU payload keeps NVIDIA on libav NVENC (no blend), and a 10-bit HDR session
-/// skips Vulkan Video for libav VAAPI's P010/Main10 wiring (no blend).
+/// dispatch: a CPU payload keeps NVIDIA on libav NVENC (no blend), and a 10-bit **AV1** session
+/// skips Vulkan Video for libav VAAPI (no blend) — 10-bit HEVC does not, its compute CSC has a
+/// BT.2020 Main10 variant that blends like the 8-bit one.
+///
+/// Whether the DEVICE can encode Main10 is only settled when the session opens (the profile query
+/// is what answers it); a device that cannot falls back to VAAPI there, and `open_video`'s
+/// `blends_cursor` backstop logs the divergence. Same shape as every other open-time fallback
+/// this prediction cannot see.
 #[cfg(target_os = "linux")]
 pub fn cursor_blend_capable(codec: Codec, cuda_planned: bool, ten_bit: bool) -> bool {
     // A negotiated PyroWave session routes to that backend before the pref is consulted
@@ -1071,6 +1078,8 @@ pub fn cursor_blend_capable(codec: Codec, cuda_planned: bool, ten_bit: bool) ->
         {
             matches!(codec, Codec::H265 | Codec::Av1)
                 && vulkan_encode_enabled()
+                // 10-bit is HEVC Main10 only on this backend — mirrors `open_amd_intel`.
+                && !(ten_bit && codec != Codec::H265)
                 && vulkan_encode_available(codec)
         }
         #[cfg(not(feature = "vulkan-encode"))]
@@ -1083,7 +1092,7 @@ pub fn cursor_blend_capable(codec: Codec, cuda_planned: bool, ten_bit: bool) ->
         linux_auto_is_vaapi,
         cuda_planned,
     );
-    cursor_blend_capable_for(backend, cuda_planned, ten_bit, direct_nvenc, vulkan_csc)
+    cursor_blend_capable_for(backend, cuda_planned, direct_nvenc, vulkan_csc)
 }
 
 /// The dispatch-mirroring core of [`cursor_blend_capable`], device-free for the unit tests.
@@ -1094,7 +1103,6 @@ pub fn cursor_blend_capable(codec: Codec, cuda_planned: bool, ten_bit: bool) ->
 fn cursor_blend_capable_for(
     backend: Option<LinuxBackend>,
     cuda_planned: bool,
-    ten_bit: bool,
     direct_nvenc: bool,
     vulkan_csc: bool,
 ) -> bool {
@@ -1104,11 +1112,11 @@ fn cursor_blend_capable_for(
         // Only the direct-SDK arm blends (VkSlotBlend), and it only takes CUDA payloads —
         // a CPU-payload session stays on libav NVENC, which cannot blend.
         Some(LinuxBackend::Nvenc) => cuda_planned && direct_nvenc,
-        // The Vulkan Video compute-CSC path blends; a 10-bit HDR session skips it for libav
-        // VAAPI (no blend). The session plan keeps a cursor-blend session off the native-NV12
-        // and RGB-direct shapes (`SessionPlan::output_format` / `VulkanVideoEncoder::open`),
-        // so CSC eligibility IS the answer.
-        Some(LinuxBackend::AmdIntel) | Some(LinuxBackend::Vulkan) => !ten_bit && vulkan_csc,
+        // The Vulkan Video compute-CSC path blends, at either depth. The session plan keeps a
+        // cursor-blend session off the native-NV12 and RGB-direct shapes
+        // (`SessionPlan::output_format` / `VulkanVideoEncoder::open`), so CSC eligibility IS the
+        // answer — including the depth term, which `vulkan_csc` already carries.
+        Some(LinuxBackend::AmdIntel) | Some(LinuxBackend::Vulkan) => vulkan_csc,
         // CPU frames: the capturer composites the metadata cursor inline before the encoder
         // runs, but the ENCODER blends nothing — the cursor channel's on-demand composite
         // contract can't be honored. Report the encoder's truth.
@@ -2081,58 +2089,32 @@ mod tests {
             Some(Pyrowave),
             false,
             false,
-            false,
             false
         ));
         // NVIDIA: only the direct-SDK arm blends (VkSlotBlend), and only for CUDA payloads.
-        assert!(cursor_blend_capable_for(
-            Some(Nvenc),
-            true,
-            false,
-            true,
-            false
-        ));
+        assert!(cursor_blend_capable_for(Some(Nvenc), true, true, false));
         assert!(
-            !cursor_blend_capable_for(Some(Nvenc), false, false, true, false),
+            !cursor_blend_capable_for(Some(Nvenc), false, true, false),
             "a CPU payload stays on libav NVENC, which cannot blend"
         );
         assert!(
-            !cursor_blend_capable_for(Some(Nvenc), true, false, false, false),
+            !cursor_blend_capable_for(Some(Nvenc), true, false, false),
             "PUNKTFUNK_NVENC_DIRECT=0 (or a build without the feature) is the libav path"
         );
-        // AMD/Intel: the Vulkan Video compute-CSC arm blends; VAAPI never does.
-        assert!(cursor_blend_capable_for(
-            Some(AmdIntel),
-            false,
-            false,
-            false,
-            true
-        ));
+        // AMD/Intel: the Vulkan Video compute-CSC arm blends — at 8 AND 10 bits, since its CSC
+        // has a BT.2020 Main10 variant. VAAPI never does. The depth term lives in `vulkan_csc`
+        // (10-bit AV1 is the one combination that still falls through to VAAPI), so this arm is
+        // now exactly "did an eligible CSC arm resolve".
+        assert!(cursor_blend_capable_for(Some(AmdIntel), false, false, true));
         assert!(
-            !cursor_blend_capable_for(Some(AmdIntel), false, false, false, false),
+            !cursor_blend_capable_for(Some(AmdIntel), false, false, false),
             "no eligible Vulkan CSC arm (H.264, PUNKTFUNK_VULKAN_ENCODE=0, unsupported \
              device) resolves to libav VAAPI, which cannot blend"
         );
-        assert!(
-            !cursor_blend_capable_for(Some(AmdIntel), false, true, false, true),
-            "a 10-bit HDR session skips Vulkan Video for VAAPI's P010 wiring — no blend"
-        );
-        assert!(cursor_blend_capable_for(
-            Some(Vulkan),
-            false,
-            false,
-            false,
-            true
-        ));
+        assert!(cursor_blend_capable_for(Some(Vulkan), false, false, true));
         // Software / unknown pref: CPU frames; the encoder blends nothing.
-        assert!(!cursor_blend_capable_for(
-            Some(Software),
-            false,
-            false,
-            true,
-            true
-        ));
-        assert!(!cursor_blend_capable_for(None, false, false, true, true));
+        assert!(!cursor_blend_capable_for(Some(Software), false, true, true));
+        assert!(!cursor_blend_capable_for(None, false, true, true));
     }
 
     /// WP7.7 guard (the cheap half): every `Encoder` trait method must be explicitly forwarded by