feat(encode/nvenc): an HDR capture stays zero-copy on NVIDIA

AMD/Intel needed no new encoder code for HDR — the VAAPI path already ingests an XR30 dmabuf into `format=p010:out_color_matrix=bt2020`. NVIDIA did: the 10-bit formats were excluded from the GPU import outright, so an HDR session fell back to a CPU readback plus swscale, which is the one thing the capture path is not allowed to ship. It turns out no CSC kernel is needed. NVENC ingests packed 10-bit RGB natively as `ARGB10`/`ABGR10` and does the conversion itself following the configured VUI matrix — which `apply_low_latency_config` already sets to BT.2020 NCL for an HDR session. So the frame travels LINEAR dmabuf → Vulkan bridge → CUDA → NVENC unconverted: no host CSC pass, no depth loss, no extra work on a contended SM. * invariant 1 is restated rather than dropped: HDR must never take the TILED EGL de-tile blit (it renders into an 8-bit `GL_RGBA8` texture). The HDR pods are LINEAR-only by construction, so the plan may build the importer; the per-frame gate — which sees the negotiated modifier the plan cannot — is what enforces the tiled half, and falls back to the CPU path if a producer ever ignores our offer. * …but only where the encoder can actually take the payload (`linux_hdr_cuda_ok`). libav's HDR route builds a P010 hardware frames context and swscales into it, so on a host without the direct-SDK backend a packed-2:10:10:10 CUDA buffer would land in a P010 surface as garbage. Those keep the CPU path. * `nvenc_cuda` stops pinning 8-bit/SDR. Depth and HDR now follow the INPUT format, like the Windows backend: a 10-bit session whose capture came back 8-bit encodes AND labels 8-bit rather than mislabelling. * the cursor-blend compute shader gains two 10-bit modes, so the pointer gamescope leaves out of its node survives the HDR path. Same display-referred blend the CPU path's `composite_cursor_rgb10` already does — the samples are PQ, and a real sRGB→PQ cursor LUT is polish, not correctness for a pointer.
2026-07-28 18:03:30 +02:00
parent 689297c86e
commit 17f824c3e9
5 changed files with 224 additions and 52 deletions
@@ -8,8 +8,17 @@
 //
 // MODE (spec constant): 0 = packed 4-byte ARGB (NVENC byte order B,G,R,A), 1 = NV12 (Y plane +
 // interleaved half-res UV at row surfH), 2 = planar YUV444 (3 full-res planes stacked at
-// pitch*surfH). BT.709 limited-range coefficients — identical to rgb2nv12_buf.comp and the
-// retired .cu, so the cursor colour matches the frame regardless of backend.
+// pitch*surfH), 3 = packed 10-bit x:R:G:B 2:10:10:10 LE (NVENC ARGB10), 4 = packed 10-bit
+// x:B:G:R (NVENC ABGR10). BT.709 limited-range coefficients for the YUV modes — identical to
+// rgb2nv12_buf.comp and the retired .cu, so the cursor colour matches the frame regardless of
+// backend.
+//
+// The 10-bit modes are the HDR path: the surface holds PQ-encoded BT.2020 samples and NVENC does
+// the CSC itself, so the blend stays in RGB — it scales the 8-bit cursor to 10 bits and blends in
+// the destination's own (PQ) encoding. That is display-referred, i.e. an approximation, exactly
+// like the 8-bit gamma-space blend above and byte-for-byte what the CPU path's
+// `composite_cursor_rgb10` does. A real sRGB→PQ cursor LUT is polish, not correctness for a
+// pointer.
 //
 // The surface SSBO is uint[] (no 8-bit storage dependency — maximum driver reach): every
 // invocation exclusively owns the 32-bit words it read-modify-writes. ARGB: one invocation per
@@ -19,8 +28,10 @@
 // block rows are likewise anchored to the surface chroma grid (even rows), so each UV sample's
 // 2x2 footprint is exactly the luma rows it averages, at any `oy`.
 //
-// Rebuild: glslangValidator -V cursor_blend.comp -o cursor_blend.spv   (vendored beside this
-// file; or glslc — CI diffs the disassembly against this source)
+// Rebuild (vendored beside this file; CI diffs the disassembly against this source), either
+// compiler — target SPIR-V 1.0 to keep the driver reach the module was chosen for:
+//   glslc --target-env=vulkan1.0 cursor_blend.comp -o cursor_blend.spv
+//   glslangValidator -V --target-env vulkan1.0 cursor_blend.comp -o cursor_blend.spv

 layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in;

@@ -57,6 +68,22 @@ uint y_of(uvec4 s) {
 float u_of(uvec4 s) { return 128.0 - 0.1006 * float(s.r) - 0.3386 * float(s.g) + 0.4392 * float(s.b); }
 float v_of(uvec4 s) { return 128.0 + 0.4392 * float(s.r) - 0.3989 * float(s.g) - 0.0403 * float(s.b); }

+// Read-modify-write one packed-2:10:10:10 pixel. `r_shift` is R's bit offset (20 for x:R:G:B,
+// 0 for x:B:G:R); G is always at 10 and B sits at the opposite end from R. Matches
+// pw_cursor.rs::composite_cursor_rgb10 exactly, including the 8→10-bit expansion (replicate the
+// top 2 bits into the bottom) and the preservation of the top 2 (alpha/x) bits.
+void rmw_rgb10(uint idx, uvec4 s, uint r_shift) {
+    uint b_shift = 20u - r_shift;
+    uint w = surf[idx];
+    uint sr = (s.r << 2) | (s.r >> 6);
+    uint sg = (s.g << 2) | (s.g >> 6);
+    uint sb = (s.b << 2) | (s.b >> 6);
+    uint dr = (((w >> r_shift) & 0x3FFu) * (255u - s.a) + sr * s.a) / 255u;
+    uint dg = (((w >> 10u)      & 0x3FFu) * (255u - s.a) + sg * s.a) / 255u;
+    uint db = (((w >> b_shift) & 0x3FFu) * (255u - s.a) + sb * s.a) / 255u;
+    surf[idx] = (w & 0xC0000000u) | (dr << r_shift) | (dg << 10u) | (db << b_shift);
+}
+
 // Read-modify-write one byte lane of a word index.
 void rmw_byte(uint word_idx, uint lane, uint val8, uint a) {
    uint w = surf[word_idx];
@@ -67,6 +94,20 @@ void rmw_byte(uint word_idx, uint lane, uint val8, uint a) {
 }

 void main() {
+    if (MODE == 3u || MODE == 4u) {
+        // Packed 10-bit: same one-invocation-per-pixel/word geometry as MODE 0.
+        int cx = int(gl_GlobalInvocationID.x);
+        int cy = int(gl_GlobalInvocationID.y);
+        if (cx >= int(pc.curW) || cy >= int(pc.curH)) return;
+        int px = pc.ox + cx, py = pc.oy + cy;
+        if (px < 0 || py < 0 || px >= int(pc.surfW) || py >= int(pc.surfH)) return;
+        uvec4 s = cursor_px(cx, cy);
+        if (s.a == 0u) return;
+        uint idx = (uint(py) * pc.pitch + uint(px) * 4u) / 4u;
+        rmw_rgb10(idx, s, MODE == 3u ? 20u : 0u);
+        return;
+    }
+
    if (MODE == 0u) {
        // ARGB: one invocation per cursor pixel; each surface pixel is one exclusive word.
        int cx = int(gl_GlobalInvocationID.x);
@@ -44,6 +44,10 @@ use ash::vk;
 /// Max cursor-overlay bitmap edge (px) — matches [`cuda::CURSOR_MAX`] and the capture-side clamp.
 pub const CURSOR_MAX: u32 = cuda::CURSOR_MAX;

+/// Number of `cursor_blend.comp` MODE variants — one specialized pipeline each, indexed by
+/// [`SlotFormat::mode`]. Bump together with the shader's MODE list.
+const PIPELINE_MODES: u32 = 5;
+
 /// The vendored SPIR-V for `cursor_blend.comp` (beside this file; rebuild with
 /// `glslangValidator -V cursor_blend.comp -o cursor_blend.spv`; CI gates drift).
 const CURSOR_SPV: &[u8] = include_bytes!("cursor_blend.spv");
@@ -58,6 +62,13 @@ pub enum SlotFormat {
    Nv12,
    /// Planar YUV444: three full-res planes stacked at `pitch × height` intervals.
    Yuv444,
+    /// Packed 10-bit `x:R:G:B` 2:10:10:10 LE (NVENC `ARGB10`) — the HDR capture format, handed to
+    /// NVENC unconverted. Same 4-bytes-per-pixel geometry as [`Argb`](Self::Argb); it needs its
+    /// own mode only because the blend must unpack 10-bit channels instead of bytes.
+    X2Rgb10,
+    /// Packed 10-bit `x:B:G:R` 2:10:10:10 LE (NVENC `ABGR10`) — [`X2Rgb10`](Self::X2Rgb10) with
+    /// R and B swapped.
+    X2Bgr10,
 }

 impl SlotFormat {
@@ -66,19 +77,35 @@ impl SlotFormat {
            SlotFormat::Argb => 0,
            SlotFormat::Nv12 => 1,
            SlotFormat::Yuv444 => 2,
+            SlotFormat::X2Rgb10 => 3,
+            SlotFormat::X2Bgr10 => 4,
        }
    }
+    /// True for the layouts that are one 32-bit word per pixel — the same slot geometry AND the
+    /// same one-invocation-per-pixel dispatch, whatever the per-channel packing inside the word.
+    fn is_packed32(self) -> bool {
+        matches!(
+            self,
+            SlotFormat::Argb | SlotFormat::X2Rgb10 | SlotFormat::X2Bgr10
+        )
+    }
    fn row_bytes(self, width: u32) -> u64 {
+        if self.is_packed32() {
+            return width as u64 * 4;
+        }
        match self {
-            SlotFormat::Argb => width as u64 * 4,
            SlotFormat::Nv12 | SlotFormat::Yuv444 => width as u64,
+            _ => unreachable!("packed formats returned above"),
        }
    }
    fn rows(self, height: u32) -> u64 {
+        if self.is_packed32() {
+            return height as u64;
+        }
        match self {
-            SlotFormat::Argb => height as u64,
            SlotFormat::Nv12 => height as u64 + (height as u64 / 2).max(1),
            SlotFormat::Yuv444 => height as u64 * 3,
+            _ => unreachable!("packed formats returned above"),
        }
    }
 }
@@ -159,7 +186,7 @@ pub struct VkSlotBlend {
    pipe_layout: vk::PipelineLayout,
    desc_pool: vk::DescriptorPool,
    /// One pipeline per [`SlotFormat`], indexed by `mode()` (spec constant).
-    pipelines: [vk::Pipeline; 3],
+    pipelines: [vk::Pipeline; PIPELINE_MODES as usize],
    /// Host-visible cursor bitmap staging (CURSOR_MAX²·4, tight rows), persistently mapped.
    cur_buf: vk::Buffer,
    cur_mem: vk::DeviceMemory,
@@ -281,7 +308,7 @@ impl VkSlotBlend {
                desc_layout: vk::DescriptorSetLayout::null(),
                pipe_layout: vk::PipelineLayout::null(),
                desc_pool: vk::DescriptorPool::null(),
-                pipelines: [vk::Pipeline::null(); 3],
+                pipelines: [vk::Pipeline::null(); PIPELINE_MODES as usize],
                cur_buf: vk::Buffer::null(),
                cur_mem: vk::DeviceMemory::null(),
                cur_map: std::ptr::null_mut(),
@@ -470,7 +497,7 @@ impl VkSlotBlend {
            self.shader = d
                .create_shader_module(&vk::ShaderModuleCreateInfo::default().code(&words), None)
                .context("create blend shader module")?;
-            for mode in 0u32..3 {
+            for mode in 0u32..PIPELINE_MODES {
                let entries = [vk::SpecializationMapEntry::default()
                    .constant_id(0)
                    .offset(0)
@@ -768,24 +795,27 @@ impl VkSlotBlend {
            ox,
            oy,
        };
-        let (gx, gy) = match fmt {
-            SlotFormat::Argb => (cw.div_ceil(8), ch.div_ceil(8)),
-            _ => {
-                let x0 = (ox >> 2) << 2;
-                let spans = ((ox + cw as i32) - x0 + 3).div_euclid(4).max(1) as u32;
-                let rows = match fmt {
-                    SlotFormat::Nv12 => {
-                        // 2-row blocks anchored to the SURFACE chroma grid (cursor_blend.comp
-                        // derives the same y0): count the blocks covering luma rows
-                        // [oy, oy+ch) — one more than ch/2 when oy is odd.
-                        let first = oy.div_euclid(2);
-                        let last = (oy + ch as i32 - 1).div_euclid(2);
-                        (last - first + 1) as u32
-                    }
-                    _ => ch,
-                };
-                (spans.div_ceil(8), rows.div_ceil(8))
-            }
+        // `is_packed32`, not `== Argb`: the two 10-bit HDR formats are packed 32-bit words too, so
+        // they take the one-invocation-per-pixel arm exactly as ARGB does. Everything else is the
+        // word-aligned-span arm.
+        let (gx, gy) = if fmt.is_packed32() {
+            // One invocation per cursor pixel = one exclusively-owned 32-bit word.
+            (cw.div_ceil(8), ch.div_ceil(8))
+        } else {
+            let x0 = (ox >> 2) << 2;
+            let spans = ((ox + cw as i32) - x0 + 3).div_euclid(4).max(1) as u32;
+            let rows = match fmt {
+                SlotFormat::Nv12 => {
+                    // 2-row blocks anchored to the SURFACE chroma grid (cursor_blend.comp
+                    // derives the same y0): count the blocks covering luma rows
+                    // [oy, oy+ch) — one more than ch/2 when oy is odd.
+                    let first = oy.div_euclid(2);
+                    let last = (oy + ch as i32 - 1).div_euclid(2);
+                    (last - first + 1) as u32
+                }
+                _ => ch,
+            };
+            (spans.div_ceil(8), rows.div_ceil(8))
        };
        Some((push, gx, gy))
    }