feat(encode): raw Vulkan Video HEVC backend on Linux (AMD/Intel) with real RFI

Add `VulkanVideoEncoder` (`VK_KHR_video_encode_h265` via ash) — the open-stack twin of the direct-NVENC RFI path, giving AMD/Intel Linux hosts real reference-frame invalidation loss recovery: a clean P-frame recovery anchor that re-references a known-good older frame instead of a full IDR. The app owns the DPB, so recovery = pointing the P-frame's single L0 reference at a resident slot strictly older than the loss (never a concealed frame). The backend owns its own ash instance/device with encode + compute queues, authors VPS/SPS/PPS (Main, conformance-window crop for non-16-aligned heights like 1080->1088), runs a DPB-ring reference-slot state machine with monotonic POC and CBR rate control, and does an on-GPU RGB->NV12 BT.709 compute CSC (embedded rgb2yuv.spv) since capture delivers packed-RGB dmabufs — importing each frame's dmabuf (explicit DRM modifier) or uploading a CPU-RGB fallback, CSC on the compute queue, then encode on the encode queue, ordered by a semaphore. Wired into `open_video_backend`: an AMD/Intel HEVC session opens this instead of libav VAAPI when `PUNKTFUNK_VULKAN_ENCODE=1` (VAAPI fallback on any open error, so it can only improve recovery, never break a stream); `PUNKTFUNK_ ENCODER=vulkan` forces it. Gated behind the new `vulkan-encode` Cargo feature, which pulls no new dependency (reuses the `ash` bindings already carried for the dmabuf zero-copy bridge). Opt-in until on-glass validated, mirroring how the direct-NVENC path shipped. Headless-validated on real RADV (RDNA3 780M, Mesa 26): open + multi-frame encode + `invalidate_ref_frames` all run through the real struct and ffmpeg decodes the output `I P P P P P` with 0 errors; the recovery frame is a clean P-frame (not an IDR); and dropping the "lost" AU still decodes cleanly because the recovery re-anchored to an older frame — the RFI heal, proven on real hardware. `cargo check`/`clippy -D warnings` green with the feature on and off. Design: design/linux-vulkan-video-encode.md. Harness: design/vkenc-probe-harness/. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-12 16:59:27 +02:00
parent 76594f27c1
commit 5ab6daa694
5 changed files with 1851 additions and 4 deletions
@@ -435,6 +435,7 @@ fn resolved_backend_label(cuda: bool) -> &'static str {
    match crate::config::config().encoder_pref.as_str() {
        "nvenc" | "nvidia" | "cuda" => "nvenc",
        "vaapi" | "amd" | "intel" => "vaapi",
+        "vulkan" | "vulkan-video" => "vulkan",
        "software" | "sw" | "openh264" => "software",
        _ => {
            if cuda || !linux_auto_is_vaapi() {
@@ -528,7 +529,29 @@ fn open_video_backend(
        // Linux binary serves any GPU; `PUNKTFUNK_ENCODER` forces a specific backend (and surfaces
        // its errors crisply instead of silently trying the other).
        let pref = crate::config::config().encoder_pref.as_str();
-        let open_vaapi = || -> Result<Box<dyn Encoder>> {
+        // AMD/Intel opener. Default = libav VAAPI. With `--features vulkan-encode` +
+        // PUNKTFUNK_VULKAN_ENCODE, an HEVC session instead opens the raw Vulkan Video backend (real
+        // RFI loss recovery the VAAPI path can't express); a failed open falls back to VAAPI so the
+        // stream never dies over the new path. `format`/`bit_depth`/`chroma` only matter to VAAPI —
+        // the Vulkan backend imports the dmabuf and does its own 8-bit 4:2:0 CSC.
+        let open_amd_intel = || -> Result<Box<dyn Encoder>> {
+            #[cfg(feature = "vulkan-encode")]
+            if codec == Codec::H265 && vulkan_encode_enabled() {
+                match vulkan_video::VulkanVideoEncoder::open(codec, width, height, fps, bitrate_bps)
+                {
+                    Ok(e) => {
+                        tracing::info!(
+                            "Linux Vulkan Video HEVC encode (real RFI via DPB reference slots) — \
+                             set PUNKTFUNK_VULKAN_ENCODE=0 for libav VAAPI"
+                        );
+                        return Ok(Box::new(e) as Box<dyn Encoder>);
+                    }
+                    Err(e) => tracing::warn!(
+                        error = %format!("{e:#}"),
+                        "Vulkan Video encode open failed — falling back to libav VAAPI"
+                    ),
+                }
+            }
            vaapi::VaapiEncoder::open(
                codec,
                format,
@@ -553,7 +576,27 @@ fn open_video_backend(
                bit_depth,
                chroma,
            ),
-            "vaapi" | "amd" | "intel" => open_vaapi(),
+            "vaapi" | "amd" | "intel" => open_amd_intel(),
+            // Force the raw Vulkan Video HEVC backend (real RFI). Needs `--features vulkan-encode`.
+            "vulkan" | "vulkan-video" => {
+                #[cfg(feature = "vulkan-encode")]
+                {
+                    if codec != Codec::H265 {
+                        anyhow::bail!(
+                            "the Vulkan Video encoder is HEVC-only; the session negotiated {codec:?}"
+                        );
+                    }
+                    vulkan_video::VulkanVideoEncoder::open(codec, width, height, fps, bitrate_bps)
+                        .map(|e| Box::new(e) as Box<dyn Encoder>)
+                }
+                #[cfg(not(feature = "vulkan-encode"))]
+                {
+                    let _ = (format, bit_depth, chroma);
+                    anyhow::bail!(
+                        "PUNKTFUNK_ENCODER=vulkan requires a build with --features vulkan-encode"
+                    )
+                }
+            }
            // GPU-less software H.264 (openh264) — for a headless / GPU-lost box. Explicit-only:
            // `auto` never picks it (a box with `/dev/nvidiactl` present but a dead driver would
            // otherwise wrongly resolve to NVENC). Needs H.264 (openh264 emits only that) and a CPU
@@ -586,11 +629,11 @@ fn open_video_backend(
                        chroma,
                    )
                } else {
-                    open_vaapi()
+                    open_amd_intel()
                }
            }
            other => anyhow::bail!(
-                "unknown PUNKTFUNK_ENCODER={other:?} — use auto (default), nvenc, vaapi, or software"
+                "unknown PUNKTFUNK_ENCODER={other:?} — use auto (default), nvenc, vaapi, vulkan, or software"
            ),
        }
    }
@@ -830,6 +873,20 @@ fn nvenc_direct_enabled() -> bool {
        .unwrap_or(true)
 }

+/// Whether the raw Vulkan Video HEVC encode backend is active for AMD/Intel. **Opt-in for now**
+/// (design/linux-vulkan-video-encode.md) — `PUNKTFUNK_VULKAN_ENCODE=1` (also `true`/`yes`/`on`)
+/// selects it over libav VAAPI for an HEVC session; it gives real reference-frame invalidation
+/// (a clean P-frame recovery anchor via explicit DPB reference slots) that the libavcodec VAAPI
+/// path can't express. Will flip to default-on after on-glass validation, like
+/// [`nvenc_direct_enabled`]. Only consulted with `--features vulkan-encode`; a failed open falls
+/// back to VAAPI, so this can only improve recovery, never break a stream.
+#[cfg(all(target_os = "linux", feature = "vulkan-encode"))]
+fn vulkan_encode_enabled() -> bool {
+    std::env::var("PUNKTFUNK_VULKAN_ENCODE")
+        .map(|v| matches!(v.trim(), "1" | "true" | "yes" | "on"))
+        .unwrap_or(false)
+}
+
 /// Cheap, side-effect-free NVIDIA-presence probe for the `auto` backend selector: the NVIDIA
 /// kernel driver exposes these device nodes, AMD/Intel boxes have neither. Deliberately does NOT
 /// create a CUDA context (that would allocate GPU state on every host that merely *might* be
@@ -1185,6 +1242,13 @@ mod sw;
 #[cfg(target_os = "linux")]
 #[path = "encode/linux/vaapi.rs"]
 mod vaapi;
+// Raw Vulkan Video HEVC encode on Linux (AMD/Intel; design/linux-vulkan-video-encode.md) — real RFI
+// via explicit DPB reference slots (the app owns the DPB), the open-stack twin of the direct-NVENC
+// path. Does an on-GPU RGB→NV12 compute CSC since capture delivers packed-RGB dmabufs. Opt-in behind
+// `PUNKTFUNK_VULKAN_ENCODE` until on-glass validated; needs `--features vulkan-encode`.
+#[cfg(all(target_os = "linux", feature = "vulkan-encode"))]
+#[path = "encode/linux/vulkan_video.rs"]
+mod vulkan_video;

 #[cfg(test)]
 mod tests {
@@ -0,0 +1,27 @@
+#version 450
+// RGB(A) -> NV12 (BT.709 limited range). One invocation per chroma sample = 2x2 luma block.
+layout(local_size_x = 8, local_size_y = 8) in;
+layout(binding = 0) uniform sampler2D rgb;                  // packed RGB input (sampled; BGRA import ok)
+layout(binding = 1, r8)  uniform writeonly image2D yImg;    // full-res Y
+layout(binding = 2, rg8) uniform writeonly image2D uvImg;   // half-res UV (interleaved)
+
+float lumaY(vec3 c) { return 16.0/255.0 + 0.1826*c.r + 0.6142*c.g + 0.0620*c.b; }
+
+void main() {
+    ivec2 sz = imageSize(yImg);
+    ivec2 uvc = ivec2(gl_GlobalInvocationID.xy);
+    ivec2 p = uvc * 2;
+    if (p.x >= sz.x || p.y >= sz.y) return;
+    vec3 c00 = texelFetch(rgb, p, 0).rgb;
+    vec3 c10 = texelFetch(rgb, p + ivec2(1, 0), 0).rgb;
+    vec3 c01 = texelFetch(rgb, p + ivec2(0, 1), 0).rgb;
+    vec3 c11 = texelFetch(rgb, p + ivec2(1, 1), 0).rgb;
+    imageStore(yImg, p,               vec4(lumaY(c00), 0, 0, 1));
+    imageStore(yImg, p + ivec2(1, 0), vec4(lumaY(c10), 0, 0, 1));
+    imageStore(yImg, p + ivec2(0, 1), vec4(lumaY(c01), 0, 0, 1));
+    imageStore(yImg, p + ivec2(1, 1), vec4(lumaY(c11), 0, 0, 1));
+    vec3 a = (c00 + c10 + c01 + c11) * 0.25;
+    float U = 128.0/255.0 - 0.1006*a.r - 0.3386*a.g + 0.4392*a.b;
+    float V = 128.0/255.0 + 0.4392*a.r - 0.3989*a.g - 0.0403*a.b;
+    imageStore(uvImg, uvc, vec4(U, V, 0, 1));
+}