From 6dc195f98257c9a2517186dba4837ec1830ab438 Mon Sep 17 00:00:00 2001
From: enricobuehler <enrico@buehler.earth>
Date: Mon, 20 Jul 2026 18:48:26 +0200
Subject: [PATCH] fix(encode): Vulkan Video poll() must block per the depth-1
 pump contract
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The session pump's depth-1 loop (capture → submit → poll) treats a poll()
None as "the backend holds the frame internally, re-poll next tick" —
true only of the libav AMF/QSV wrappers, whose codecs genuinely hold ~2
frames. The Vulkan Video backend probed its slot fence with
get_fence_status instead, so the AU of every frame — finished on the
ASIC in ~5 ms — sat unharvested until the tick AFTER the next frame was
submitted: encode_us read as one full frame period (~17 ms at 60 Hz vs
VAAPI's ~5.3 ms on the same VCN, the AMD field report) and every frame
shipped a frame period late — one frame of avoidable glass-to-glass
latency on the path that exists to beat libav VAAPI.

Wait the oldest in-flight slot's fence in poll() instead, matching the
sync NVENC backend's blocking lock_bitstream and the documented
Capturer::pipeline_depth contract ("capture → submit → poll-blocks").
Bounded by ENCODE_FENCE_TIMEOUT_NS like enqueue()'s backpressure wait,
for the same reason: poll runs on the thread the stall recovery runs on,
so a wedged GPU must surface as an error, not park it. None now only
means "nothing submitted". Covers H265 and AV1 (shared poll).

Verified: cargo check + DPB/RPS unit tests green (home-worker-5); the
GPU smoke tests fail at session open on that box's NVIDIA driver
identically on unmodified origin/main — pre-existing, needs the RADV
box for on-glass.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../pf-encode/src/enc/linux/vulkan_video.rs   | 28 +++++++++++++++----
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/crates/pf-encode/src/enc/linux/vulkan_video.rs b/crates/pf-encode/src/enc/linux/vulkan_video.rs
index 3c0f8588..c078b1e9 100644
--- a/crates/pf-encode/src/enc/linux/vulkan_video.rs
+++ b/crates/pf-encode/src/enc/linux/vulkan_video.rs
@@ -2065,18 +2065,34 @@ impl Encoder for VulkanVideoEncoder {
 
     fn poll(&mut self) -> Result<Option<EncodedFrame>> {
         // Backpressure-drained frames (already read, oldest) come out first, then the oldest slot
-        // still in flight once its fence signals — both in submission order. Non-blocking: an
-        // unfinished frame returns None so the caller keeps the pipeline moving.
+        // still in flight — both in submission order. BLOCKING, per the depth-1 pump contract
+        // (`Capturer::pipeline_depth`: "capture → submit → poll-blocks", the convention the sync
+        // NVENC backend's `lock_bitstream` follows): the pump polls right after submit and treats
+        // `None` as "the backend holds the frame internally, re-poll next tick" (true only of the
+        // libav AMF/QSV wrappers). A `get_fence_status` probe here therefore deferred every AU to
+        // the NEXT tick — shipped a full frame period after the ASIC finished, so a ~5 ms VCN
+        // encode read as `encode_us`≈interval (~17 ms at 60 Hz, the AMD field report) and cost one
+        // frame of avoidable glass-to-glass latency. `None` now only means "nothing submitted".
         if let Some(f) = self.pending.pop_front() {
             return Ok(Some(f));
         }
         let Some(&slot) = self.in_flight.front() else {
             return Ok(None);
         };
-        // SAFETY: probing a fence + reading back this slot's own owned objects under `&mut self`.
-        let ready = unsafe { self.device.get_fence_status(self.frames[slot].fence)? };
-        if !ready {
-            return Ok(None);
+        // Bounded like `enqueue`'s backpressure wait, and for the same reason: this is the thread
+        // the stall recovery runs on, so a wedged GPU must surface as an error, not park it.
+        // SAFETY: waiting a fence owned by this encoder's slot under `&mut self`.
+        match unsafe {
+            self.device
+                .wait_for_fences(&[self.frames[slot].fence], true, ENCODE_FENCE_TIMEOUT_NS)
+        } {
+            Ok(()) => {}
+            Err(vk::Result::TIMEOUT) => anyhow::bail!(
+                "vulkan-encode: fence for slot {slot} did not signal within {} ms — GPU or \
+                 driver wedged; failing the poll so the session can reset",
+                ENCODE_FENCE_TIMEOUT_NS / 1_000_000
+            ),
+            Err(e) => return Err(e.into()),
         }
         self.in_flight.pop_front();
         // SAFETY: fence signaled ⟹ this slot's CSC+encode is complete; read its bitstream.