From ef239691df9d2f9f81b34beded65d8c1a7f73bdd Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Sat, 25 Jul 2026 02:37:05 +0200 Subject: [PATCH] feat(encode): make cursor blending a queryable capability, not an assumption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `open_video`'s `cursor_blend` argument was a request with no answer: lib.rs did `let _ = cursor_blend;` and only three backends ever read `CapturedFrame::cursor`. So a session could ask for a composited pointer, get a backend that silently discards it, and stream with no mouse cursor and nothing in the logs. Two separately-confirmed audit findings — the VAAPI dmabuf path and the libav-NVENC CUDA path — are symptoms of that one hole. `EncoderCaps::blends_cursor` makes it a fact each backend states. The four exhaustive `EncoderCaps { .. }` constructors mean adding the field is a compile error until every backend answers, which is the enforcement mechanism for future backends rather than a side effect. Vulkan Video answers from its ACTUAL configured source rather than statically: only the CSC path composites (`prep_cursor` feeds the compute shader), while the RGB-direct/EFC front-end and the native-NV12 source have no compositing stage at all and merely warn once that the pointer is being dropped. `open_video` warns when a session asked for blending and the opened backend cannot deliver it. A warning is deliberately all it does: `open_video` cannot re-plan capture, so refusing would trade a missing pointer for a dead session. The host owns `plan.cursor_blend` and is the only layer that can fall back to capturer-side compositing — this gives it something to base that on. Enforcement is NOT included. The reviewed design proposed refusing the client's host-composite flip to keep the client drawing its own pointer, but `CursorRenderMode` is client->host only: there is no host->client counterpart, so refusing yields no pointer at all — the same failure it claimed to prevent. Co-Authored-By: Claude Opus 5 (1M context) --- crates/pf-encode/src/enc/codec.rs | 13 +++++++++++++ crates/pf-encode/src/enc/linux/mod.rs | 3 +++ crates/pf-encode/src/enc/linux/nvenc_cuda.rs | 2 ++ crates/pf-encode/src/enc/linux/pyrowave.rs | 2 ++ crates/pf-encode/src/enc/linux/vulkan_video.rs | 5 +++++ crates/pf-encode/src/enc/windows/amf.rs | 2 ++ crates/pf-encode/src/enc/windows/pyrowave.rs | 2 ++ crates/pf-encode/src/enc/windows/qsv.rs | 2 ++ crates/pf-encode/src/lib.rs | 17 +++++++++++++++++ 9 files changed, 48 insertions(+) diff --git a/crates/pf-encode/src/enc/codec.rs b/crates/pf-encode/src/enc/codec.rs index 1b8297b4..a66dad86 100644 --- a/crates/pf-encode/src/enc/codec.rs +++ b/crates/pf-encode/src/enc/codec.rs @@ -294,6 +294,19 @@ pub struct EncoderCaps { /// `USER_FLAG_RECOVERY_POINT` on every Nth emitted AU, re-phased at each IDR). 0 when intra-refresh /// is off. Only consulted when [`intra_refresh_recovery`](Self::intra_refresh_recovery) is set. pub intra_refresh_period: u32, + /// The encoder composites [`CapturedFrame::cursor`] into the picture it encodes. + /// + /// `open_video`'s `cursor_blend` argument is a REQUEST, and for most of this crate's life it was + /// nothing else: `lib.rs` literally did `let _ = cursor_blend;` and only three backends ever read + /// `frame.cursor`. So a session could ask for a composited pointer, get a backend that silently + /// discards it, and stream with no mouse cursor at all — the confirmed symptom on the VAAPI + /// dmabuf path and the libav-NVENC CUDA path. + /// + /// This makes the answer queryable instead of assumed. It is deliberately a plain fact about the + /// encoder, not a policy: what to DO when a session wants blending and the backend cannot is the + /// host's call, since only the host can re-plan capture (fall back to capturer-side compositing). + /// `open_video` can only warn, which it does. + pub blends_cursor: bool, } /// A hardware encoder. One per session; runs on the encode thread. diff --git a/crates/pf-encode/src/enc/linux/mod.rs b/crates/pf-encode/src/enc/linux/mod.rs index 929c2d2d..21abd638 100644 --- a/crates/pf-encode/src/enc/linux/mod.rs +++ b/crates/pf-encode/src/enc/linux/mod.rs @@ -620,6 +620,9 @@ impl NvencEncoder { impl Encoder for NvencEncoder { fn caps(&self) -> super::EncoderCaps { super::EncoderCaps { + // libav NVENC hands the frame straight to the encoder — `frame.cursor` is never read, + // so a cursor-as-metadata session loses its pointer on this backend (audit finding). + blends_cursor: false, // 4:4:4 iff this session opened FREXT — the CPU swscale path or the zero-copy GPU // convert. RFI/HDR-SEI stay unsupported on libavcodec NVENC (the trait defaults). chroma_444: self.want_444, diff --git a/crates/pf-encode/src/enc/linux/nvenc_cuda.rs b/crates/pf-encode/src/enc/linux/nvenc_cuda.rs index a7011e6e..64983def 100644 --- a/crates/pf-encode/src/enc/linux/nvenc_cuda.rs +++ b/crates/pf-encode/src/enc/linux/nvenc_cuda.rs @@ -1840,6 +1840,8 @@ impl Encoder for NvencCudaEncoder { fn caps(&self) -> EncoderCaps { EncoderCaps { + // Composites `frame.cursor` via the SPIR-V blend over the Vulkan-allocated input slot. + blends_cursor: true, supports_rfi: self.rfi_supported, supports_hdr_metadata: self.hdr, chroma_444: self.chroma_444, diff --git a/crates/pf-encode/src/enc/linux/pyrowave.rs b/crates/pf-encode/src/enc/linux/pyrowave.rs index 00e6777f..20d264de 100644 --- a/crates/pf-encode/src/enc/linux/pyrowave.rs +++ b/crates/pf-encode/src/enc/linux/pyrowave.rs @@ -1209,6 +1209,8 @@ impl Encoder for PyroWaveEncoder { // hardcoded `default()` here mis-reports a 4:4:4 open as 4:2:0 and fires a spurious // "chroma disagrees with the negotiated Welcome" warn. EncoderCaps { + // The wavelet CSC composites the metadata cursor. + blends_cursor: true, chroma_444: self.chroma444, ..EncoderCaps::default() } diff --git a/crates/pf-encode/src/enc/linux/vulkan_video.rs b/crates/pf-encode/src/enc/linux/vulkan_video.rs index 06e779b0..28753560 100644 --- a/crates/pf-encode/src/enc/linux/vulkan_video.rs +++ b/crates/pf-encode/src/enc/linux/vulkan_video.rs @@ -3308,6 +3308,11 @@ impl Encoder for VulkanVideoEncoder { fn caps(&self) -> EncoderCaps { EncoderCaps { supports_rfi: true, + // Only the CSC path composites the metadata cursor (`prep_cursor` feeds the compute + // shader). The RGB-direct/EFC front-end and the native-NV12 source have no compositing + // stage at all — both merely warn once that the pointer is being dropped — so this is + // the encoder's real answer, not a static one. + blends_cursor: self.rgb.is_none() && !self.native_nv12, ..Default::default() } } diff --git a/crates/pf-encode/src/enc/windows/amf.rs b/crates/pf-encode/src/enc/windows/amf.rs index ec420dc3..9def7d26 100644 --- a/crates/pf-encode/src/enc/windows/amf.rs +++ b/crates/pf-encode/src/enc/windows/amf.rs @@ -2426,6 +2426,8 @@ impl Encoder for AmfEncoder { fn caps(&self) -> EncoderCaps { EncoderCaps { + // As Windows NVENC: the capturer composites; this backend never reads `frame.cursor`. + blends_cursor: false, // LTR-RFI: AMD's reference invalidation is the user long-term-reference path (mark a // frame, force a later one to re-reference it). True only when the live driver accepted // the LTR slots at open — otherwise loss recovery falls back to a full IDR. diff --git a/crates/pf-encode/src/enc/windows/pyrowave.rs b/crates/pf-encode/src/enc/windows/pyrowave.rs index 249ebacc..ffaa754a 100644 --- a/crates/pf-encode/src/enc/windows/pyrowave.rs +++ b/crates/pf-encode/src/enc/windows/pyrowave.rs @@ -681,6 +681,8 @@ impl Encoder for PyroWaveEncoder { // after the caps() default was written — a hardcoded `default()` here mis-reports a 4:4:4 // open as 4:2:0 and fires a spurious "chroma disagrees with the negotiated Welcome" warn). EncoderCaps { + // The Windows capturer composites the pointer itself; this backend never reads it. + blends_cursor: false, chroma_444: self.chroma444, ..EncoderCaps::default() } diff --git a/crates/pf-encode/src/enc/windows/qsv.rs b/crates/pf-encode/src/enc/windows/qsv.rs index f70a3925..c2315b90 100644 --- a/crates/pf-encode/src/enc/windows/qsv.rs +++ b/crates/pf-encode/src/enc/windows/qsv.rs @@ -1407,6 +1407,8 @@ impl Encoder for QsvEncoder { fn caps(&self) -> EncoderCaps { EncoderCaps { + // As Windows NVENC: the capturer composites; this backend never reads `frame.cursor`. + blends_cursor: false, supports_rfi: self.ltr_active, // In-band mastering/CLL at IDR (HEVC prefix SEI / AV1 metadata OBU); AVC sessions // are never HDR. diff --git a/crates/pf-encode/src/lib.rs b/crates/pf-encode/src/lib.rs index 689b6f7e..bdca9e21 100644 --- a/crates/pf-encode/src/lib.rs +++ b/crates/pf-encode/src/lib.rs @@ -177,6 +177,23 @@ pub fn open_video( }, } }; + // The session asked for a composited pointer; say so loudly if the backend that actually opened + // cannot deliver one. `cursor_blend` was a REQUEST with no answer for most of this crate's life + // (`let _ = cursor_blend;` below), and the result was a stream with no mouse cursor and nothing + // in the logs — confirmed on the VAAPI dmabuf path and the libav-NVENC CUDA path. + // + // A warning is deliberately all this does. `open_video` cannot re-plan capture, so refusing here + // would only trade a missing pointer for a dead session; the host owns `plan.cursor_blend` and is + // the only layer that can fall back to capturer-side compositing. This makes the condition + // visible and queryable (`EncoderCaps::blends_cursor`) so that decision can be made upstream. + if cursor_blend && !inner.caps().blends_cursor { + tracing::warn!( + backend, + "session negotiated a composited cursor but this encode backend does not blend \ + CapturedFrame::cursor — the pointer will be MISSING from the stream unless the \ + capturer composites it" + ); + } Ok(Box::new(TrackedEncoder { inner, _session: pf_gpu::session_begin(gpu),