feat(encode): make cursor blending a queryable capability, not an assumption

`open_video`'s `cursor_blend` argument was a request with no answer: lib.rs did
`let _ = cursor_blend;` and only three backends ever read `CapturedFrame::cursor`.
So a session could ask for a composited pointer, get a backend that silently
discards it, and stream with no mouse cursor and nothing in the logs. Two
separately-confirmed audit findings — the VAAPI dmabuf path and the libav-NVENC CUDA
path — are symptoms of that one hole.

`EncoderCaps::blends_cursor` makes it a fact each backend states. The four exhaustive
`EncoderCaps { .. }` constructors mean adding the field is a compile error until every
backend answers, which is the enforcement mechanism for future backends rather than a
side effect. Vulkan Video answers from its ACTUAL configured source rather than
statically: only the CSC path composites (`prep_cursor` feeds the compute shader),
while the RGB-direct/EFC front-end and the native-NV12 source have no compositing
stage at all and merely warn once that the pointer is being dropped.

`open_video` warns when a session asked for blending and the opened backend cannot
deliver it. A warning is deliberately all it does: `open_video` cannot re-plan
capture, so refusing would trade a missing pointer for a dead session. The host owns
`plan.cursor_blend` and is the only layer that can fall back to capturer-side
compositing — this gives it something to base that on.

Enforcement is NOT included. The reviewed design proposed refusing the client's
host-composite flip to keep the client drawing its own pointer, but `CursorRenderMode`
is client->host only: there is no host->client counterpart, so refusing yields no
pointer at all — the same failure it claimed to prevent.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-25 02:41:57 +02:00
co-authored by Claude Opus 5
parent ab63e0dad3
commit ef239691df
9 changed files with 48 additions and 0 deletions
+13
View File
@@ -294,6 +294,19 @@ pub struct EncoderCaps {
/// `USER_FLAG_RECOVERY_POINT` on every Nth emitted AU, re-phased at each IDR). 0 when intra-refresh
/// is off. Only consulted when [`intra_refresh_recovery`](Self::intra_refresh_recovery) is set.
pub intra_refresh_period: u32,
/// The encoder composites [`CapturedFrame::cursor`] into the picture it encodes.
///
/// `open_video`'s `cursor_blend` argument is a REQUEST, and for most of this crate's life it was
/// nothing else: `lib.rs` literally did `let _ = cursor_blend;` and only three backends ever read
/// `frame.cursor`. So a session could ask for a composited pointer, get a backend that silently
/// discards it, and stream with no mouse cursor at all — the confirmed symptom on the VAAPI
/// dmabuf path and the libav-NVENC CUDA path.
///
/// This makes the answer queryable instead of assumed. It is deliberately a plain fact about the
/// encoder, not a policy: what to DO when a session wants blending and the backend cannot is the
/// host's call, since only the host can re-plan capture (fall back to capturer-side compositing).
/// `open_video` can only warn, which it does.
pub blends_cursor: bool,
}
/// A hardware encoder. One per session; runs on the encode thread.
+3
View File
@@ -620,6 +620,9 @@ impl NvencEncoder {
impl Encoder for NvencEncoder {
fn caps(&self) -> super::EncoderCaps {
super::EncoderCaps {
// libav NVENC hands the frame straight to the encoder — `frame.cursor` is never read,
// so a cursor-as-metadata session loses its pointer on this backend (audit finding).
blends_cursor: false,
// 4:4:4 iff this session opened FREXT — the CPU swscale path or the zero-copy GPU
// convert. RFI/HDR-SEI stay unsupported on libavcodec NVENC (the trait defaults).
chroma_444: self.want_444,
@@ -1840,6 +1840,8 @@ impl Encoder for NvencCudaEncoder {
fn caps(&self) -> EncoderCaps {
EncoderCaps {
// Composites `frame.cursor` via the SPIR-V blend over the Vulkan-allocated input slot.
blends_cursor: true,
supports_rfi: self.rfi_supported,
supports_hdr_metadata: self.hdr,
chroma_444: self.chroma_444,
@@ -1209,6 +1209,8 @@ impl Encoder for PyroWaveEncoder {
// hardcoded `default()` here mis-reports a 4:4:4 open as 4:2:0 and fires a spurious
// "chroma disagrees with the negotiated Welcome" warn.
EncoderCaps {
// The wavelet CSC composites the metadata cursor.
blends_cursor: true,
chroma_444: self.chroma444,
..EncoderCaps::default()
}
@@ -3308,6 +3308,11 @@ impl Encoder for VulkanVideoEncoder {
fn caps(&self) -> EncoderCaps {
EncoderCaps {
supports_rfi: true,
// Only the CSC path composites the metadata cursor (`prep_cursor` feeds the compute
// shader). The RGB-direct/EFC front-end and the native-NV12 source have no compositing
// stage at all — both merely warn once that the pointer is being dropped — so this is
// the encoder's real answer, not a static one.
blends_cursor: self.rgb.is_none() && !self.native_nv12,
..Default::default()
}
}
+2
View File
@@ -2426,6 +2426,8 @@ impl Encoder for AmfEncoder {
fn caps(&self) -> EncoderCaps {
EncoderCaps {
// As Windows NVENC: the capturer composites; this backend never reads `frame.cursor`.
blends_cursor: false,
// LTR-RFI: AMD's reference invalidation is the user long-term-reference path (mark a
// frame, force a later one to re-reference it). True only when the live driver accepted
// the LTR slots at open — otherwise loss recovery falls back to a full IDR.
@@ -681,6 +681,8 @@ impl Encoder for PyroWaveEncoder {
// after the caps() default was written — a hardcoded `default()` here mis-reports a 4:4:4
// open as 4:2:0 and fires a spurious "chroma disagrees with the negotiated Welcome" warn).
EncoderCaps {
// The Windows capturer composites the pointer itself; this backend never reads it.
blends_cursor: false,
chroma_444: self.chroma444,
..EncoderCaps::default()
}
+2
View File
@@ -1407,6 +1407,8 @@ impl Encoder for QsvEncoder {
fn caps(&self) -> EncoderCaps {
EncoderCaps {
// As Windows NVENC: the capturer composites; this backend never reads `frame.cursor`.
blends_cursor: false,
supports_rfi: self.ltr_active,
// In-band mastering/CLL at IDR (HEVC prefix SEI / AV1 metadata OBU); AVC sessions
// are never HDR.
+17
View File
@@ -177,6 +177,23 @@ pub fn open_video(
},
}
};
// The session asked for a composited pointer; say so loudly if the backend that actually opened
// cannot deliver one. `cursor_blend` was a REQUEST with no answer for most of this crate's life
// (`let _ = cursor_blend;` below), and the result was a stream with no mouse cursor and nothing
// in the logs — confirmed on the VAAPI dmabuf path and the libav-NVENC CUDA path.
//
// A warning is deliberately all this does. `open_video` cannot re-plan capture, so refusing here
// would only trade a missing pointer for a dead session; the host owns `plan.cursor_blend` and is
// the only layer that can fall back to capturer-side compositing. This makes the condition
// visible and queryable (`EncoderCaps::blends_cursor`) so that decision can be made upstream.
if cursor_blend && !inner.caps().blends_cursor {
tracing::warn!(
backend,
"session negotiated a composited cursor but this encode backend does not blend \
CapturedFrame::cursor the pointer will be MISSING from the stream unless the \
capturer composites it"
);
}
Ok(Box::new(TrackedEncoder {
inner,
_session: pf_gpu::session_begin(gpu),