feat(encode): make cursor blending a queryable capability, not an assumption
`open_video`'s `cursor_blend` argument was a request with no answer: lib.rs did
`let _ = cursor_blend;` and only three backends ever read `CapturedFrame::cursor`.
So a session could ask for a composited pointer, get a backend that silently
discards it, and stream with no mouse cursor and nothing in the logs. Two
separately-confirmed audit findings — the VAAPI dmabuf path and the libav-NVENC CUDA
path — are symptoms of that one hole.
`EncoderCaps::blends_cursor` makes it a fact each backend states. The four exhaustive
`EncoderCaps { .. }` constructors mean adding the field is a compile error until every
backend answers, which is the enforcement mechanism for future backends rather than a
side effect. Vulkan Video answers from its ACTUAL configured source rather than
statically: only the CSC path composites (`prep_cursor` feeds the compute shader),
while the RGB-direct/EFC front-end and the native-NV12 source have no compositing
stage at all and merely warn once that the pointer is being dropped.
`open_video` warns when a session asked for blending and the opened backend cannot
deliver it. A warning is deliberately all it does: `open_video` cannot re-plan
capture, so refusing would trade a missing pointer for a dead session. The host owns
`plan.cursor_blend` and is the only layer that can fall back to capturer-side
compositing — this gives it something to base that on.
Enforcement is NOT included. The reviewed design proposed refusing the client's
host-composite flip to keep the client drawing its own pointer, but `CursorRenderMode`
is client->host only: there is no host->client counterpart, so refusing yields no
pointer at all — the same failure it claimed to prevent.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -294,6 +294,19 @@ pub struct EncoderCaps {
|
||||
/// `USER_FLAG_RECOVERY_POINT` on every Nth emitted AU, re-phased at each IDR). 0 when intra-refresh
|
||||
/// is off. Only consulted when [`intra_refresh_recovery`](Self::intra_refresh_recovery) is set.
|
||||
pub intra_refresh_period: u32,
|
||||
/// The encoder composites [`CapturedFrame::cursor`] into the picture it encodes.
|
||||
///
|
||||
/// `open_video`'s `cursor_blend` argument is a REQUEST, and for most of this crate's life it was
|
||||
/// nothing else: `lib.rs` literally did `let _ = cursor_blend;` and only three backends ever read
|
||||
/// `frame.cursor`. So a session could ask for a composited pointer, get a backend that silently
|
||||
/// discards it, and stream with no mouse cursor at all — the confirmed symptom on the VAAPI
|
||||
/// dmabuf path and the libav-NVENC CUDA path.
|
||||
///
|
||||
/// This makes the answer queryable instead of assumed. It is deliberately a plain fact about the
|
||||
/// encoder, not a policy: what to DO when a session wants blending and the backend cannot is the
|
||||
/// host's call, since only the host can re-plan capture (fall back to capturer-side compositing).
|
||||
/// `open_video` can only warn, which it does.
|
||||
pub blends_cursor: bool,
|
||||
}
|
||||
|
||||
/// A hardware encoder. One per session; runs on the encode thread.
|
||||
|
||||
@@ -620,6 +620,9 @@ impl NvencEncoder {
|
||||
impl Encoder for NvencEncoder {
|
||||
fn caps(&self) -> super::EncoderCaps {
|
||||
super::EncoderCaps {
|
||||
// libav NVENC hands the frame straight to the encoder — `frame.cursor` is never read,
|
||||
// so a cursor-as-metadata session loses its pointer on this backend (audit finding).
|
||||
blends_cursor: false,
|
||||
// 4:4:4 iff this session opened FREXT — the CPU swscale path or the zero-copy GPU
|
||||
// convert. RFI/HDR-SEI stay unsupported on libavcodec NVENC (the trait defaults).
|
||||
chroma_444: self.want_444,
|
||||
|
||||
@@ -1840,6 +1840,8 @@ impl Encoder for NvencCudaEncoder {
|
||||
|
||||
fn caps(&self) -> EncoderCaps {
|
||||
EncoderCaps {
|
||||
// Composites `frame.cursor` via the SPIR-V blend over the Vulkan-allocated input slot.
|
||||
blends_cursor: true,
|
||||
supports_rfi: self.rfi_supported,
|
||||
supports_hdr_metadata: self.hdr,
|
||||
chroma_444: self.chroma_444,
|
||||
|
||||
@@ -1209,6 +1209,8 @@ impl Encoder for PyroWaveEncoder {
|
||||
// hardcoded `default()` here mis-reports a 4:4:4 open as 4:2:0 and fires a spurious
|
||||
// "chroma disagrees with the negotiated Welcome" warn.
|
||||
EncoderCaps {
|
||||
// The wavelet CSC composites the metadata cursor.
|
||||
blends_cursor: true,
|
||||
chroma_444: self.chroma444,
|
||||
..EncoderCaps::default()
|
||||
}
|
||||
|
||||
@@ -3308,6 +3308,11 @@ impl Encoder for VulkanVideoEncoder {
|
||||
fn caps(&self) -> EncoderCaps {
|
||||
EncoderCaps {
|
||||
supports_rfi: true,
|
||||
// Only the CSC path composites the metadata cursor (`prep_cursor` feeds the compute
|
||||
// shader). The RGB-direct/EFC front-end and the native-NV12 source have no compositing
|
||||
// stage at all — both merely warn once that the pointer is being dropped — so this is
|
||||
// the encoder's real answer, not a static one.
|
||||
blends_cursor: self.rgb.is_none() && !self.native_nv12,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2426,6 +2426,8 @@ impl Encoder for AmfEncoder {
|
||||
|
||||
fn caps(&self) -> EncoderCaps {
|
||||
EncoderCaps {
|
||||
// As Windows NVENC: the capturer composites; this backend never reads `frame.cursor`.
|
||||
blends_cursor: false,
|
||||
// LTR-RFI: AMD's reference invalidation is the user long-term-reference path (mark a
|
||||
// frame, force a later one to re-reference it). True only when the live driver accepted
|
||||
// the LTR slots at open — otherwise loss recovery falls back to a full IDR.
|
||||
|
||||
@@ -681,6 +681,8 @@ impl Encoder for PyroWaveEncoder {
|
||||
// after the caps() default was written — a hardcoded `default()` here mis-reports a 4:4:4
|
||||
// open as 4:2:0 and fires a spurious "chroma disagrees with the negotiated Welcome" warn).
|
||||
EncoderCaps {
|
||||
// The Windows capturer composites the pointer itself; this backend never reads it.
|
||||
blends_cursor: false,
|
||||
chroma_444: self.chroma444,
|
||||
..EncoderCaps::default()
|
||||
}
|
||||
|
||||
@@ -1407,6 +1407,8 @@ impl Encoder for QsvEncoder {
|
||||
|
||||
fn caps(&self) -> EncoderCaps {
|
||||
EncoderCaps {
|
||||
// As Windows NVENC: the capturer composites; this backend never reads `frame.cursor`.
|
||||
blends_cursor: false,
|
||||
supports_rfi: self.ltr_active,
|
||||
// In-band mastering/CLL at IDR (HEVC prefix SEI / AV1 metadata OBU); AVC sessions
|
||||
// are never HDR.
|
||||
|
||||
@@ -177,6 +177,23 @@ pub fn open_video(
|
||||
},
|
||||
}
|
||||
};
|
||||
// The session asked for a composited pointer; say so loudly if the backend that actually opened
|
||||
// cannot deliver one. `cursor_blend` was a REQUEST with no answer for most of this crate's life
|
||||
// (`let _ = cursor_blend;` below), and the result was a stream with no mouse cursor and nothing
|
||||
// in the logs — confirmed on the VAAPI dmabuf path and the libav-NVENC CUDA path.
|
||||
//
|
||||
// A warning is deliberately all this does. `open_video` cannot re-plan capture, so refusing here
|
||||
// would only trade a missing pointer for a dead session; the host owns `plan.cursor_blend` and is
|
||||
// the only layer that can fall back to capturer-side compositing. This makes the condition
|
||||
// visible and queryable (`EncoderCaps::blends_cursor`) so that decision can be made upstream.
|
||||
if cursor_blend && !inner.caps().blends_cursor {
|
||||
tracing::warn!(
|
||||
backend,
|
||||
"session negotiated a composited cursor but this encode backend does not blend \
|
||||
CapturedFrame::cursor — the pointer will be MISSING from the stream unless the \
|
||||
capturer composites it"
|
||||
);
|
||||
}
|
||||
Ok(Box::new(TrackedEncoder {
|
||||
inner,
|
||||
_session: pf_gpu::session_begin(gpu),
|
||||
|
||||
Reference in New Issue
Block a user