feat(vdisplay): compositor-embedded pointer for sessions without the cursor channel (Phase B)

Since the cursor-channel work, every Linux virtual output was created in
metadata pointer mode — making ALL sessions depend on host-side cursor
compositing, including the ones that can never use the channel (Moonlight/
GameStream, legacy clients, capture-mode starts). Those sessions paid the
blend bring-up per session and, whenever a visible cursor was composited,
the loss of NVENC's stream-ordered submit — for a strictly worse cursor
than the compositor's own.

Mirror the Windows no-regression gate: the already-wired per-session
set_hw_cursor(cursor_forward) now drives the Linux backends too. A
cursor-channel session gets metadata (shapes forwarded, composite flip
blends host-side — today's validated path, unchanged); every other session
gets the pointer compositor-EMBEDDED at creation (KWin zkde pointer=2,
Mutter cursor-mode=1, wlroots/hyprland portal CursorMode::Embedded — their
pre-channel default; the portal pair also gains the Metadata arm for
channel sessions, wired but untested on-glass). SessionPlan.cursor_blend
narrows to cursor-forward sessions and rides into the direct-SDK NVENC
open, which skips the Vulkan slot-blend bring-up entirely when off —
embedded sessions ring on plain CUDA surfaces and pay zero cursor cost,
per-session or per-frame.

The keep-alive registry's reuse key grows the created pointer mode (new
VirtualDisplay::hw_cursor getter): a kept embedded display has no cursor
metadata for a channel session to forward, and a kept metadata display
would leave a channel-less session with no pointer in its frames.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-22 23:47:53 +02:00
co-authored by Claude Fable 5
parent d2c46eaf3c
commit 925130d1f9
10 changed files with 205 additions and 46 deletions
+12 -3
View File
@@ -573,6 +573,11 @@ pub struct NvencCudaEncoder {
/// allocations and composite mode degrades to no cursor. `cursor_serial` tracks the
/// uploaded bitmap.
vk_blend: Option<VkSlotBlend>,
/// The session may hand this encoder cursor overlays (`SessionPlan.cursor_blend` — only
/// cursor-channel sessions since Phase B). Off = skip the Vulkan bring-up entirely and ring
/// on plain CUDA surfaces: embedded-pointer sessions never carry an overlay, so they pay
/// zero blend cost, per-session or per-frame.
blend_wanted: bool,
cursor_tried: bool,
cursor_serial: u64,
/// Suppress-until-success latch for the per-frame blend warn: a persistent failure sits in
@@ -645,6 +650,7 @@ impl NvencCudaEncoder {
_cuda: bool,
bit_depth: u8,
chroma: ChromaFormat,
cursor_blend: bool,
) -> Result<Self> {
// The runtime `.so` load is the real "is NVENC possible here" gate: fail the open with a
// clear reason instead of an opaque session error on the first frame.
@@ -684,6 +690,7 @@ impl NvencCudaEncoder {
force_kf: false,
pending_anchor: false,
vk_blend: None,
blend_wanted: cursor_blend,
cursor_tried: false,
cursor_serial: u64::MAX,
cursor_blend_warned: false,
@@ -1174,7 +1181,7 @@ impl NvencCudaEncoder {
// SPIR-V cursor blend can composite into the very bytes NVENC encodes; any bring-up
// or per-slot failure falls back to plain pitched CUDA allocations (sessions always
// encode — composite mode just loses the cursor, warned below).
if !self.cursor_tried {
if !self.cursor_tried && self.blend_wanted {
self.cursor_tried = true;
match VkSlotBlend::new() {
Ok(v) => self.vk_blend = Some(v),
@@ -1576,8 +1583,10 @@ impl Encoder for NvencCudaEncoder {
} else if !self.cursor_blend_warned {
self.cursor_blend_warned = true;
tracing::warn!(
"NVENC (Linux): cursor overlay present but no Vulkan blend (bring-up failed \
earlier) — cursor not composited"
blend_wanted = self.blend_wanted,
"NVENC (Linux): cursor overlay present but no Vulkan blend (bring-up failed, \
or a non-blend session unexpectedly carried an overlay) — cursor not \
composited"
);
}
}
+11 -6
View File
@@ -270,7 +270,7 @@ fn open_video_backend(
chroma: ChromaFormat,
cursor_blend: bool,
) -> Result<(Box<dyn Encoder>, &'static str)> {
let _ = cursor_blend; // consumed only by the Linux vulkan-encode arm below
let _ = cursor_blend; // consumed only by the Linux vulkan-encode + direct-NVENC arms below
validate_dimensions(codec, width, height)?;
// Refresh/fps must be positive and sane: fps feeds the encoder time_base (`Rational(1, fps)`)
// and the pts→ns conversion (`pts * 1e9 / fps`), so 0 builds a 1/0 rational / divides by zero.
@@ -394,6 +394,7 @@ fn open_video_backend(
cuda,
bit_depth,
chroma,
cursor_blend,
)
.map(|e| (e, "nvenc"))
};
@@ -729,12 +730,15 @@ fn open_nvenc_probed(
cuda: bool,
bit_depth: u8,
chroma: ChromaFormat,
cursor_blend: bool,
) -> Result<Box<dyn Encoder>> {
// Direct-SDK NVENC (design/linux-direct-nvenc.md): the DEFAULT on NVIDIA, and only for a CUDA
// capture payload (it registers CUDADEVICEPTR inputs — a CPU/dmabuf frame can't feed it, so those
// keep the libav path; and `cuda` is false on AMD/Intel, so they stay on VAAPI). Set
// PUNKTFUNK_NVENC_DIRECT=0 to fall back to libav. It self-clamps the bitrate internally (its own
// level-ceiling binary search at session open), so it skips the probe-loop stepping below.
#[cfg(not(feature = "nvenc"))]
let _ = cursor_blend; // consumed by the direct-SDK arm below
// Direct-SDK NVENC (design/linux-direct-nvenc.md): the DEFAULT on NVIDIA, and only for a CUDA
// capture payload (it registers CUDADEVICEPTR inputs — a CPU/dmabuf frame can't feed it, so those
// keep the libav path; and `cuda` is false on AMD/Intel, so they stay on VAAPI). Set
// PUNKTFUNK_NVENC_DIRECT=0 to fall back to libav. It self-clamps the bitrate internally (its own
// level-ceiling binary search at session open), so it skips the probe-loop stepping below.
#[cfg(feature = "nvenc")]
if cuda && nvenc_direct_enabled() {
tracing::info!(
@@ -751,6 +755,7 @@ fn open_nvenc_probed(
cuda,
bit_depth,
chroma,
cursor_blend,
)?) as Box<dyn Encoder>);
}
// The silent-degrade trap: a build without `--features nvenc` compiles the direct-SDK