diff --git a/crates/pf-encode/src/enc/linux/nvenc_cuda.rs b/crates/pf-encode/src/enc/linux/nvenc_cuda.rs index 2899ee0b..22663768 100644 --- a/crates/pf-encode/src/enc/linux/nvenc_cuda.rs +++ b/crates/pf-encode/src/enc/linux/nvenc_cuda.rs @@ -1613,14 +1613,22 @@ impl Encoder for NvencCudaEncoder { // `async_rt` must be absent too: in two-thread mode the frame may be recycled right after // submit returns while the stream still holds its copy (belt-and-braces — an escalated // session was rebuilt without the binding, so `stream_ordered` is false there anyway). - // Cursor-bearing frames additionally force the CPU-synced path: the Vulkan blend sits - // between the CUDA copy and the encode, and its cross-API ordering is fence/CPU- - // established, not stream-ordered. Frames without a cursor (games hide it; client-draws - // sessions strip it) keep the stream-ordered fast path untouched. - let ordered = self.stream_ordered - && self.async_rt.is_none() - && self.pending.is_empty() - && captured.cursor.is_none(); + let base_ordered = + self.stream_ordered && self.async_rt.is_none() && self.pending.is_empty(); + // Cursor-bearing frames stay on the fast path when the blend itself can be stream- + // ordered: the Vulkan dispatch waits/advances a timeline semaphore CUDA also holds, so + // copy→blend→encode orders entirely on-device (`VkSlotBlend::blend_ref_ordered`). Where + // that isn't available (no timeline export, or the ring fell back to plain CUDA slots) + // a cursor forces the CPU-synced path: the blend's cross-API ordering is then fence/CPU- + // established, sitting between the copy and the encode. That slow path is why cursor + // frames USED to be gated out entirely — under gamescope the compositor re-attaches the + // live pointer to EVERY frame, and the per-frame CPU syncs (exposed to the running + // game's GPU load) capped a 120 fps session near 80 (submit p50 ~10 ms). + let cursor_ordered = base_ordered + && captured.cursor.is_some() + && matches!(self.ring[slot].surface, SlotSurface::Vk(_)) + && self.vk_blend.as_ref().is_some_and(|vk| vk.ordered_ready()); + let ordered = base_ordered && (captured.cursor.is_none() || cursor_ordered); let t0 = std::time::Instant::now(); // Copy the captured buffer into this slot's input surface before encoding it. @@ -1629,29 +1637,45 @@ impl Encoder for NvencCudaEncoder { // Cursor-as-metadata: blend the overlay into this slot's OWNED input surface via the // SPIR-V compute pass (a dispatch over the cursor's rect — never the compositor's - // dmabuf). Cursor-bearing frames forced `ordered = false` above, so the CUDA copy has - // completed before the Vulkan dispatch and the fence-waited dispatch completes before - // the encode below — the cross-API ordering is CPU-established. Any failure degrades to - // no cursor, never a dropped frame. + // dmabuf). On the `cursor_ordered` path the enqueued copy, the dispatch, and the encode + // are ordered on-device through the timeline semaphore (no CPU sync — see the gate + // above). Otherwise `ordered` is false: the CUDA copy completed before the Vulkan + // dispatch and the fence-waited dispatch completes before the encode below — the + // cross-API ordering is CPU-established. Any failure degrades to no cursor, never a + // dropped frame (a failed ordered blend leaves the copy→encode stream ordering intact). if let Some(ov) = &captured.cursor { if let (Some(vk), SlotSurface::Vk(vref)) = (self.vk_blend.as_mut(), &self.ring[slot].surface) { if self.cursor_serial != ov.serial { + // Quiesces any in-flight ordered blend internally before touching the + // staging buffer (bitmap changes are rare — shape flips). vk.upload_cursor(ov.rgba.as_slice(), ov.w, ov.h); self.cursor_serial = ov.serial; } // surfW = content width; the blend derives plane strides from the slot's luma // height. Cursor pixels past the content land in cropped padding rows — harmless. - let r = vk.blend_ref( - vref, - slot_fmt_of(self.buffer_fmt), - self.width, - ov.w, - ov.h, - ov.x, - ov.y, - ); + let r = if cursor_ordered { + vk.blend_ref_ordered( + vref, + slot_fmt_of(self.buffer_fmt), + self.width, + ov.w, + ov.h, + ov.x, + ov.y, + ) + } else { + vk.blend_ref( + vref, + slot_fmt_of(self.buffer_fmt), + self.width, + ov.w, + ov.h, + ov.x, + ov.y, + ) + }; if let Err(e) = r { if !self.cursor_blend_warned { self.cursor_blend_warned = true; @@ -1686,7 +1710,9 @@ impl Encoder for NvencCudaEncoder { // stack-local and outlives the synchronous `encode_picture`. The input surface for `slot` was // just filled by the device→device copy — either synchronized (blocking mode) or ordered // before this encode by the session's IO-stream binding (`ordered` — same stream, see the - // gate above) — and is not overwritten until this slot is reused POOL submits later, by + // gate above; on the `cursor_ordered` path the blend's writes are likewise ordered before + // the encode, via the timeline-semaphore wait `blend_ref_ordered` enqueued on that same + // stream) — and is not overwritten until this slot is reused POOL submits later, by // which time this encode was polled (POOL ≥ in-flight depth; in ordered mode the poll's // blocking lock additionally proves the enqueued copy completed). unsafe { @@ -2673,6 +2699,85 @@ mod tests { ); } + /// ON-HARDWARE (RTX box `.21`): cursor-bearing frames must KEEP the stream-ordered fast + /// path — the gamescope 80-fps-on-a-120-session fix. With the timeline-semaphore blend + /// available, `submit` takes `blend_ref_ordered` (the ticket advances by 2 per frame) + /// instead of the CPU-synced fence-wait blend, and AUs keep flowing — including across a + /// cursor-bitmap change (exercises the upload quiesce) and per-frame position moves. + #[test] + #[ignore = "requires an NVIDIA GPU + driver — run manually on the RTX box (.21)"] + fn nvenc_cuda_cursor_blend_stream_ordered() { + const W: u32 = 1280; + const H: u32 = 720; + pf_zerocopy::cuda::make_current().expect("shared CUDA context current"); + // Respect an explicit operator opt-out (or two-thread mode) rather than fail. + if !stream_ordered_requested() || async_retrieve_requested() { + println!("skipped: stream-ordered submit disabled by env"); + return; + } + let mut enc = NvencCudaEncoder::open( + Codec::H265, + PixelFormat::Nv12, + W, + H, + 60, + 8_000_000, + true, + 8, + ChromaFormat::Yuv420, + true, // cursor_blend: bring up the Vulkan slot ring + blend + ) + .expect("open NVENC CUDA session"); + let cursor = |serial: u64, x: i32, y: i32| pf_frame::CursorOverlay { + x, + y, + w: 32, + h: 32, + rgba: std::sync::Arc::new(vec![0xFF; 32 * 32 * 4]), + serial, + hot_x: 0, + hot_y: 0, + visible: true, + }; + let mut aus = 0usize; + for i in 0..6u32 { + let mut frame = nv12_frame(W, H, i); + // Bitmap serial flips at frame 3 (upload quiesce over in-flight ordered blends); + // the position moves every frame (push-constant path). + frame.cursor = Some(cursor( + if i < 3 { 1 } else { 2 }, + 40 + i as i32 * 9, + 60 + i as i32 * 5, + )); + enc.submit_indexed(&frame, i).expect("submit cursor frame"); + while enc.poll().expect("poll").is_some() { + aus += 1; + } + } + assert_eq!(aus, 6, "every cursor frame must deliver an AU"); + assert!( + enc.stream_ordered, + "IO-stream binding must arm on a default-env session" + ); + let vk = enc + .vk_blend + .as_ref() + .expect("Vulkan slot blend must come up on an RTX box"); + assert!( + vk.ordered_ready(), + "timeline semaphore must export to CUDA on this driver" + ); + assert_eq!( + vk.ordered_ticket(), + 12, + "all 6 cursor blends must take the ordered path (2 timeline values each)" + ); + println!( + "nvenc_cuda cursor stream-ordered: 6 cursor AUs, ticket={}", + vk.ordered_ticket() + ); + } + /// ON-HARDWARE (RTX box `.21`): the §7 LN3 pipelined-retrieve escalation — /// `set_pipelined(true)` on a live sync session must rebuild it without the IO-stream /// binding, spawn the retrieve thread on the re-open, and keep delivering AUs (the first diff --git a/crates/pf-zerocopy/src/imp/cuda.rs b/crates/pf-zerocopy/src/imp/cuda.rs index 0daef3c9..7a5a98f1 100644 --- a/crates/pf-zerocopy/src/imp/cuda.rs +++ b/crates/pf-zerocopy/src/imp/cuda.rs @@ -262,6 +262,15 @@ unsafe fn copy_async(copy: &CUDA_MEMCPY2D, what: &str) -> Result<()> { ck(cuMemcpy2DAsync_v2(copy, copy_stream()), what) } +/// Block until everything enqueued on THIS THREAD's copy stream completed — the shared tail of +/// the multi-plane blocking copies (stream FIFO: one sync after the last enqueue covers every +/// plane, where the per-plane `copy_blocking` paid one exposed CPU wait EACH — and under a +/// game's GPU load each exposed wait eats scheduling latency). The shared context must be +/// current. +unsafe fn sync_copy_stream() -> Result<()> { + ck(cuStreamSynchronize(copy_stream()), "cuStreamSynchronize") +} + /// `copy_blocking` when `sync`, else `copy_async` — the shared tail of the public `copy_*_to_device` /// helpers, whose `sync: false` mode carries `copy_async`'s source-lifetime contract. unsafe fn copy_issue(copy: &CUDA_MEMCPY2D, what: &str, sync: bool) -> Result<()> { @@ -985,17 +994,24 @@ pub fn copy_nv12_to_device( Height: h / 2, ..Default::default() }; - // SAFETY: two unsafe `copy_issue` device→device copies; the caller must have the shared - // context current (documented). `&y`/`&uv` are live local `CUDA_MEMCPY2D`s outliving each - // enqueue. All four device pointers are valid: `src.ptr`/`src_uv_ptr` come from a live - // NV12 `DeviceBuffer` (its `.uv` presence was checked via `ok_or_else`), `y_dst`/`uv_dst` are - // the caller's live NVENC surface planes; the luma copy is `w`×`h`, the chroma copy - // `(w/2)*2`×`h/2`, each within its planes; `sync: false` shifts the source-lifetime obligation - // to the caller (documented above). Wrappers → live table. + // SAFETY: two unsafe `copy_async` device→device enqueues + an optional stream sync; the + // caller must have the shared context current (documented). `&y`/`&uv` are live local + // `CUDA_MEMCPY2D`s outliving each enqueue. All four device pointers are valid: + // `src.ptr`/`src_uv_ptr` come from a live NV12 `DeviceBuffer` (its `.uv` presence was + // checked via `ok_or_else`), `y_dst`/`uv_dst` are the caller's live NVENC surface planes; + // the luma copy is `w`×`h`, the chroma copy `(w/2)*2`×`h/2`, each within its planes. With + // `sync` the single trailing stream sync covers both enqueues (FIFO) before we return — the + // same completion guarantee as the old per-plane blocking copies at half the exposed waits; + // `sync: false` shifts the source-lifetime obligation to the caller (documented above). + // Wrappers → live table. unsafe { - copy_issue(&y, "cuMemcpy2DAsync_v2(nv12 Y dev->dev)", sync)?; - copy_issue(&uv, "cuMemcpy2DAsync_v2(nv12 UV dev->dev)", sync) + copy_async(&y, "cuMemcpy2DAsync_v2(nv12 Y dev->dev)")?; + copy_async(&uv, "cuMemcpy2DAsync_v2(nv12 UV dev->dev)")?; + if sync { + sync_copy_stream()?; + } } + Ok(()) } /// Copy our imported stacked-YUV444 [`DeviceBuffer`] into NVENC's three-plane CUDA surface @@ -1024,13 +1040,19 @@ pub fn copy_yuv444_to_device( Height: h, ..Default::default() }; - // SAFETY: unsafe `copy_issue` device→device copy; the caller must have the shared + // SAFETY: unsafe `copy_async` device→device enqueue; the caller must have the shared // context current (documented). `©` is a live local outliving the enqueue; // `src.ptr + pitch·h·i` stays within the live 3·H-row stacked allocation (`yuv444` // checked above), `dst_ptr`/`dst_pitch` is the caller's live NVENC plane; `w`×`h` fits - // both; `sync: false` shifts the source-lifetime obligation to the caller (documented - // above). Wrapper → live table. - unsafe { copy_issue(©, "cuMemcpy2DAsync_v2(yuv444 plane dev->dev)", sync)? }; + // both. Completion is the trailing stream sync below (`sync`) or the caller's + // stream-ordering obligation (`sync: false`, documented above). Wrapper → live table. + unsafe { copy_async(©, "cuMemcpy2DAsync_v2(yuv444 plane dev->dev)")? }; + } + if sync { + // SAFETY: one stream sync after the last enqueue covers all three planes (FIFO) — the + // same completion guarantee as the old per-plane blocking copies at a third of the + // exposed waits. Context current per the caller's contract. Wrapper → live table. + unsafe { sync_copy_stream()? }; } Ok(()) } @@ -1169,6 +1191,101 @@ impl Drop for ExternalDmabuf { } } +/// A Vulkan **timeline** semaphore imported as a CUDA external semaphore — the cross-API ordering +/// primitive for the stream-ordered cursor blend (`vkslot.rs`): CUDA [`signal`](Self::signal)s a +/// value on this thread's copy stream once the input copy is enqueued, the Vulkan blend waits for +/// and then advances the timeline on its queue, and CUDA [`wait`](Self::wait)s that advanced +/// value before the encode — all ordering on-device, no CPU sync anywhere. One imported handle +/// per [`VkSlotBlend`](super::vkslot::VkSlotBlend); values are monotonic for its lifetime. +pub struct ExternalSemaphore { + sem: CUexternalSemaphore, +} + +// SAFETY: `CUexternalSemaphore` is an opaque driver handle with no thread affinity (the driver +// API allows use from any thread with the context current). It is uniquely owned here, used from +// the encode thread but moved with its `VkSlotBlend`, and destroyed exactly once in `Drop` — +// `Send` (not `Sync`) matches that single-thread-at-a-time use, like `ExternalDmabuf`. +unsafe impl Send for ExternalSemaphore {} + +impl ExternalSemaphore { + /// Import a Vulkan timeline semaphore exported as an OPAQUE_FD (`vkGetSemaphoreFdKHR`). The + /// fd is handed over: the driver owns it on success, we close it on failure. The shared + /// context must be current. + pub fn import_owned_timeline_fd(fd: i32) -> Result { + let mut desc = CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC { + type_: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_FD, + ..Default::default() + }; + desc.handle[0] = fd as u32 as u64; // union member `int fd` (little-endian low bytes) + let mut sem: CUexternalSemaphore = std::ptr::null_mut(); + // SAFETY: `cuImportExternalSemaphore` reads `&desc`, a live local `#[repr(C)]` + // `CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC` (layout-asserted below) outliving the synchronous + // call: `type_` is TIMELINE_SEMAPHORE_FD and `handle[0]` holds the fd in the union's + // `int fd` low bytes. `&mut sem` is a live null-init out-param the driver writes the + // imported handle into. Distinct locals → no aliasing. Wrapper → live table (caller holds + // the context current). + let r = unsafe { cuImportExternalSemaphore(&mut sem, &desc) }; + if r != 0 { + // SAFETY: import failed (`r != 0`), so the driver did NOT take ownership of `fd`; we + // still own it and close it exactly once here. `libc::close` acts on the integer alone. + unsafe { libc::close(fd) }; + bail!("cuImportExternalSemaphore failed ({r}) — timeline-semaphore fd export/import unsupported?"); + } + Ok(ExternalSemaphore { sem }) + } + + /// Enqueue a signal that sets the timeline to `value` once all prior work on THIS THREAD's + /// copy stream (the stream `copy_stream_handle` exposes) completes. No CPU wait. + pub fn signal(&self, value: u64) -> Result<()> { + let params = CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS { + value, + ..Default::default() + }; + // SAFETY: `self.sem` is the live imported handle (this struct only exists after a + // successful import; destroyed only in `Drop`). `&self.sem`/`¶ms` are live locals the + // synchronous enqueue reads (count 1); the driver retains no pointer into Rust memory. + // The stream is this thread's live copy stream. Wrapper → live table (context current). + unsafe { + ck( + cuSignalExternalSemaphoresAsync(&self.sem, ¶ms, 1, copy_stream()), + "cuSignalExternalSemaphoresAsync", + ) + } + } + + /// Enqueue a wait: work enqueued on THIS THREAD's copy stream after this call runs only once + /// the timeline reaches `value`. No CPU wait. + pub fn wait(&self, value: u64) -> Result<()> { + let params = CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS { + value, + ..Default::default() + }; + // SAFETY: same contract as `signal` — live handle, live locals across the synchronous + // enqueue, this thread's live copy stream. Wrapper → live table (context current). + unsafe { + ck( + cuWaitExternalSemaphoresAsync(&self.sem, ¶ms, 1, copy_stream()), + "cuWaitExternalSemaphoresAsync", + ) + } + } +} + +impl Drop for ExternalSemaphore { + fn drop(&mut self) { + // SAFETY: `self.sem` is the valid imported handle this struct exclusively owns, destroyed + // exactly once here. The shared context is made current first because drop may run off + // the import thread (`VkSlotBlend` teardown quiesces the GPU before dropping, so no + // enqueued signal/wait still references the semaphore). Result ignored (best-effort). + unsafe { + if let Some(c) = CONTEXT.get() { + let _ = cuCtxSetCurrent(c.0); + } + let _ = cuDestroyExternalSemaphore(self.sem); + } + } +} + /// Copy a pitched span starting at `src_ptr` (e.g. an [`ExternalDmabuf`] mapping at the chunk /// offset) into `dst`. The shared context must be current on this thread. pub fn copy_pitched_to_buffer( @@ -1243,3 +1360,31 @@ pub fn copy_pitched_nv12_to_buffer( copy_blocking(&uv, "cuMemcpy2DAsync_v2(ext->dev nv12 UV)") } } + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::{offset_of, size_of}; + + /// The external-semaphore param structs are hand-flattened from cuda.h unions — assert the + /// layout against the C definitions so a transcription slip fails in CI, not in the driver. + #[test] + fn external_semaphore_struct_layouts_match_cuda_h() { + // CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC: type(4)+pad(4)+union(16)+flags(4)+reserved(64) = 96. + assert_eq!(size_of::(), 96); + assert_eq!(offset_of!(CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC, handle), 8); + assert_eq!(offset_of!(CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC, flags), 24); + + // CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS: params{fence(8)+nvSciSync(8)+keyedMutex(8)+ + // reserved[12](48)} = 72, flags at 72, reserved[16] → 144 total. + assert_eq!(size_of::(), 144); + assert_eq!(offset_of!(CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS, value), 0); + assert_eq!(offset_of!(CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS, flags), 72); + + // CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS: params{fence(8)+nvSciSync(8)+keyedMutex(16, + // tail-padded)+reserved[10](40)} = 72, flags at 72, reserved[16] → 144 total. + assert_eq!(size_of::(), 144); + assert_eq!(offset_of!(CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS, value), 0); + assert_eq!(offset_of!(CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS, flags), 72); + } +} diff --git a/crates/pf-zerocopy/src/imp/cuda/ffi.rs b/crates/pf-zerocopy/src/imp/cuda/ffi.rs index 046ad04e..67c243a3 100644 --- a/crates/pf-zerocopy/src/imp/cuda/ffi.rs +++ b/crates/pf-zerocopy/src/imp/cuda/ffi.rs @@ -20,6 +20,7 @@ pub type CUdeviceptr = u64; pub type CUgraphicsResource = *mut c_void; pub type CUarray = *mut c_void; pub type CUexternalMemory = *mut c_void; // opaque CUextMemory_st* +pub type CUexternalSemaphore = *mut c_void; // opaque CUextSemaphore_st* /// `CUmemorytype` (cuda.h): HOST=1, DEVICE=2, ARRAY=3, UNIFIED=4. pub const CU_MEMORYTYPE_DEVICE: c_uint = 2; @@ -84,6 +85,60 @@ pub struct CUDA_EXTERNAL_MEMORY_BUFFER_DESC { pub const CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD: c_uint = 1; +/// `CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC` (cuda.h, 64-bit layout). Same union-flattening as the +/// memory desc above: `handle` is a union whose largest member is the win32 two-pointer struct +/// (16 bytes, align 8); for the fd-carrying types only the first 4 bytes (the `int fd`) are read. +/// No `size` field — a semaphore has none. +#[repr(C)] +#[derive(Default)] +pub struct CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC { + pub type_: c_uint, // CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_FD = 9 + pub(crate) _pad: u32, + pub handle: [u64; 2], // union { int fd; {void*,void*} win32; const void* nvSciSyncObj } + pub flags: c_uint, + pub(crate) reserved: [c_uint; 16], + pub(crate) _pad2: u32, +} + +/// `CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS` (cuda.h, 64-bit layout), flattened: `params` nests +/// `fence.value` (the only member we set — the timeline value to signal), the `nvSciSync` union, +/// `keyedMutex.key`, then 12 reserved words; `flags` + 16 reserved words follow. 144 bytes total +/// (layout-asserted in `super`'s tests). +#[repr(C)] +#[derive(Default)] +pub struct CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS { + pub value: u64, // params.fence.value — the timeline value this signal sets + pub(crate) _nv_sci_sync: u64, + pub(crate) _keyed_mutex_key: u64, + pub(crate) _params_reserved: [c_uint; 12], + pub flags: c_uint, + pub(crate) reserved: [c_uint; 16], + pub(crate) _pad: u32, +} + +/// `CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS` (cuda.h, 64-bit layout), flattened like the signal +/// params. The C `keyedMutex` member is `{ u64 key; u32 timeoutMs; }` — size 16 with tail +/// padding, hence the explicit pad word before the 10 reserved words. 144 bytes total. +#[repr(C)] +#[derive(Default)] +pub struct CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS { + pub value: u64, // params.fence.value — wait until the timeline reaches this value + pub(crate) _nv_sci_sync: u64, + pub(crate) _keyed_mutex_key: u64, + pub(crate) _keyed_mutex_timeout: c_uint, + pub(crate) _keyed_mutex_pad: u32, + pub(crate) _params_reserved: [c_uint; 10], + pub flags: c_uint, + pub(crate) reserved: [c_uint; 16], + pub(crate) _pad: u32, +} + +/// `CUexternalSemaphoreHandleType` (cuda.h): a Vulkan **timeline** semaphore exported as an +/// OPAQUE_FD (`vkGetSemaphoreFdKHR`). Needs driver ≥ 460 (CUDA 11.2) — far below the NVENC 12.1 +/// floor this backend already requires, so import failure means "driver refused", not "too old +/// to try". +pub const CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_FD: c_uint = 9; + /// `CUipcMemHandle` (cuda.h): an opaque 64-byte struct identifying a device allocation across /// processes. Produced by `cuIpcGetMemHandle` in the exporting process, consumed by /// `cuIpcOpenMemHandle` in the importer — passed **by value**, matching the C @@ -139,6 +194,23 @@ pub(crate) struct CudaApi { *const CUDA_EXTERNAL_MEMORY_BUFFER_DESC, ) -> CUresult, cuDestroyExternalMemory: unsafe extern "C" fn(CUexternalMemory) -> CUresult, + cuImportExternalSemaphore: unsafe extern "C" fn( + *mut CUexternalSemaphore, + *const CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC, + ) -> CUresult, + cuDestroyExternalSemaphore: unsafe extern "C" fn(CUexternalSemaphore) -> CUresult, + cuSignalExternalSemaphoresAsync: unsafe extern "C" fn( + *const CUexternalSemaphore, + *const CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS, + c_uint, + CUstream, + ) -> CUresult, + cuWaitExternalSemaphoresAsync: unsafe extern "C" fn( + *const CUexternalSemaphore, + *const CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS, + c_uint, + CUstream, + ) -> CUresult, cuIpcGetMemHandle: unsafe extern "C" fn(*mut CUipcMemHandle, CUdeviceptr) -> CUresult, cuIpcOpenMemHandle: unsafe extern "C" fn(*mut CUdeviceptr, CUipcMemHandle, c_uint) -> CUresult, cuIpcCloseMemHandle: unsafe extern "C" fn(CUdeviceptr) -> CUresult, @@ -206,6 +278,14 @@ pub(crate) fn cuda_api() -> Option<&'static CudaApi> { .get(b"cuExternalMemoryGetMappedBuffer\0") .ok()?, cuDestroyExternalMemory: *lib.get(b"cuDestroyExternalMemory\0").ok()?, + // External-semaphore interop (the stream-ordered cursor blend): all four are + // CUDA 10.0 entry points, far older than anything else this table requires. + cuImportExternalSemaphore: *lib.get(b"cuImportExternalSemaphore\0").ok()?, + cuDestroyExternalSemaphore: *lib.get(b"cuDestroyExternalSemaphore\0").ok()?, + cuSignalExternalSemaphoresAsync: *lib + .get(b"cuSignalExternalSemaphoresAsync\0") + .ok()?, + cuWaitExternalSemaphoresAsync: *lib.get(b"cuWaitExternalSemaphoresAsync\0").ok()?, cuIpcGetMemHandle: *lib.get(b"cuIpcGetMemHandle\0").ok()?, // CUDA 11 renamed the entry point (per-thread-stream ABI split); every modern // driver exports `_v2`, but accept the unsuffixed one too (same signature). @@ -381,6 +461,43 @@ pub(crate) unsafe fn cuDestroyExternalMemory(ext_mem: CUexternalMemory) -> CUres None => CU_ERROR_NOT_LOADED, } } +pub(crate) unsafe fn cuImportExternalSemaphore( + ext_sem_out: *mut CUexternalSemaphore, + sem_handle_desc: *const CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC, +) -> CUresult { + match cuda_api() { + Some(a) => (a.cuImportExternalSemaphore)(ext_sem_out, sem_handle_desc), + None => CU_ERROR_NOT_LOADED, + } +} +pub(crate) unsafe fn cuDestroyExternalSemaphore(ext_sem: CUexternalSemaphore) -> CUresult { + match cuda_api() { + Some(a) => (a.cuDestroyExternalSemaphore)(ext_sem), + None => CU_ERROR_NOT_LOADED, + } +} +pub(crate) unsafe fn cuSignalExternalSemaphoresAsync( + ext_sems: *const CUexternalSemaphore, + params: *const CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS, + count: c_uint, + stream: CUstream, +) -> CUresult { + match cuda_api() { + Some(a) => (a.cuSignalExternalSemaphoresAsync)(ext_sems, params, count, stream), + None => CU_ERROR_NOT_LOADED, + } +} +pub(crate) unsafe fn cuWaitExternalSemaphoresAsync( + ext_sems: *const CUexternalSemaphore, + params: *const CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS, + count: c_uint, + stream: CUstream, +) -> CUresult { + match cuda_api() { + Some(a) => (a.cuWaitExternalSemaphoresAsync)(ext_sems, params, count, stream), + None => CU_ERROR_NOT_LOADED, + } +} pub(crate) unsafe fn cuIpcGetMemHandle(handle: *mut CUipcMemHandle, dptr: CUdeviceptr) -> CUresult { match cuda_api() { Some(a) => (a.cuIpcGetMemHandle)(handle, dptr), diff --git a/crates/pf-zerocopy/src/imp/vkslot.rs b/crates/pf-zerocopy/src/imp/vkslot.rs index 92968d45..1a3dec0e 100644 --- a/crates/pf-zerocopy/src/imp/vkslot.rs +++ b/crates/pf-zerocopy/src/imp/vkslot.rs @@ -12,11 +12,24 @@ //! //! The direct-SDK NVENC encoder allocates its input ring through [`VkSlotBlend::alloc_slot`] //! instead of `cuMemAllocPitch`: same contiguous layouts (`InputSurface` docs), but the memory is -//! Vulkan external memory both APIs address. Per cursor-bearing frame the encoder CPU-syncs its -//! CUDA copy, then [`VkSlotBlend::blend`] dispatches the compute blend over the cursor's -//! rectangle and fence-waits — the same coherence ceremony [`super::vulkan::VkBridge`] ships for -//! its CSC (fence-ordered cross-API access on NVIDIA, no queue-family transfer needed). Frames -//! without a cursor never touch Vulkan, keeping the stream-ordered fast path intact. +//! Vulkan external memory both APIs address. +//! +//! Cursor-bearing frames blend one of two ways: +//! +//! * **Stream-ordered** ([`blend_ref_ordered`](VkSlotBlend::blend_ref_ordered), the fast path +//! when the driver exports a timeline semaphore to CUDA): the encoder enqueues its CUDA copy +//! with no CPU sync, CUDA signals the shared timeline on the copy stream, the blend submission +//! waits for and then advances it on the Vulkan queue, and CUDA waits the advanced value +//! before the encode — the whole copy→blend→encode chain orders on-device, so a visible +//! cursor costs the submit path nothing. (Before this, EVERY cursor frame took the CPU-synced +//! path below; under gamescope — which composites the pointer into every frame — that +//! serialized submit behind the game's GPU load and capped a 120 fps session near 80.) +//! * **CPU-synced** ([`blend_ref`](VkSlotBlend::blend_ref), the fallback when timeline bring-up +//! failed): the encoder blocks on its CUDA copy, the blend fence-waits — the same coherence +//! ceremony [`super::vulkan::VkBridge`] ships for its CSC (fence-ordered cross-API access on +//! NVIDIA, no queue-family transfer needed). +//! +//! Frames without a cursor never touch Vulkan at all. //! //! Falls back cleanly: if bring-up fails the encoder allocates plain CUDA surfaces and composite //! mode degrades to no cursor (warned once) — never a failed session. @@ -87,12 +100,36 @@ pub struct VkSlotRef { } /// One allocated slot's backing objects, freed together in reverse order (CUDA mapping first). +/// Each slot carries its own command buffer + descriptor set so ordered blends can be in flight +/// on several slots at once (the shared-single-set design raced the next recording against a +/// still-executing submission); the set's bindings are written once here — the slot buffer never +/// changes and the cursor staging buffer is shared. struct SlotAlloc { buffer: vk::Buffer, memory: vk::DeviceMemory, /// CUDA's import of the exported OPAQUE_FD — must drop BEFORE the Vulkan memory is freed. cuda: cuda::ExternalDmabuf, - size: u64, + cmd: vk::CommandBuffer, + desc: vk::DescriptorSet, +} + +/// The cross-API ordering state for stream-ordered blends: one Vulkan timeline semaphore, +/// exported as an OPAQUE_FD and imported into CUDA. `None` when the driver lacks timeline +/// semaphores or the export/import failed — blends then stay CPU-synced ([`VkSlotBlend::blend_ref`]). +struct Timeline { + sem: vk::Semaphore, + /// `vkWaitSemaphoresKHR` — the CPU-side quiesce for cursor-bitmap uploads and teardown + /// (the 1.1 device gets the entry point from `VK_KHR_timeline_semaphore`). + ts: ash::khr::timeline_semaphore::Device, + /// CUDA's import; its `signal`/`wait` enqueue on the encode thread's copy stream. + cuda: cuda::ExternalSemaphore, + /// Last timeline value handed out (monotonic for the device's lifetime, never reused — + /// each ordered blend consumes two: copy-done, then blend-done). + ticket: u64, + /// Last blend-done value an ACCEPTED `vkQueueSubmit` will signal — what upload/teardown + /// quiesce on. Only advanced on submit success: a value from a failed submit would never + /// be signaled and a quiesce on it would time out. + last_blend: u64, } /// 28-byte push-constant block matching `cursor_blend.comp`'s `Push`. @@ -114,14 +151,12 @@ pub struct VkSlotBlend { ext_fd: ash::khr::external_memory_fd::Device, queue: vk::Queue, cmd_pool: vk::CommandPool, - cmd: vk::CommandBuffer, fence: vk::Fence, mem_props: vk::PhysicalDeviceMemoryProperties, shader: vk::ShaderModule, desc_layout: vk::DescriptorSetLayout, pipe_layout: vk::PipelineLayout, desc_pool: vk::DescriptorPool, - desc_set: vk::DescriptorSet, /// One pipeline per [`SlotFormat`], indexed by `mode()` (spec constant). pipelines: [vk::Pipeline; 3], /// Host-visible cursor bitmap staging (CURSOR_MAX²·4, tight rows), persistently mapped. @@ -129,6 +164,8 @@ pub struct VkSlotBlend { cur_mem: vk::DeviceMemory, cur_map: *mut u8, slots: Vec, + /// Stream-ordered blend support (`None` = CPU-synced blends only). See [`Timeline`]. + timeline: Option, } // SAFETY: raw Vulkan handles + a persistently-mapped pointer, all uniquely owned by this struct @@ -182,14 +219,44 @@ impl VkSlotBlend { let qci = [vk::DeviceQueueCreateInfo::default() .queue_family_index(qf) .queue_priorities(&prio)]; - let exts = [ash::khr::external_memory_fd::NAME.as_ptr()]; - let device = match instance.create_device( - phys, - &vk::DeviceCreateInfo::default() - .queue_create_infos(&qci) - .enabled_extension_names(&exts), - None, - ) { + // Timeline-semaphore export to CUDA (the stream-ordered blend) is optional: probe the + // device extensions + feature bit and enable them only where present, so a driver + // without them still gets a working (CPU-synced) blend device. NVIDIA has shipped + // both extensions since ~2019; the probe is for exotic/legacy stacks. + let want_timeline = { + let have_exts = instance + .enumerate_device_extension_properties(phys) + .map(|props| { + let has = |name: &std::ffi::CStr| { + props + .iter() + .any(|p| p.extension_name_as_c_str().is_ok_and(|n| n == name)) + }; + has(ash::khr::timeline_semaphore::NAME) + && has(ash::khr::external_semaphore_fd::NAME) + }) + .unwrap_or(false); + have_exts && { + let mut tl = vk::PhysicalDeviceTimelineSemaphoreFeatures::default(); + let mut f2 = vk::PhysicalDeviceFeatures2::default().push_next(&mut tl); + instance.get_physical_device_features2(phys, &mut f2); + tl.timeline_semaphore == vk::TRUE + } + }; + let mut exts = vec![ash::khr::external_memory_fd::NAME.as_ptr()]; + if want_timeline { + exts.push(ash::khr::timeline_semaphore::NAME.as_ptr()); + exts.push(ash::khr::external_semaphore_fd::NAME.as_ptr()); + } + let mut tl_enable = + vk::PhysicalDeviceTimelineSemaphoreFeatures::default().timeline_semaphore(true); + let mut dci = vk::DeviceCreateInfo::default() + .queue_create_infos(&qci) + .enabled_extension_names(&exts); + if want_timeline { + dci = dci.push_next(&mut tl_enable); + } + let device = match instance.create_device(phys, &dci, None) { Ok(d) => d, Err(e) => { instance.destroy_instance(None); @@ -207,30 +274,93 @@ impl VkSlotBlend { ext_fd, queue, cmd_pool: vk::CommandPool::null(), - cmd: vk::CommandBuffer::null(), fence: vk::Fence::null(), mem_props, shader: vk::ShaderModule::null(), desc_layout: vk::DescriptorSetLayout::null(), pipe_layout: vk::PipelineLayout::null(), desc_pool: vk::DescriptorPool::null(), - desc_set: vk::DescriptorSet::null(), pipelines: [vk::Pipeline::null(); 3], cur_buf: vk::Buffer::null(), cur_mem: vk::DeviceMemory::null(), cur_map: std::ptr::null_mut(), slots: Vec::new(), + timeline: None, }; me.init_objects(qf).inspect_err(|_| { // `Drop` runs the same teardown and tolerates the nulls left by a partial init. })?; + if want_timeline { + // Non-fatal: any failure (export refused, CUDA import refused) leaves + // `timeline` None and blends CPU-synced — never a failed bring-up. + if let Err(e) = me.init_timeline() { + tracing::info!( + error = %format!("{e:#}"), + "cursor blend timeline export unavailable — blends stay CPU-synced" + ); + } + } tracing::info!( + stream_ordered = me.timeline.is_some(), "Vulkan slot blend ready (exportable NVENC inputs + SPIR-V cursor blend)" ); Ok(me) } } + /// Bring up the [`Timeline`]: create an exportable timeline semaphore, export its OPAQUE_FD, + /// import it into CUDA. Only called when the device was created with the timeline extensions. + fn init_timeline(&mut self) -> Result<()> { + // SAFETY: ash calls on the live `self.device` with builder infos from locals outliving + // each synchronous call. The semaphore is destroyed on every failure path after its + // creation; the exported fd is owned by `import_owned_timeline_fd` (the driver takes it + // on success, it closes it on failure). The shared CUDA context is current: the encoder + // brings this device up from its encode thread with the context established. + unsafe { + let mut type_ci = vk::SemaphoreTypeCreateInfo::default() + .semaphore_type(vk::SemaphoreType::TIMELINE) + .initial_value(0); + let mut export = vk::ExportSemaphoreCreateInfo::default() + .handle_types(vk::ExternalSemaphoreHandleTypeFlags::OPAQUE_FD); + let sem = self + .device + .create_semaphore( + &vk::SemaphoreCreateInfo::default() + .push_next(&mut type_ci) + .push_next(&mut export), + None, + ) + .context("create timeline semaphore")?; + let sem_fd = ash::khr::external_semaphore_fd::Device::new(&self.instance, &self.device); + let fd = match sem_fd.get_semaphore_fd( + &vk::SemaphoreGetFdInfoKHR::default() + .semaphore(sem) + .handle_type(vk::ExternalSemaphoreHandleTypeFlags::OPAQUE_FD), + ) { + Ok(f) => f, + Err(e) => { + self.device.destroy_semaphore(sem, None); + return Err(e).context("vkGetSemaphoreFdKHR(timeline)"); + } + }; + let cuda_sem = match cuda::ExternalSemaphore::import_owned_timeline_fd(fd) { + Ok(c) => c, + Err(e) => { + self.device.destroy_semaphore(sem, None); + return Err(e); + } + }; + self.timeline = Some(Timeline { + sem, + ts: ash::khr::timeline_semaphore::Device::new(&self.instance, &self.device), + cuda: cuda_sem, + ticket: 0, + last_blend: 0, + }); + Ok(()) + } + } + /// The non-device objects: command machinery, cursor staging, descriptor + pipelines. fn init_objects(&mut self, qf: u32) -> Result<()> { // SAFETY: same contract as `new` — ash calls on the live `self.device` with builder infos @@ -246,14 +376,6 @@ impl VkSlotBlend { None, ) .context("create command pool")?; - self.cmd = d - .allocate_command_buffers( - &vk::CommandBufferAllocateInfo::default() - .command_pool(self.cmd_pool) - .level(vk::CommandBufferLevel::PRIMARY) - .command_buffer_count(1), - ) - .context("allocate command buffer")?[0]; self.fence = d .create_fence(&vk::FenceCreateInfo::default(), None) .context("create fence")?; @@ -320,25 +442,21 @@ impl VkSlotBlend { None, ) .context("create pipeline layout")?; + // Per-slot sets (one per ring slot, written once at `alloc_slot`; freed by + // `free_slots` on ring rebuilds, hence FREE_DESCRIPTOR_SET). 64 is comfortably + // above any ring depth (the encoder's POOL is 8). let pool_sizes = [vk::DescriptorPoolSize::default() .ty(vk::DescriptorType::STORAGE_BUFFER) - .descriptor_count(2)]; + .descriptor_count(128)]; self.desc_pool = d .create_descriptor_pool( &vk::DescriptorPoolCreateInfo::default() - .max_sets(1) + .flags(vk::DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET) + .max_sets(64) .pool_sizes(&pool_sizes), None, ) .context("create descriptor pool")?; - let dls = [self.desc_layout]; - self.desc_set = d - .allocate_descriptor_sets( - &vk::DescriptorSetAllocateInfo::default() - .descriptor_pool(self.desc_pool) - .set_layouts(&dls), - ) - .context("allocate descriptor set")?[0]; // The shader + one pipeline per MODE (spec constant 0). if CURSOR_SPV.len() % 4 != 0 { @@ -465,6 +583,59 @@ impl VkSlotBlend { return Err(e).context("cuImportExternalMemory(slot OPAQUE_FD)"); } }; + // The slot's own descriptor set + command buffer (see `SlotAlloc`). Bindings are + // written once here: binding 0 is this slot's buffer (immutable for its lifetime), + // binding 1 the shared cursor staging buffer. + let dls = [self.desc_layout]; + let desc = match d.allocate_descriptor_sets( + &vk::DescriptorSetAllocateInfo::default() + .descriptor_pool(self.desc_pool) + .set_layouts(&dls), + ) { + Ok(s) => s[0], + Err(e) => { + drop(ext); // CUDA's view of the memory goes first + d.free_memory(memory, None); + d.destroy_buffer(buffer, None); + return Err(e).context("allocate slot descriptor set"); + } + }; + let surf_info = [vk::DescriptorBufferInfo::default() + .buffer(buffer) + .offset(0) + .range(size)]; + let cur_info = [vk::DescriptorBufferInfo::default() + .buffer(self.cur_buf) + .offset(0) + .range((CURSOR_MAX * CURSOR_MAX * 4) as u64)]; + let writes = [ + vk::WriteDescriptorSet::default() + .dst_set(desc) + .dst_binding(0) + .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) + .buffer_info(&surf_info), + vk::WriteDescriptorSet::default() + .dst_set(desc) + .dst_binding(1) + .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) + .buffer_info(&cur_info), + ]; + d.update_descriptor_sets(&writes, &[]); + let cmd = match d.allocate_command_buffers( + &vk::CommandBufferAllocateInfo::default() + .command_pool(self.cmd_pool) + .level(vk::CommandBufferLevel::PRIMARY) + .command_buffer_count(1), + ) { + Ok(c) => c[0], + Err(e) => { + let _ = d.free_descriptor_sets(self.desc_pool, &[desc]); + drop(ext); // CUDA's view of the memory goes first + d.free_memory(memory, None); + d.destroy_buffer(buffer, None); + return Err(e).context("allocate slot command buffer"); + } + }; let r = VkSlotRef { ptr: ext.ptr, pitch: pitch as usize, @@ -475,7 +646,8 @@ impl VkSlotBlend { buffer, memory, cuda: ext, - size, + cmd, + desc, }); Ok(r) } @@ -485,12 +657,26 @@ impl VkSlotBlend { /// first (field order in [`SlotAlloc`] frees `cuda` via its own `Drop` before we free the VK /// objects explicitly here). pub fn free_slots(&mut self) { + // Ordered blends return with their work still on the queue — quiesce before freeing the + // buffers/sets it references. `device_wait_idle` (not a timeline wait) so a submission + // whose CUDA copy-done signal never fired is still covered; this tiny device idles in + // microseconds outside that pathological case. CPU-synced blends need nothing (they + // fence-wait before returning), so a `None` timeline skips it. + if self.timeline.is_some() && !self.slots.is_empty() { + // SAFETY: single-threaded owner; no other thread touches this device or its queue. + unsafe { + let _ = self.device.device_wait_idle(); + } + } for s in self.slots.drain(..) { drop(s.cuda); // CUDA's view of the memory goes first - // SAFETY: `buffer`/`memory` were created in `alloc_slot`, are uniquely owned by the - // drained `SlotAlloc`, and are destroyed exactly once here. No queue work can be - // in flight: every `blend` fence-waits before returning. + // SAFETY: `buffer`/`memory`/`cmd`/`desc` were created in `alloc_slot`, are uniquely + // owned by the drained `SlotAlloc`, and are destroyed exactly once here. No queue + // work is in flight: CPU-synced blends fence-wait before returning, and ordered + // blends were quiesced by the `device_wait_idle` above. unsafe { + let _ = self.device.free_descriptor_sets(self.desc_pool, &[s.desc]); + self.device.free_command_buffers(self.cmd_pool, &[s.cmd]); self.device.destroy_buffer(s.buffer, None); self.device.free_memory(s.memory, None); } @@ -498,21 +684,188 @@ impl VkSlotBlend { } /// Upload the cursor RGBA (`cw*ch*4`, tight rows) into the mapped staging buffer. Call only - /// when the bitmap changes; position moves are push constants. + /// when the bitmap changes; position moves are push constants. Quiesces any in-flight + /// ordered blend first (bitmap changes are rare — pointer-shape flips — so the occasional + /// bounded CPU wait here is the trade that keeps the per-frame path sync-free). pub fn upload_cursor(&mut self, rgba: &[u8], cw: u32, ch: u32) { + if let Some(t) = &self.timeline { + if t.last_blend > 0 { + let sems = [t.sem]; + let values = [t.last_blend]; + // SAFETY: `t.sem` is the live timeline semaphore; the wait info's arrays are + // locals outliving the synchronous call. `last_blend` only holds values an + // accepted submit will signal, so the wait terminates (the timeout is the + // backstop for a wedged queue — then we proceed and risk one torn cursor + // bitmap, never UB: the staging buffer stays alive regardless). + let r = unsafe { + t.ts.wait_semaphores( + &vk::SemaphoreWaitInfo::default() + .semaphores(&sems) + .values(&values), + 1_000_000_000, + ) + }; + if let Err(e) = r { + tracing::warn!( + error = ?e, + "cursor upload quiesce failed — proceeding (a torn cursor bitmap for \ + one frame at worst)" + ); + } + } + } let cw = cw.min(CURSOR_MAX); let ch = ch.min(CURSOR_MAX); let len = (cw * ch * 4) as usize; let len = len.min(rgba.len()); // SAFETY: `cur_map` is the live persistent mapping of the CURSOR_MAX²·4 host-coherent // allocation (created in `init_objects`, unmapped only in `Drop`); `len` is clamped to - // both the source slice and the buffer capacity. No blend is in flight (every `blend` - // fence-waits before returning), so no GPU read races this host write. + // both the source slice and the buffer capacity. No blend reads race this host write: + // CPU-synced blends fence-wait before returning, and ordered blends were quiesced via + // the timeline wait above. unsafe { std::ptr::copy_nonoverlapping(rgba.as_ptr(), self.cur_map, len); } } + /// Stream-ordered blends available: the timeline semaphore was exported to CUDA at bring-up, + /// so [`blend_ref_ordered`](Self::blend_ref_ordered) can order copy→blend→encode on-device. + pub fn ordered_ready(&self) -> bool { + self.timeline.is_some() + } + + /// Last timeline value handed out (0 = no ordered blend submitted yet) — test observability + /// for "did the ordered path actually run". + pub fn ordered_ticket(&self) -> u64 { + self.timeline.as_ref().map_or(0, |t| t.ticket) + } + + /// Push constants + dispatch geometry for one blend (must match cursor_blend.comp): ARGB = + /// per cursor px; NV12/YUV444 = per word-aligned 4-px span × (2-row blocks | rows). `None` + /// when the clamped cursor rect is empty (nothing to dispatch). + fn blend_geometry( + slot: &VkSlotRef, + fmt: SlotFormat, + surf_w: u32, + cw: u32, + ch: u32, + ox: i32, + oy: i32, + ) -> Option<(Push, u32, u32)> { + let cw = cw.min(CURSOR_MAX); + let ch = ch.min(CURSOR_MAX); + if cw == 0 || ch == 0 { + return None; + } + let push = Push { + pitch: slot.pitch as u32, + surf_w, + surf_h: slot.height, + cur_w: cw, + cur_h: ch, + ox, + oy, + }; + let (gx, gy) = match fmt { + SlotFormat::Argb => (cw.div_ceil(8), ch.div_ceil(8)), + _ => { + let x0 = (ox >> 2) << 2; + let spans = ((ox + cw as i32) - x0 + 3).div_euclid(4).max(1) as u32; + let rows = match fmt { + SlotFormat::Nv12 => ch.div_ceil(2), + _ => ch, + }; + (spans.div_ceil(8), rows.div_ceil(8)) + } + }; + Some((push, gx, gy)) + } + + /// Record the blend (barriers + bind + push constants + dispatch) into `id`'s own command + /// buffer and return it, ready for submission. + /// + /// # Safety + /// The slot's previous submission must have completed: sync blends fence-wait before + /// returning; ordered blends rely on the encoder's ring-reuse discipline (a slot is reused + /// only after its encode — GPU-ordered after its blend — was polled). + unsafe fn record_blend( + &self, + id: usize, + fmt: SlotFormat, + push: &Push, + gx: u32, + gy: u32, + ) -> Result { + let alloc = self + .slots + .get(id) + .ok_or_else(|| anyhow!("bad slot id {id}"))?; + let d = &self.device; + let cmd = alloc.cmd; + d.begin_command_buffer( + cmd, + &vk::CommandBufferBeginInfo::default() + .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), + ) + .context("begin blend cmd")?; + // CUDA wrote the frame into this memory outside Vulkan's view — make it visible to + // the shader (external-memory coherence ceremony; NVIDIA honors this with the + // fence/semaphore ordering alone, the barrier is the spec-shaped belt-and-braces). + let acquire = [vk::MemoryBarrier::default() + .src_access_mask(vk::AccessFlags::MEMORY_WRITE) + .dst_access_mask(vk::AccessFlags::SHADER_READ | vk::AccessFlags::SHADER_WRITE)]; + d.cmd_pipeline_barrier( + cmd, + vk::PipelineStageFlags::TOP_OF_PIPE, + vk::PipelineStageFlags::COMPUTE_SHADER, + vk::DependencyFlags::empty(), + &acquire, + &[], + &[], + ); + d.cmd_bind_pipeline( + cmd, + vk::PipelineBindPoint::COMPUTE, + self.pipelines[fmt.mode() as usize], + ); + d.cmd_bind_descriptor_sets( + cmd, + vk::PipelineBindPoint::COMPUTE, + self.pipe_layout, + 0, + &[alloc.desc], + &[], + ); + let bytes = std::slice::from_raw_parts( + (push as *const Push) as *const u8, + std::mem::size_of::(), + ); + d.cmd_push_constants( + cmd, + self.pipe_layout, + vk::ShaderStageFlags::COMPUTE, + 0, + bytes, + ); + d.cmd_dispatch(cmd, gx.max(1), gy.max(1), 1); + // Release the shader's writes so the downstream CUDA/NVENC reads (fence- or + // semaphore-ordered) see them. + let release = [vk::MemoryBarrier::default() + .src_access_mask(vk::AccessFlags::SHADER_WRITE) + .dst_access_mask(vk::AccessFlags::MEMORY_READ)]; + d.cmd_pipeline_barrier( + cmd, + vk::PipelineStageFlags::COMPUTE_SHADER, + vk::PipelineStageFlags::BOTTOM_OF_PIPE, + vk::DependencyFlags::empty(), + &release, + &[], + &[], + ); + d.end_command_buffer(cmd).context("end blend cmd")?; + Ok(cmd) + } + /// Blend the uploaded cursor into `slot` at `(ox, oy)`: record, submit, fence-wait. The /// caller has CPU-synced its CUDA frame copy first; the fence wait makes the shader's writes /// visible to the subsequent NVENC encode (the `VkBridge` precedent: fence-ordered cross-API @@ -528,129 +881,20 @@ impl VkSlotBlend { ox: i32, oy: i32, ) -> Result<()> { - let alloc = self - .slots - .get(slot.id) - .ok_or_else(|| anyhow!("bad slot id {}", slot.id))?; - let cw = cw.min(CURSOR_MAX); - let ch = ch.min(CURSOR_MAX); - if cw == 0 || ch == 0 { + let Some((push, gx, gy)) = Self::blend_geometry(slot, fmt, surf_w, cw, ch, ox, oy) else { return Ok(()); - } - let push = Push { - pitch: slot.pitch as u32, - surf_w, - surf_h: slot.height, - cur_w: cw, - cur_h: ch, - ox, - oy, }; - // Dispatch geometry (must match cursor_blend.comp): ARGB = per cursor px; NV12/YUV444 = - // per word-aligned 4-px span × (2-row blocks | rows). - let (gx, gy) = match fmt { - SlotFormat::Argb => (cw.div_ceil(8), ch.div_ceil(8)), - _ => { - let x0 = (ox >> 2) << 2; - let spans = ((ox + cw as i32) - x0 + 3).div_euclid(4).max(1) as u32; - let rows = match fmt { - SlotFormat::Nv12 => ch.div_ceil(2), - _ => ch, - }; - (spans.div_ceil(8), rows.div_ceil(8)) - } - }; - // SAFETY: single-threaded record/submit/wait on handles this struct owns. The descriptor - // update is safe because no prior submission is in flight (every blend fence-waits and - // the fence is reset before reuse). Buffer infos and barrier structs are locals outliving - // their synchronous calls. The dispatch's shader accesses stay in-bounds by the shader's - // own guards (surfW/surfH/curW/curH from `push`) against the slot allocation sized in - // `alloc_slot` for exactly that geometry. + // SAFETY: single-threaded record/submit/wait on handles this struct owns. The slot's + // previous submission completed (`record_blend`'s contract: this path fence-waits every + // blend, and an earlier ORDERED blend on this slot finished before the encoder reused it + // — ring-reuse discipline). Submit-info arrays are locals outliving the synchronous + // calls. The dispatch's shader accesses stay in-bounds by the shader's own guards + // (surfW/surfH/curW/curH from `push`) against the slot allocation sized in `alloc_slot` + // for exactly that geometry. unsafe { let d = &self.device; - let surf_info = [vk::DescriptorBufferInfo::default() - .buffer(alloc.buffer) - .offset(0) - .range(alloc.size)]; - let cur_info = [vk::DescriptorBufferInfo::default() - .buffer(self.cur_buf) - .offset(0) - .range((CURSOR_MAX * CURSOR_MAX * 4) as u64)]; - let writes = [ - vk::WriteDescriptorSet::default() - .dst_set(self.desc_set) - .dst_binding(0) - .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) - .buffer_info(&surf_info), - vk::WriteDescriptorSet::default() - .dst_set(self.desc_set) - .dst_binding(1) - .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) - .buffer_info(&cur_info), - ]; - d.update_descriptor_sets(&writes, &[]); - - d.begin_command_buffer( - self.cmd, - &vk::CommandBufferBeginInfo::default() - .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), - ) - .context("begin blend cmd")?; - // CUDA wrote the frame into this memory outside Vulkan's view — make it visible to - // the shader (external-memory coherence ceremony; NVIDIA honors this with the fence - // ordering alone, the barrier is the spec-shaped belt-and-braces). - let acquire = [vk::MemoryBarrier::default() - .src_access_mask(vk::AccessFlags::MEMORY_WRITE) - .dst_access_mask(vk::AccessFlags::SHADER_READ | vk::AccessFlags::SHADER_WRITE)]; - d.cmd_pipeline_barrier( - self.cmd, - vk::PipelineStageFlags::TOP_OF_PIPE, - vk::PipelineStageFlags::COMPUTE_SHADER, - vk::DependencyFlags::empty(), - &acquire, - &[], - &[], - ); - d.cmd_bind_pipeline( - self.cmd, - vk::PipelineBindPoint::COMPUTE, - self.pipelines[fmt.mode() as usize], - ); - d.cmd_bind_descriptor_sets( - self.cmd, - vk::PipelineBindPoint::COMPUTE, - self.pipe_layout, - 0, - &[self.desc_set], - &[], - ); - let bytes = std::slice::from_raw_parts( - (&push as *const Push) as *const u8, - std::mem::size_of::(), - ); - d.cmd_push_constants( - self.cmd, - self.pipe_layout, - vk::ShaderStageFlags::COMPUTE, - 0, - bytes, - ); - d.cmd_dispatch(self.cmd, gx.max(1), gy.max(1), 1); - // Release the shader's writes so the post-fence CUDA/NVENC reads see them. - let release = [vk::MemoryBarrier::default() - .src_access_mask(vk::AccessFlags::SHADER_WRITE) - .dst_access_mask(vk::AccessFlags::MEMORY_READ)]; - d.cmd_pipeline_barrier( - self.cmd, - vk::PipelineStageFlags::COMPUTE_SHADER, - vk::PipelineStageFlags::BOTTOM_OF_PIPE, - vk::DependencyFlags::empty(), - &release, - &[], - &[], - ); - d.end_command_buffer(self.cmd).context("end blend cmd")?; - let cmds = [self.cmd]; + let cmd = self.record_blend(slot.id, fmt, &push, gx, gy)?; + let cmds = [cmd]; let submit = [vk::SubmitInfo::default().command_buffers(&cmds)]; d.queue_submit(self.queue, &submit, self.fence) .context("submit blend")?; @@ -660,11 +904,121 @@ impl VkSlotBlend { } Ok(()) } + + /// Stream-ordered blend — no CPU sync anywhere (the fix for cursor frames serializing the + /// NVENC submit path). The caller has ENQUEUED (not synced) its CUDA frame copy on this + /// thread's copy stream; this method then + /// 1. enqueues a CUDA signal of `copy_done` on that stream (fires once the copy completes), + /// 2. submits the blend waiting `copy_done` and signaling `blend_done` on the Vulkan queue, + /// 3. enqueues a CUDA wait of `blend_done` — so later stream work (the encode, via the + /// session's IO-stream binding) orders after the blend's writes. + /// + /// Timeline values are fresh per call and never reused. Failure leaves no dangling waiter: + /// the CUDA wait is enqueued only after the submit was accepted, and an orphaned CUDA + /// signal is legal (timeline signals may skip values — a later, larger signal satisfies + /// any waiter). + #[allow(clippy::too_many_arguments)] // same unpacked kernel args as `blend_ref` + pub fn blend_ref_ordered( + &mut self, + slot: &VkSlotRef, + fmt: SlotFormat, + surf_w: u32, + cw: u32, + ch: u32, + ox: i32, + oy: i32, + ) -> Result<()> { + let Some((push, gx, gy)) = Self::blend_geometry(slot, fmt, surf_w, cw, ch, ox, oy) else { + return Ok(()); + }; + let (copy_done, blend_done, sem) = { + let t = self + .timeline + .as_mut() + .ok_or_else(|| anyhow!("ordered blend without timeline support"))?; + t.ticket += 2; + (t.ticket - 1, t.ticket, t.sem) + }; + // Signal FIRST: if this fails nothing was submitted and the fresh values are simply + // skipped. (The reverse order could wedge the queue — a submitted blend waiting on a + // copy-done signal that never got enqueued.) + self.timeline + .as_ref() + .expect("checked above") + .cuda + .signal(copy_done) + .context("cuSignalExternalSemaphoresAsync (copy done)")?; + // SAFETY: single-threaded record/submit on handles this struct owns. The slot's previous + // submission completed (`record_blend`'s contract — ring-reuse discipline, see above). + // All submit-info arrays and the timeline-submit chain are locals outliving the + // synchronous `queue_submit`. No fence: completion is observed through the timeline + // (the encode polls it via the CUDA wait below; teardown quiesces via + // `device_wait_idle`). + unsafe { + let cmd = self.record_blend(slot.id, fmt, &push, gx, gy)?; + let cmds = [cmd]; + let wait_sems = [sem]; + let wait_vals = [copy_done]; + let wait_stages = [vk::PipelineStageFlags::COMPUTE_SHADER]; + let sig_sems = [sem]; + let sig_vals = [blend_done]; + let mut tsi = vk::TimelineSemaphoreSubmitInfo::default() + .wait_semaphore_values(&wait_vals) + .signal_semaphore_values(&sig_vals); + let submit = [vk::SubmitInfo::default() + .command_buffers(&cmds) + .wait_semaphores(&wait_sems) + .wait_dst_stage_mask(&wait_stages) + .signal_semaphores(&sig_sems) + .push_next(&mut tsi)]; + self.device + .queue_submit(self.queue, &submit, vk::Fence::null()) + .context("submit ordered blend")?; + } + let t = self.timeline.as_mut().expect("checked above"); + // Only now: `blend_done` WILL be signaled, so quiesces may rely on it. + t.last_blend = blend_done; + if let Err(e) = t.cuda.wait(blend_done) { + // The blend is in flight but the encode would no longer order after it — restore + // the ordering with a bounded CPU wait (the CPU-synced path's cost, once). The + // frame still composites correctly. + let sems = [t.sem]; + let values = [blend_done]; + // SAFETY: live timeline semaphore; wait-info arrays are locals outliving the + // synchronous call; `blend_done` was accepted for signaling just above, so the + // wait terminates (timeout backstops a wedged queue). + let r = unsafe { + t.ts.wait_semaphores( + &vk::SemaphoreWaitInfo::default() + .semaphores(&sems) + .values(&values), + 1_000_000_000, + ) + }; + tracing::warn!( + error = %format!("{e:#}"), + cpu_wait = ?r, + "ordered cursor blend: CUDA-side wait enqueue failed — ordering restored with \ + a CPU wait this frame" + ); + } + Ok(()) + } } impl Drop for VkSlotBlend { fn drop(&mut self) { self.free_slots(); + if let Some(t) = self.timeline.take() { + drop(t.cuda); // CUDA's import of the semaphore goes first (mirrors the slot order) + // SAFETY: `sem` was created in `init_timeline`, is uniquely owned, and is destroyed + // exactly once here. No submission still references it: ordered blends only exist + // alongside allocated slots, and `free_slots` just quiesced (device_wait_idle) any + // in-flight work before this point. + unsafe { + self.device.destroy_semaphore(t.sem, None); + } + } // SAFETY: every handle below was created in `new`/`init_objects` (or is null from a // partial init — Vulkan destroy/free calls are defined no-ops on null handles) and is // uniquely owned; each is destroyed exactly once here, pipelines/layouts/pools before the