diff --git a/crates/pf-capture/src/linux/mod.rs b/crates/pf-capture/src/linux/mod.rs index ded153c8..5eae5cac 100644 --- a/crates/pf-capture/src/linux/mod.rs +++ b/crates/pf-capture/src/linux/mod.rs @@ -748,6 +748,9 @@ mod pipewire { /// (`ImportKind::Tiled444`) and feed NVENC native full-chroma YUV — takes precedence over /// `nv12` (a 4:4:4 session must never subsample). yuv444: bool, + /// The LINEAR (gamescope) NV12 compute CSC failed once — RGB for the rest of the stream + /// (T2.5b's per-stream fallback latch; cleared by the next stream's fresh `Ud`). + linear_nv12_failed: bool, /// Rate-limit counter for the latest-frame-only diagnostic log (see `.process`). dbg_log_n: u64, /// Cursor-as-metadata state, composited into the CPU de-pad path (see `consume_frame`). @@ -1352,15 +1355,17 @@ mod pipewire { // sample LINEAR). let modifier = (ud.modifier != 0).then_some(ud.modifier); if let Some(fourcc) = pf_frame::drm_fourcc(fmt) { - // GPU converts only on the tiled EGL/GL path (`modifier.is_some()`): a 4:4:4 - // session gets the planar-YUV444 convert (full chroma, takes precedence over - // NV12 — 4:4:4 must never subsample), otherwise `PUNKTFUNK_NV12` gets NV12 — - // both feed NVENC native YUV so it skips its internal RGB→YUV CSC. The - // LINEAR/Vulkan (gamescope) path stays RGB — its converts aren't wired here; - // a 4:4:4 session on LINEAR frames falls to the encoder's clear-error path - // (`want_444` with an RGB CUDA payload) rather than silently subsampling. + // GPU converts: a 4:4:4 session gets the planar-YUV444 convert on the tiled + // EGL/GL path (full chroma, takes precedence over NV12 — 4:4:4 must never + // subsample), otherwise `PUNKTFUNK_NV12` gets NV12 — tiled via the EGL/GL + // blit, LINEAR/gamescope via the Vulkan bridge's compute CSC (latency plan + // T2.5b) — so NVENC encodes native YUV and skips its internal RGB→YUV CSC on + // the contended SM. A 4:4:4 session on LINEAR frames has no convert and + // stays RGB, falling to the encoder's clear-error path (`want_444` with an + // RGB CUDA payload) rather than silently subsampling. A LINEAR NV12 convert + // failure latches RGB for the stream (mid-frame fallback, no drop). let yuv444 = ud.yuv444 && modifier.is_some(); - let nv12 = ud.nv12 && !yuv444 && modifier.is_some(); + let mut nv12 = ud.nv12 && !ud.yuv444; let imported = if let Some(m) = modifier { if yuv444 { importer.import_yuv444(&plane, w as u32, h as u32, fourcc, Some(m)) @@ -1369,7 +1374,20 @@ mod pipewire { } else { importer.import(&plane, w as u32, h as u32, fourcc, Some(m)) } + } else if nv12 && !ud.linear_nv12_failed { + match importer.import_linear_nv12(&plane, w as u32, h as u32) { + Ok(buf) => Ok(buf), + Err(e) => { + ud.linear_nv12_failed = true; + nv12 = false; + tracing::warn!(error = %format!("{e:#}"), + "LINEAR NV12 compute CSC failed — RGB for the rest of this \ + stream (NVENC does the CSC internally)"); + importer.import_linear(&plane, w as u32, h as u32) + } + } } else { + nv12 = false; importer.import_linear(&plane, w as u32, h as u32) }; match imported { @@ -1742,6 +1760,7 @@ mod pipewire { vaapi_passthrough, nv12: pf_zerocopy::nv12_enabled(), yuv444: want_444, + linear_nv12_failed: false, dbg_log_n: 0, cursor: CursorState::default(), }; diff --git a/crates/pf-zerocopy/src/imp/client.rs b/crates/pf-zerocopy/src/imp/client.rs index 3e66cf3d..3183b3e5 100644 --- a/crates/pf-zerocopy/src/imp/client.rs +++ b/crates/pf-zerocopy/src/imp/client.rs @@ -320,6 +320,17 @@ impl RemoteImporter { self.import_impl(plane, ImportKind::Linear, width, height, 0, None) } + /// Mirror of [`super::egl::EglImporter::import_linear_nv12`] (LINEAR dmabuf → Vulkan-bridge + /// compute CSC → two-plane NV12 buffer, latency plan T2.5b). + pub fn import_linear_nv12( + &mut self, + plane: &DmabufPlane, + width: u32, + height: u32, + ) -> Result { + self.import_impl(plane, ImportKind::LinearNv12, width, height, 0, None) + } + fn import_impl( &mut self, plane: &DmabufPlane, diff --git a/crates/pf-zerocopy/src/imp/cuda.rs b/crates/pf-zerocopy/src/imp/cuda.rs index ce1ab1bf..4e9a0e3f 100644 --- a/crates/pf-zerocopy/src/imp/cuda.rs +++ b/crates/pf-zerocopy/src/imp/cuda.rs @@ -1386,3 +1386,50 @@ pub fn copy_pitched_to_buffer( // within both. Wrapper → live table. unsafe { copy_blocking(©, "cuMemcpy2DAsync_v2(ext->dev)") } } + +/// De-stride an NV12 pair from an external mapping (the Vulkan bridge's exportable buffer after +/// its compute CSC — latency plan T2.5b) into a pooled two-plane NV12 [`DeviceBuffer`]: the Y +/// plane (`width` bytes × `height` rows) and the interleaved UV plane (`width` bytes × ⌈h/2⌉ +/// rows), each de-strided from `src_pitch` to the pool's own plane pitches. Same contract as +/// [`copy_pitched_to_buffer`]: the shared context must be current. +pub fn copy_pitched_nv12_to_buffer( + y_src: CUdeviceptr, + uv_src: CUdeviceptr, + src_pitch: usize, + dst: &DeviceBuffer, +) -> Result<()> { + let Some((uv_ptr, uv_pitch)) = dst.uv else { + anyhow::bail!("copy_pitched_nv12_to_buffer: destination is not an NV12 buffer"); + }; + let y = CUDA_MEMCPY2D { + srcMemoryType: CU_MEMORYTYPE_DEVICE, + srcDevice: y_src, + srcPitch: src_pitch, + dstMemoryType: CU_MEMORYTYPE_DEVICE, + dstDevice: dst.ptr, + dstPitch: dst.pitch, + WidthInBytes: dst.width as usize, + Height: dst.height as usize, + ..Default::default() + }; + let uv = CUDA_MEMCPY2D { + srcMemoryType: CU_MEMORYTYPE_DEVICE, + srcDevice: uv_src, + srcPitch: src_pitch, + dstMemoryType: CU_MEMORYTYPE_DEVICE, + dstDevice: uv_ptr, + dstPitch: uv_pitch, + // W/2 interleaved UV samples × 2 bytes = `width` bytes per row. + WidthInBytes: dst.width as usize, + Height: dst.height.div_ceil(2) as usize, + ..Default::default() + }; + // SAFETY: same contract as `copy_pitched_to_buffer` — the caller holds the shared context + // current; both `CUDA_MEMCPY2D`s are live locals describing spans inside the caller's live + // external mapping (`y_src`/`uv_src` at `src_pitch`) and `dst`'s live pooled planes; each + // `copy_blocking` synchronizes before returning. + unsafe { + copy_blocking(&y, "cuMemcpy2DAsync_v2(ext->dev nv12 Y)")?; + copy_blocking(&uv, "cuMemcpy2DAsync_v2(ext->dev nv12 UV)") + } +} diff --git a/crates/pf-zerocopy/src/imp/egl.rs b/crates/pf-zerocopy/src/imp/egl.rs index 941d6c3a..8e8a9060 100644 --- a/crates/pf-zerocopy/src/imp/egl.rs +++ b/crates/pf-zerocopy/src/imp/egl.rs @@ -504,6 +504,9 @@ pub struct EglImporter { /// created lazily on the first LINEAR frame, + the destination pool. vk: Option, linear_pool: Option, + /// NV12 twin of [`linear_pool`](Self::linear_pool) for the bridge's compute-CSC output + /// (T2.5b) — separate pools because a session may fall back RGB mid-stream. + linear_nv12_pool: Option, gbm: *mut c_void, render_fd: c_int, } @@ -647,6 +650,7 @@ impl EglImporter { yuv444_blit: None, vk: None, linear_pool: None, + linear_nv12_pool: None, gbm, render_fd, }) @@ -677,6 +681,38 @@ impl EglImporter { ) } + /// Like [`import_linear`](Self::import_linear), but the bridge's compute CSC converts to a + /// two-plane **NV12** buffer (latency plan T2.5b) — the gamescope/LINEAR analogue of + /// [`import_nv12`](Self::import_nv12), so NVENC encodes native YUV on the dedicated-session + /// path too instead of paying its internal RGB→YUV CSC on the contended SM. + pub fn import_linear_nv12( + &mut self, + plane: &DmabufPlane, + width: u32, + height: u32, + ) -> Result { + cuda::make_current()?; + if self + .linear_nv12_pool + .as_ref() + .map(|p| (p.width(), p.height())) + != Some((width, height)) + { + self.linear_nv12_pool = Some(cuda::BufferPool::new_nv12(width, height)?); + } + if self.vk.is_none() { + self.vk = Some(super::vulkan::VkBridge::new()?); + } + self.vk.as_mut().unwrap().import_linear_nv12( + plane.fd, + plane.offset, + plane.stride, + width, + height, + self.linear_nv12_pool.as_ref().unwrap(), + ) + } + /// Drop the Vulkan bridge's cached per-fd import (see [`super::vulkan::VkBridge::forget_fd`]). /// No-op when the bridge hasn't been built (tiled-only captures). pub fn forget_linear_fd(&mut self, fd: i32) { diff --git a/crates/pf-zerocopy/src/imp/mod.rs b/crates/pf-zerocopy/src/imp/mod.rs index 9d028e06..a0374cc0 100644 --- a/crates/pf-zerocopy/src/imp/mod.rs +++ b/crates/pf-zerocopy/src/imp/mod.rs @@ -166,6 +166,20 @@ impl Importer { } } + /// LINEAR dmabuf → Vulkan-bridge compute CSC → two-plane NV12 buffer (latency plan T2.5b — + /// the gamescope analogue of [`import_nv12`](Self::import_nv12)). + pub fn import_linear_nv12( + &mut self, + plane: &DmabufPlane, + width: u32, + height: u32, + ) -> anyhow::Result { + match self { + Importer::Remote(r) => r.import_linear_nv12(plane, width, height), + Importer::InProc(i) => i.import_linear_nv12(plane, width, height), + } + } + /// True once the worker process is gone/wedged (every further call fails fast). Always /// `false` in-process — an in-process driver fault doesn't return. pub fn dead(&self) -> bool { diff --git a/crates/pf-zerocopy/src/imp/proto.rs b/crates/pf-zerocopy/src/imp/proto.rs index 375e83c3..01635347 100644 --- a/crates/pf-zerocopy/src/imp/proto.rs +++ b/crates/pf-zerocopy/src/imp/proto.rs @@ -41,6 +41,10 @@ pub enum ImportKind { /// variants' wire tags must never shift — an old worker receiving this fails the decode and /// the import-fail machinery handles it like any other worker error. Tiled444, + /// LINEAR dmabuf → Vulkan-bridge compute CSC → two-plane NV12 CUDA buffer (latency plan + /// T2.5b — the gamescope analogue of [`TiledNv12`](Self::TiledNv12)). Appended last, same + /// wire-tag rule as [`Tiled444`](Self::Tiled444). + LinearNv12, } /// host → worker. diff --git a/crates/pf-zerocopy/src/imp/rgb2nv12_buf.comp b/crates/pf-zerocopy/src/imp/rgb2nv12_buf.comp new file mode 100644 index 00000000..fe809206 --- /dev/null +++ b/crates/pf-zerocopy/src/imp/rgb2nv12_buf.comp @@ -0,0 +1,77 @@ +#version 450 +// LINEAR BGRx dmabuf bytes -> NV12 (BT.709 limited range), BUFFER to BUFFER — the Vulkan-bridge +// CSC for the gamescope path (latency plan T2.5b). The bridge deals in VkBuffers (NVIDIA's EGL +// can't sample LINEAR and CUDA rejects raw dmabuf fds), so unlike `pf-encode`'s image-based +// `rgb2yuv.comp` sibling this reads packed texels straight out of the imported dmabuf buffer and +// writes the two NV12 planes into the exportable buffer CUDA reads. Same BT.709 coefficients as +// the sibling shader — keep the two in sync. +// +// One invocation converts a 4x2 luma block: 4 Y bytes = exactly one u32 word per row, and its +// two chroma samples = one u32 word — so every store is a whole word and no two invocations +// touch the same word (an 8-bit-storage-free way to write byte planes race-free). All pitches +// and the UV offset are in WORDS and must be word-aligned (the Rust side sizes them so). +// +// Rebuild: glslc rgb2nv12_buf.comp -o rgb2nv12_buf.spv +layout(local_size_x = 8, local_size_y = 8) in; + +layout(std430, binding = 0) readonly buffer Src { + uint spx[]; // packed 4-byte texels, byte order B,G,R,X (little-endian XRGB8888/ARGB8888) +}; +layout(std430, binding = 1) writeonly buffer Dst { + uint dw[]; // [0, uv_off_w): Y rows at y_pitch_w; [uv_off_w, ...): interleaved UV rows +}; + +layout(push_constant) uniform Push { + uint width; // source pixels + uint height; + uint src_off_w; // dmabuf plane offset, in words (offset bytes / 4) + uint src_pitch_w; // dmabuf row stride, in words + uint y_pitch_w; // dst Y row pitch, in words + uint uv_off_w; // dst UV plane offset, in words + uint uv_pitch_w; // dst UV row pitch, in words +} pc; + +// Edge-clamped fetch: alignment padding (x >= width from the 4-wide block, odd-height chroma) +// duplicates the last real texel instead of reading out of bounds. +vec3 texel(uint x, uint y) { + x = min(x, pc.width - 1u); + y = min(y, pc.height - 1u); + uint t = spx[pc.src_off_w + y * pc.src_pitch_w + x]; + // B,G,R,X byte order: r = bits 16..23, g = 8..15, b = 0..7. + return vec3(float((t >> 16) & 0xFFu), float((t >> 8) & 0xFFu), float(t & 0xFFu)) / 255.0; +} + +// BT.709 limited range — numerically identical to rgb2yuv.comp's lumaY/U/V, scaled to bytes. +uint lumaB(vec3 c) { + return uint(clamp(16.0 + 255.0 * (0.1826 * c.r + 0.6142 * c.g + 0.0620 * c.b) + 0.5, 0.0, 255.0)); +} + +void main() { + uint bx = gl_GlobalInvocationID.x; // 4-px column block + uint by = gl_GlobalInvocationID.y; // 2-row block + uint x0 = bx * 4u; + uint y0 = by * 2u; + if (x0 >= pc.width || y0 >= pc.height) { + return; + } + // Two Y words (4 bytes each, rows y0 and y0+1). + for (uint row = 0u; row < 2u; row++) { + uint y = y0 + row; + uint w = lumaB(texel(x0, y)) + | (lumaB(texel(x0 + 1u, y)) << 8) + | (lumaB(texel(x0 + 2u, y)) << 16) + | (lumaB(texel(x0 + 3u, y)) << 24); + dw[y * pc.y_pitch_w + bx] = w; + } + // One UV word: the block's two chroma samples (each a 2x2 average). + uint uvw = 0u; + for (uint s = 0u; s < 2u; s++) { + uint cx = x0 + s * 2u; + vec3 a = (texel(cx, y0) + texel(cx + 1u, y0) + texel(cx, y0 + 1u) + texel(cx + 1u, y0 + 1u)) + * 0.25; + uint U = uint(clamp(128.0 + 255.0 * (-0.1006 * a.r - 0.3386 * a.g + 0.4392 * a.b) + 0.5, 0.0, 255.0)); + uint V = uint(clamp(128.0 + 255.0 * (0.4392 * a.r - 0.3989 * a.g - 0.0403 * a.b) + 0.5, 0.0, 255.0)); + uvw |= (U | (V << 8)) << (16u * s); + } + dw[pc.uv_off_w + by * pc.uv_pitch_w + bx] = uvw; +} diff --git a/crates/pf-zerocopy/src/imp/rgb2nv12_buf.spv b/crates/pf-zerocopy/src/imp/rgb2nv12_buf.spv new file mode 100644 index 00000000..7e46d8dd Binary files /dev/null and b/crates/pf-zerocopy/src/imp/rgb2nv12_buf.spv differ diff --git a/crates/pf-zerocopy/src/imp/vulkan.rs b/crates/pf-zerocopy/src/imp/vulkan.rs index 560c1106..bcc76318 100644 --- a/crates/pf-zerocopy/src/imp/vulkan.rs +++ b/crates/pf-zerocopy/src/imp/vulkan.rs @@ -40,6 +40,20 @@ struct DstBuf { cuda: cuda::ExternalDmabuf, } +/// The lazy compute-CSC pipeline (`rgb2nv12_buf.comp`) for [`VkBridge::import_linear_nv12`]. +struct Csc { + module: vk::ShaderModule, + dset_layout: vk::DescriptorSetLayout, + playout: vk::PipelineLayout, + pipeline: vk::Pipeline, + dpool: vk::DescriptorPool, + dset: vk::DescriptorSet, +} + +/// The buffer-to-buffer RGB→NV12 compute shader (see `rgb2nv12_buf.comp` beside this file; +/// rebuild with `glslc rgb2nv12_buf.comp -o rgb2nv12_buf.spv`). +const CSC_SPV: &[u8] = include_bytes!("rgb2nv12_buf.spv"); + pub struct VkBridge { _entry: ash::Entry, instance: ash::Instance, @@ -52,6 +66,9 @@ pub struct VkBridge { mem_props: vk::PhysicalDeviceMemoryProperties, src_cache: HashMap, dst: Option, + /// Built on the first [`import_linear_nv12`](Self::import_linear_nv12); RGB-only bridges + /// never pay for it. + csc: Option, } // SAFETY: `VkBridge` owns ash Vulkan handles (instance/device/queue/command pool+buffer/fence), a @@ -94,18 +111,15 @@ impl VkBridge { .ok_or_else(|| anyhow!("no NVIDIA Vulkan device"))?; let mem_props = instance.get_physical_device_memory_properties(phys); - // Any queue family supporting transfer (graphics/compute imply it). + // A COMPUTE-capable family (compute implies transfer): the copy path only needs + // transfer, but the NV12 CSC dispatch (T2.5b) needs compute — on every NVIDIA + // device family 0 is graphics+compute+transfer, so this picks the same family the + // old transfer-only predicate did. let qf = instance .get_physical_device_queue_family_properties(phys) .iter() - .position(|q| { - q.queue_flags.intersects( - vk::QueueFlags::TRANSFER - | vk::QueueFlags::GRAPHICS - | vk::QueueFlags::COMPUTE, - ) - }) - .ok_or_else(|| anyhow!("no transfer-capable queue family"))? + .position(|q| q.queue_flags.contains(vk::QueueFlags::COMPUTE)) + .ok_or_else(|| anyhow!("no compute-capable queue family"))? as u32; let exts = [ @@ -161,6 +175,7 @@ impl VkBridge { mem_props, src_cache: HashMap::new(), dst: None, + csc: None, }) } } @@ -189,7 +204,11 @@ impl VkBridge { .create_buffer( &vk::BufferCreateInfo::default() .size(size) - .usage(vk::BufferUsageFlags::TRANSFER_SRC) + // STORAGE so the NV12 compute CSC can read it as an SSBO (T2.5b); harmless + // for the plain copy path. + .usage( + vk::BufferUsageFlags::TRANSFER_SRC | vk::BufferUsageFlags::STORAGE_BUFFER, + ) .push_next(&mut ext_info), None, ) @@ -256,7 +275,10 @@ impl VkBridge { .create_buffer( &vk::BufferCreateInfo::default() .size(size) - .usage(vk::BufferUsageFlags::TRANSFER_DST) + // STORAGE so the NV12 compute CSC can write it as an SSBO (T2.5b). + .usage( + vk::BufferUsageFlags::TRANSFER_DST | vk::BufferUsageFlags::STORAGE_BUFFER, + ) .push_next(&mut ext_info), None, ) @@ -302,6 +324,246 @@ impl VkBridge { Ok(()) } + /// Build the RGB→NV12 compute pipeline once (T2.5b): two-SSBO descriptor set + a 28-byte + /// push-constant block matching `rgb2nv12_buf.comp`'s `Push`. + unsafe fn ensure_csc(&mut self) -> Result<()> { + if self.csc.is_some() { + return Ok(()); + } + let words: Vec = CSC_SPV + .chunks_exact(4) + .map(|c| u32::from_le_bytes(c.try_into().unwrap())) + .collect(); + let module = self + .device + .create_shader_module(&vk::ShaderModuleCreateInfo::default().code(&words), None) + .context("create CSC shader module")?; + let bindings = [ + vk::DescriptorSetLayoutBinding::default() + .binding(0) + .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) + .descriptor_count(1) + .stage_flags(vk::ShaderStageFlags::COMPUTE), + vk::DescriptorSetLayoutBinding::default() + .binding(1) + .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) + .descriptor_count(1) + .stage_flags(vk::ShaderStageFlags::COMPUTE), + ]; + let dset_layout = self + .device + .create_descriptor_set_layout( + &vk::DescriptorSetLayoutCreateInfo::default().bindings(&bindings), + None, + ) + .context("create CSC dset layout")?; + let pc = [vk::PushConstantRange::default() + .stage_flags(vk::ShaderStageFlags::COMPUTE) + .size(28)]; + let layouts = [dset_layout]; + let playout = self + .device + .create_pipeline_layout( + &vk::PipelineLayoutCreateInfo::default() + .set_layouts(&layouts) + .push_constant_ranges(&pc), + None, + ) + .context("create CSC pipeline layout")?; + let entry = c"main"; + let stage = vk::PipelineShaderStageCreateInfo::default() + .stage(vk::ShaderStageFlags::COMPUTE) + .module(module) + .name(entry); + let pipeline = self + .device + .create_compute_pipelines( + vk::PipelineCache::null(), + &[vk::ComputePipelineCreateInfo::default() + .stage(stage) + .layout(playout)], + None, + ) + .map_err(|(_, e)| anyhow!("create CSC pipeline: {e}"))?[0]; + let sizes = [vk::DescriptorPoolSize::default() + .ty(vk::DescriptorType::STORAGE_BUFFER) + .descriptor_count(2)]; + let dpool = self + .device + .create_descriptor_pool( + &vk::DescriptorPoolCreateInfo::default() + .max_sets(1) + .pool_sizes(&sizes), + None, + ) + .context("create CSC descriptor pool")?; + let dset = self + .device + .allocate_descriptor_sets( + &vk::DescriptorSetAllocateInfo::default() + .descriptor_pool(dpool) + .set_layouts(&layouts), + ) + .context("allocate CSC descriptor set")?[0]; + self.csc = Some(Csc { + module, + dset_layout, + playout, + pipeline, + dpool, + dset, + }); + tracing::info!("Vulkan-bridge NV12 compute CSC ready (LINEAR path feeds NVENC native YUV)"); + Ok(()) + } + + /// Bridge one LINEAR dmabuf frame into a pooled NV12 CUDA buffer (latency plan T2.5b): + /// instead of the plain byte copy, the compute CSC reads the imported RGB texels and writes + /// both NV12 planes into the exportable buffer, so NVENC on the gamescope path encodes + /// native YUV (its internal RGB→YUV CSC on the contended SM disappears). `pool` must be an + /// NV12 pool ([`cuda::BufferPool::new_nv12`]). + pub fn import_linear_nv12( + &mut self, + fd: i32, + offset: u32, + stride: u32, + width: u32, + height: u32, + pool: &cuda::BufferPool, + ) -> Result { + anyhow::ensure!( + offset % 4 == 0 && stride % 4 == 0, + "LINEAR dmabuf offset/stride not word-aligned ({offset}/{stride})" + ); + // Exportable-buffer NV12 layout the shader writes: 4-aligned Y pitch, UV plane (⌈h/2⌉ + // rows at the same pitch) directly after the Y plane. + let y_pitch = (width as u64 + 3) & !3; + let uv_off = y_pitch * height as u64; + let dst_size = uv_off + y_pitch * height.div_ceil(2) as u64; + // SAFETY: same structure and proofs as `import_linear` — `fd` is the caller's live dmabuf + // (dup'd by `import_src`), sizes are checked (`import_src` asserts the fd covers + // `offset + stride*height`; `ensure_dst(dst_size)` makes the exportable buffer at least + // the shader's whole write range, whose last word is `dst_size - 4`). The descriptor + // update binds the live cached src buffer and the live dst buffer WHOLE_SIZE; every + // `*Info`/array is a local outliving its synchronous call; `cmd`/`queue`/`fence` are this + // bridge's own single-thread handles. The dispatch covers ⌈w/32⌉×⌈h/16⌉ groups of 8×8 + // invocations, each writing only whole words inside the proven dst range (shader + // contract). The host `wait_for_fences` retires the compute pass (with a shader-write → + // memory barrier recorded before end) BEFORE CUDA reads the shared memory. + unsafe { + let span = offset as u64 + stride as u64 * height as u64; + if !self.src_cache.contains_key(&fd) { + let size = libc::lseek(fd, 0, libc::SEEK_END); + anyhow::ensure!(size > 0, "lseek(dmabuf)"); + anyhow::ensure!(size as u64 >= span, "dmabuf smaller than frame span"); + self.import_src(fd, size as u64)?; + } + let src_buffer = self.src_cache[&fd].buffer; + self.ensure_dst(dst_size)?; + self.ensure_csc()?; + let (dst_buffer, dst_cuda_ptr) = { + let d = self.dst.as_ref().unwrap(); + (d.buffer, d.cuda.ptr) + }; + let csc = self.csc.as_ref().unwrap(); + + let src_info = [vk::DescriptorBufferInfo::default() + .buffer(src_buffer) + .range(vk::WHOLE_SIZE)]; + let dst_info = [vk::DescriptorBufferInfo::default() + .buffer(dst_buffer) + .range(vk::WHOLE_SIZE)]; + let writes = [ + vk::WriteDescriptorSet::default() + .dst_set(csc.dset) + .dst_binding(0) + .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) + .buffer_info(&src_info), + vk::WriteDescriptorSet::default() + .dst_set(csc.dset) + .dst_binding(1) + .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) + .buffer_info(&dst_info), + ]; + self.device.update_descriptor_sets(&writes, &[]); + + self.device + .begin_command_buffer( + self.cmd, + &vk::CommandBufferBeginInfo::default() + .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), + ) + .context("begin cmd")?; + self.device + .cmd_bind_pipeline(self.cmd, vk::PipelineBindPoint::COMPUTE, csc.pipeline); + self.device.cmd_bind_descriptor_sets( + self.cmd, + vk::PipelineBindPoint::COMPUTE, + csc.playout, + 0, + &[csc.dset], + &[], + ); + let push: [u32; 7] = [ + width, + height, + offset / 4, + stride / 4, + (y_pitch / 4) as u32, + (uv_off / 4) as u32, + (y_pitch / 4) as u32, + ]; + let push_bytes: &[u8] = std::slice::from_raw_parts(push.as_ptr().cast(), 28); + self.device.cmd_push_constants( + self.cmd, + csc.playout, + vk::ShaderStageFlags::COMPUTE, + 0, + push_bytes, + ); + self.device + .cmd_dispatch(self.cmd, width.div_ceil(32), height.div_ceil(16), 1); + // Make the shader writes available before the external (CUDA) read. + let barrier = vk::MemoryBarrier::default() + .src_access_mask(vk::AccessFlags::SHADER_WRITE) + .dst_access_mask(vk::AccessFlags::MEMORY_READ); + self.device.cmd_pipeline_barrier( + self.cmd, + vk::PipelineStageFlags::COMPUTE_SHADER, + vk::PipelineStageFlags::BOTTOM_OF_PIPE, + vk::DependencyFlags::empty(), + &[barrier], + &[], + &[], + ); + self.device + .end_command_buffer(self.cmd) + .context("end cmd")?; + let cmds = [self.cmd]; + let submit = vk::SubmitInfo::default().command_buffers(&cmds); + self.device + .queue_submit(self.queue, &[submit], self.fence) + .context("queue submit")?; + self.device + .wait_for_fences(&[self.fence], true, 1_000_000_000) + .context("fence wait")?; + self.device + .reset_fences(&[self.fence]) + .context("reset fence")?; + + // De-stride both NV12 planes from the CUDA view into a pooled two-plane buffer. + cuda::make_current()?; + let out = pool.get()?; + cuda::copy_pitched_nv12_to_buffer( + dst_cuda_ptr, + dst_cuda_ptr + uv_off, + y_pitch as usize, + &out, + )?; + Ok(out) + } + } + /// Drop the cached import for `fd` (the PipeWire buffer it wrapped is gone — pool recycle / /// renegotiation — or the caller is about to store a different dmabuf under the same slot). /// Without this the cache could serve a stale imported buffer for a reused fd number, or @@ -414,6 +676,14 @@ impl Drop for VkBridge { self.device.destroy_buffer(d.buffer, None); self.device.free_memory(d.memory, None); } + if let Some(c) = self.csc.take() { + self.device.destroy_pipeline(c.pipeline, None); + self.device.destroy_pipeline_layout(c.playout, None); + self.device.destroy_descriptor_pool(c.dpool, None); // frees `c.dset` with it + self.device + .destroy_descriptor_set_layout(c.dset_layout, None); + self.device.destroy_shader_module(c.module, None); + } self.device.destroy_fence(self.fence, None); self.device.destroy_command_pool(self.cmd_pool, None); self.device.destroy_device(None); diff --git a/crates/pf-zerocopy/src/imp/worker.rs b/crates/pf-zerocopy/src/imp/worker.rs index c51de928..d6ce0a3a 100644 --- a/crates/pf-zerocopy/src/imp/worker.rs +++ b/crates/pf-zerocopy/src/imp/worker.rs @@ -299,6 +299,9 @@ impl EglBackend { req.modifier, )?, ImportKind::Linear => self.importer.import_linear(&plane, req.width, req.height)?, + ImportKind::LinearNv12 => self + .importer + .import_linear_nv12(&plane, req.width, req.height)?, }; // Assign / look up the buffer's id and export its CUDA IPC identity on first delivery. cuda::make_current()?;