From 0dacb37088672b98c273ee7e5d6076585c94185f Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 10 Jul 2026 18:08:41 +0200 Subject: [PATCH] =?UTF-8?q?feat(linux):=20zero-copy=204:4:4=20=E2=80=94=20?= =?UTF-8?q?the=20EGL=20worker=20converts=20to=20planar=20YUV444=20on=20the?= =?UTF-8?q?=20GPU?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 4:4:4 session no longer falls to the CPU path (SHM capture + swscale RGB→YUV444P + re-upload — the fps-ceiling triple tax). The zero-copy worker grows a Yuv444Blit: three full-res R8 GL passes (the proven BT.709 coefficients; studio or full range per PUNKTFUNK_444_FULLRANGE, read by both processes so pixels and VUI flip together) into ONE stacked 3-plane pitched CUDA allocation — which keeps the worker↔host wire and IPC single-plane. The encoder copies the planes into ffmpeg's yuv444p CUDA surface and hevc_nvenc emits Range-Extensions 4:4:4 natively. ImportKind::Tiled444 is APPENDED to the worker protocol (a worker outliving a replaced host binary must keep the old tags stable; an old worker just errors the import, which the fail machinery already handles). A 4:4:4 session on a LINEAR/gamescope capture — no convert wired there — fails with a clear message instead of letting hevc_nvenc silently subsample. caps().chroma_444 now keys off the session (it missed the GPU path when keyed off the swscale's existence). Live-verified on the CachyOS VM (RTX 5070 Ti): per-frame "imported to CUDA yuv444=true", stream Rext/yuv444p/bt709 in both tv and pc range, no CPU-path warning. Co-Authored-By: Claude Fable 5 --- crates/punktfunk-host/src/capture.rs | 18 +- .../punktfunk-host/src/capture/linux/mod.rs | 74 ++++-- crates/punktfunk-host/src/encode/linux/mod.rs | 65 +++-- .../punktfunk-host/src/encode/linux/vaapi.rs | 2 +- crates/punktfunk-host/src/encode/sw.rs | 4 +- .../src/encode/windows/ffmpeg_win.rs | 2 +- .../src/linux/zerocopy/client.rs | 15 ++ .../punktfunk-host/src/linux/zerocopy/cuda.rs | 124 +++++++++ .../punktfunk-host/src/linux/zerocopy/egl.rs | 244 +++++++++++++++++- .../punktfunk-host/src/linux/zerocopy/mod.rs | 21 +- .../src/linux/zerocopy/proto.rs | 7 +- .../src/linux/zerocopy/worker.rs | 7 + crates/punktfunk-host/src/session_plan.rs | 22 +- 13 files changed, 532 insertions(+), 73 deletions(-) diff --git a/crates/punktfunk-host/src/capture.rs b/crates/punktfunk-host/src/capture.rs index a92b6d07..b4f6ad88 100644 --- a/crates/punktfunk-host/src/capture.rs +++ b/crates/punktfunk-host/src/capture.rs @@ -37,12 +37,19 @@ pub enum PixelFormat { /// `P010` (DXGI `P010`): 10-bit BT.2020 PQ limited-range YUV 4:2:0. HDR analogue of [`Nv12`]: /// video-processor output for HEVC Main10 / HDR10, handed to NVENC as `YUV420_10BIT`. P010, + /// Planar 8-bit YUV **4:4:4** (BT.709; range per `PUNKTFUNK_444_FULLRANGE`). Produced by the + /// Linux zero-copy worker's GPU convert for a 4:4:4 session ([`FramePayload::Cuda`] with + /// `DeviceBuffer::yuv444` — three full-res planes stacked in one allocation); NVENC encodes + /// it natively under the Range-Extensions profile. Never a CPU payload. + Yuv444, } impl PixelFormat { pub fn bytes_per_pixel(self) -> usize { match self { PixelFormat::Rgb | PixelFormat::Bgr => 3, + // Three full-res 1-byte planes (GPU-resident only; no CPU payload carries this). + PixelFormat::Yuv444 => 3, _ => 4, } } @@ -381,11 +388,12 @@ pub fn capture_virtual_output( _capture: crate::session_plan::CaptureBackend, ) -> Result> { // The Linux host stays 8-bit (HDR is blocked upstream) and the portal negotiates its own pixel - // format, so only `want.gpu` is honored here: it gates GPU zero-copy capture (the capture backend - // is always the portal — the `CaptureBackend` arg is a Windows-only dispatch). `gpu = false` - // (a 4:4:4 NVENC session) forces the CPU mmap path so the encoder gets CPU-resident RGB to swscale - // into YUV444P — otherwise it would receive CUDA frames and bail. - linux::PortalCapturer::from_virtual_output(vout, want.gpu) + // format, so `want.gpu` gates GPU zero-copy capture (the capture backend is always the portal — + // the `CaptureBackend` arg is a Windows-only dispatch) and `want.chroma_444` selects the + // worker's planar-YUV444 GPU convert for tiled dmabufs (the 4:4:4 zero-copy path). `gpu = + // false` (4:4:4 without zero-copy) forces the CPU mmap path so the encoder gets CPU-resident + // RGB to swscale into YUV444P. + linux::PortalCapturer::from_virtual_output(vout, want.gpu, want.chroma_444) .map(|c| Box::new(c) as Box) } diff --git a/crates/punktfunk-host/src/capture/linux/mod.rs b/crates/punktfunk-host/src/capture/linux/mod.rs index 1ad18687..4ec46455 100644 --- a/crates/punktfunk-host/src/capture/linux/mod.rs +++ b/crates/punktfunk-host/src/capture/linux/mod.rs @@ -101,30 +101,37 @@ impl PortalCapturer { "ScreenCast portal session started; connecting PipeWire" ); // This portal path (GameStream / monitor capture) is always 4:2:0, so allow zero-copy as before. - Ok(spawn_pipewire(Some(fd), node_id, None, true)?.into_capturer(node_id, None)) + Ok(spawn_pipewire(Some(fd), node_id, None, true, false)?.into_capturer(node_id, None)) } /// Build a capturer from an already-created virtual output ([`crate::vdisplay::VirtualOutput`]): /// connect PipeWire to its node (`remote_fd` selects portal-remote vs. default-daemon) and /// take ownership of its keepalive so the output lives exactly as long as this capturer. This /// is how the client's requested resolution becomes the captured resolution without scaling. - /// `allow_zerocopy` mirrors [`OutputFormat::gpu`](crate::capture::OutputFormat): `false` forces the - /// CPU mmap path (a 4:4:4 NVENC session needs CPU-resident RGB), `true` keeps the GPU zero-copy - /// path subject to `PUNKTFUNK_ZEROCOPY`. + /// `allow_zerocopy` mirrors [`OutputFormat::gpu`](crate::capture::OutputFormat): `false` forces + /// the CPU mmap path, `true` keeps the GPU zero-copy path subject to `PUNKTFUNK_ZEROCOPY`. + /// `want_444` (a 4:4:4 session) makes the zero-copy worker convert tiled dmabufs to planar + /// YUV444 on the GPU instead of NV12/RGB. pub fn from_virtual_output( vout: crate::vdisplay::VirtualOutput, allow_zerocopy: bool, + want_444: bool, ) -> Result { tracing::info!( node_id = vout.node_id, allow_zerocopy, + want_444, "connecting PipeWire to virtual output" ); let node_id = vout.node_id; - Ok( - spawn_pipewire(vout.remote_fd, node_id, vout.preferred_mode, allow_zerocopy)? - .into_capturer(node_id, Some(vout.keepalive)), - ) + Ok(spawn_pipewire( + vout.remote_fd, + node_id, + vout.preferred_mode, + allow_zerocopy, + want_444, + )? + .into_capturer(node_id, Some(vout.keepalive))) } } @@ -173,11 +180,12 @@ fn spawn_pipewire( node_id: u32, preferred: Option<(u32, u32, u32)>, // Allow GPU zero-copy capture (dmabuf→CUDA/VA). `false` forces the CPU mmap path even when - // `PUNKTFUNK_ZEROCOPY` is set — a 4:4:4 NVENC session needs CPU-resident RGB (the encoder - // swscales RGB→YUV444P; `hevc_nvenc` can't 4:4:4 from a CUDA RGB surface), so the session plan - // passes `gpu = false` for it. Without this, a 4:4:4 session under `PUNKTFUNK_ZEROCOPY=1` would - // get CUDA frames and the encoder would bail (`want_444 && cuda`). + // `PUNKTFUNK_ZEROCOPY` is set (the session plan passes `gpu = false` when 4:4:4 has no + // zero-copy convert available — see `SessionPlan::output_format`). allow_zerocopy: bool, + // 4:4:4 session: tiled dmabufs convert to planar YUV444 on the GPU (`ImportKind::Tiled444`) + // instead of NV12/RGB, so the session stays zero-copy at full chroma. + want_444: bool, ) -> Result { // Frames flow from the pipewire thread over a small bounded channel. let (frame_tx, frame_rx) = sync_channel::(8); @@ -212,6 +220,7 @@ fn spawn_pipewire( streaming_cb, broken_cb, zerocopy, + want_444, preferred, quit_rx, ) { @@ -609,6 +618,10 @@ mod pipewire { /// `PUNKTFUNK_NV12`: on the tiled EGL/GL zero-copy path, convert to NV12 on the GPU and feed /// NVENC native YUV (Tier 2A). Off ⇒ the BGRx path is unchanged. nv12: bool, + /// 4:4:4 session: on the tiled EGL/GL zero-copy path, convert to planar YUV444 on the GPU + /// (`ImportKind::Tiled444`) and feed NVENC native full-chroma YUV — takes precedence over + /// `nv12` (a 4:4:4 session must never subsample). + yuv444: bool, /// Rate-limit counter for the latest-frame-only diagnostic log (see `.process`). dbg_log_n: u64, } @@ -1012,13 +1025,19 @@ mod pipewire { // sample LINEAR). let modifier = (ud.modifier != 0).then_some(ud.modifier); if let Some(fourcc) = crate::zerocopy::drm_fourcc(fmt) { - // NV12 convert (Tier 2A) only on the tiled EGL/GL path (`modifier.is_some()`): - // produce native YUV so NVENC skips its internal RGB→YUV CSC. The LINEAR/Vulkan - // (gamescope) path stays RGB — its convert isn't wired here. When NV12 is - // produced the frame's format is reported as `Nv12` so the encoder opens native. - let nv12 = ud.nv12 && modifier.is_some(); + // GPU converts only on the tiled EGL/GL path (`modifier.is_some()`): a 4:4:4 + // session gets the planar-YUV444 convert (full chroma, takes precedence over + // NV12 — 4:4:4 must never subsample), otherwise `PUNKTFUNK_NV12` gets NV12 — + // both feed NVENC native YUV so it skips its internal RGB→YUV CSC. The + // LINEAR/Vulkan (gamescope) path stays RGB — its converts aren't wired here; + // a 4:4:4 session on LINEAR frames falls to the encoder's clear-error path + // (`want_444` with an RGB CUDA payload) rather than silently subsampling. + let yuv444 = ud.yuv444 && modifier.is_some(); + let nv12 = ud.nv12 && !yuv444 && modifier.is_some(); let imported = if let Some(m) = modifier { - if nv12 { + if yuv444 { + importer.import_yuv444(&plane, w as u32, h as u32, fourcc, Some(m)) + } else if nv12 { importer.import_nv12(&plane, w as u32, h as u32, fourcc, Some(m)) } else { importer.import(&plane, w as u32, h as u32, fourcc, Some(m)) @@ -1038,6 +1057,7 @@ mod pipewire { h, modifier = ud.modifier, nv12, + yuv444, "zero-copy: dmabuf imported to CUDA (no CPU copy)" ); } @@ -1049,7 +1069,13 @@ mod pipewire { width: w as u32, height: h as u32, pts_ns, - format: if nv12 { PixelFormat::Nv12 } else { fmt }, + format: if yuv444 { + PixelFormat::Yuv444 + } else if nv12 { + PixelFormat::Nv12 + } else { + fmt + }, payload: FramePayload::Cuda(devbuf), }); return; @@ -1225,6 +1251,8 @@ mod pipewire { streaming: Arc, broken: Arc, zerocopy: bool, + // 4:4:4 session: tiled dmabufs take the worker's planar-YUV444 GPU convert. + want_444: bool, preferred: Option<(u32, u32, u32)>, quit_rx: pw::channel::Receiver<()>, ) -> Result<()> { @@ -1332,7 +1360,12 @@ mod pipewire { } ); } - if want_dmabuf && !vaapi_passthrough && crate::zerocopy::nv12_enabled() { + if want_dmabuf && !vaapi_passthrough && want_444 { + tracing::info!( + "4:4:4 zero-copy: tiled dmabufs convert to planar YUV444 (BT.709) on the GPU — \ + NVENC fed native full-chroma YUV, no CPU pixel path" + ); + } else if want_dmabuf && !vaapi_passthrough && crate::zerocopy::nv12_enabled() { tracing::info!( "PUNKTFUNK_NV12: tiled dmabufs convert to NV12 (BT.709 limited) on the GPU — NVENC \ fed native YUV (no internal RGB→YUV CSC)" @@ -1352,6 +1385,7 @@ mod pipewire { importer, vaapi_passthrough, nv12: crate::zerocopy::nv12_enabled(), + yuv444: want_444, dbg_log_n: 0, }; diff --git a/crates/punktfunk-host/src/encode/linux/mod.rs b/crates/punktfunk-host/src/encode/linux/mod.rs index 53c5e2bb..615d6e83 100644 --- a/crates/punktfunk-host/src/encode/linux/mod.rs +++ b/crates/punktfunk-host/src/encode/linux/mod.rs @@ -40,7 +40,7 @@ fn sws_src_pixel(format: PixelFormat) -> Result { PixelFormat::Rgba => Pixel::RGBA, PixelFormat::Rgb => Pixel::RGB24, PixelFormat::Bgr => Pixel::BGR24, - PixelFormat::Nv12 | PixelFormat::P010 | PixelFormat::Rgb10a2 => { + PixelFormat::Nv12 | PixelFormat::P010 | PixelFormat::Rgb10a2 | PixelFormat::Yuv444 => { bail!("NVENC 4:4:4 CPU-input path supports packed RGB/BGR only; got {format:?}") } }) @@ -139,6 +139,9 @@ fn nvenc_input(format: PixelFormat) -> (Pixel, bool) { // Linux it's produced by the GPU convert on the zero-copy tiled path (`PUNKTFUNK_NV12`); on // Windows by the D3D11 video processor. PixelFormat::Nv12 => (Pixel::NV12, false), + // Planar YUV444 from the zero-copy worker's GPU convert (a 4:4:4 session) — native + // full-chroma YUV in, `hevc_nvenc` emits Range-Extensions 4:4:4. + PixelFormat::Yuv444 => (Pixel::YUV444P, false), // Rgb10a2 (HDR) and P010 (the Windows 10-bit video-processor output) are produced only by // the Windows paths; the Linux capturer never emits them. Map to BGRA so the match is // exhaustive — unreachable here. @@ -155,10 +158,13 @@ pub struct NvencEncoder { frame: Option, /// Zero-copy path: CUDA hwdevice/hwframes contexts (the encoder takes `AV_PIX_FMT_CUDA`). cuda: Option, - /// 4:4:4 path only: swscale context converting the captured packed RGB/BGR → planar YUV444P - /// (BT.709 limited) into [`Self::frame`], because `hevc_nvenc` only emits 4:4:4 from a YUV444 - /// *input* (RGB-in is always 4:2:0). `None` on the ordinary 4:2:0 RGB path. Freed in `Drop`. + /// 4:4:4 CPU path only: swscale context converting the captured packed RGB/BGR → planar + /// YUV444P into [`Self::frame`], because `hevc_nvenc` only emits 4:4:4 from a YUV444 *input* + /// (RGB-in is always 4:2:0). `None` on the 4:2:0 paths AND on the zero-copy 4:4:4 path (the + /// worker's GPU convert delivers YUV444 CUDA frames). Freed in `Drop`. sws_444: Option<*mut ffi::SwsContext>, + /// This session opened as full-chroma 4:4:4 (FREXT) — via either input path. + want_444: bool, src_format: PixelFormat, expand: bool, width: u32, @@ -241,16 +247,10 @@ impl NvencEncoder { } // Full-chroma 4:4:4 (HEVC Range Extensions). `hevc_nvenc` only emits 4:4:4 from a YUV444 // *input* frame — feeding RGB always subsamples to 4:2:0 regardless of profile (verified on - // the RTX 5070 Ti). So a 4:4:4 session swscales the captured RGB → YUV444P (BT.709 limited) - // and feeds that with `profile=rext`. The negotiator gates this to HEVC + the single-process - // CPU-capture topology, so `cuda` must be false here; defend the contract. + // the RTX 5070 Ti). Two ways to produce that input: the zero-copy worker's GPU convert + // (planar-YUV444 CUDA frames — `cuda` true), or the CPU path's swscale RGB→YUV444P. Both + // feed `profile=rext`; the range follows `PUNKTFUNK_444_FULLRANGE` in both. let want_444 = chroma.is_444() && codec == Codec::H265; - if want_444 && cuda { - bail!( - "NVENC 4:4:4 needs CPU RGB frames (the session forces non-zero-copy capture for \ - 4:4:4); got a CUDA frame — capture/encoder negotiation mismatch" - ); - } ffmpeg::init().context("ffmpeg init")?; if std::env::var_os("PUNKTFUNK_FFMPEG_DEBUG").is_some() { // SAFETY: `av_log_set_level` sets libav's global integer log level; `48` (= AV_LOG_DEBUG) @@ -384,9 +384,11 @@ impl NvencEncoder { None }; - // 4:4:4: build the RGB→YUV444P swscale (BT.709 limited, no rescale). Mirrors the VAAPI CPU - // path's RGB→NV12 scaler, but the dst is full-chroma planar 4:4:4. - let sws_444 = if want_444 { + // 4:4:4 CPU path: build the RGB→YUV444P swscale (BT.709, range per the flag; no rescale). + // Mirrors the VAAPI CPU path's RGB→NV12 scaler, but the dst is full-chroma planar 4:4:4. + // Skipped on the zero-copy path (`cuda`): the worker's GPU convert already delivers + // planar YUV444 CUDA frames — no CPU pixels exist to scale. + let sws_444 = if want_444 && !cuda { let src_av = pixel_to_av(sws_src_pixel(format)?); // SAFETY: `sws_getContext` allocates a swscale context for the given src/dst dims + pixel // formats. Both dims are the encoder's positive `width`/`height` as `c_int`; `src_av` is a @@ -514,6 +516,7 @@ impl NvencEncoder { frame, cuda: cuda_hw, sws_444, + want_444, src_format: format, expand, width, @@ -529,9 +532,9 @@ impl NvencEncoder { impl Encoder for NvencEncoder { fn caps(&self) -> super::EncoderCaps { super::EncoderCaps { - // 4:4:4 iff this session opened the RGB→YUV444P swscale path (FREXT). RFI/HDR-SEI stay - // unsupported on libavcodec NVENC (the trait defaults). - chroma_444: self.sws_444.is_some(), + // 4:4:4 iff this session opened FREXT — the CPU swscale path or the zero-copy GPU + // convert. RFI/HDR-SEI stay unsupported on libavcodec NVENC (the trait defaults). + chroma_444: self.want_444, intra_refresh: self.intra_refresh, ..super::EncoderCaps::default() } @@ -739,9 +742,27 @@ impl NvencEncoder { ffi::av_frame_free(&mut f); bail!("av_hwframe_get_buffer(CUDA) failed ({r})"); } - // NV12 surfaces are two-plane (Y in data[0], interleaved UV in data[1]); the RGB - // surfaces are single-plane. Copy the matching layout into NVENC's pooled surface. - let copy_res = if buf.is_nv12() { + // NV12 surfaces are two-plane (Y in data[0], interleaved UV in data[1]); YUV444 + // surfaces are three-plane (`yuv444p` frames ctx — data[0..3]); the RGB surfaces are + // single-plane. Copy the matching layout into NVENC's pooled surface. A 4:4:4 session + // whose buffer ISN'T YUV444 (a LINEAR/gamescope capture the worker can't convert) + // fails loudly here rather than letting `hevc_nvenc` silently subsample RGB to 4:2:0. + let copy_res = if buf.yuv444 { + let dsts = core::array::from_fn(|i| { + ( + (*f).data[i] as crate::zerocopy::cuda::CUdeviceptr, + (*f).linesize[i] as usize, + ) + }); + crate::zerocopy::cuda::copy_yuv444_to_device(buf, dsts) + } else if self.want_444 { + ffi::av_frame_free(&mut f); + bail!( + "4:4:4 session but the zero-copy frame is not YUV444 (LINEAR/gamescope \ + capture has no GPU 4:4:4 convert) — unset PUNKTFUNK_ZEROCOPY to use the \ + CPU 4:4:4 path on this compositor" + ); + } else if buf.is_nv12() { let y_ptr = (*f).data[0] as crate::zerocopy::cuda::CUdeviceptr; let y_pitch = (*f).linesize[0] as usize; let uv_ptr = (*f).data[1] as crate::zerocopy::cuda::CUdeviceptr; diff --git a/crates/punktfunk-host/src/encode/linux/vaapi.rs b/crates/punktfunk-host/src/encode/linux/vaapi.rs index e17c984f..698f6e33 100644 --- a/crates/punktfunk-host/src/encode/linux/vaapi.rs +++ b/crates/punktfunk-host/src/encode/linux/vaapi.rs @@ -70,7 +70,7 @@ fn vaapi_sws_src(format: PixelFormat) -> Result { PixelFormat::Rgba => Pixel::RGBA, PixelFormat::Rgb => Pixel::RGB24, PixelFormat::Bgr => Pixel::BGR24, - PixelFormat::Nv12 | PixelFormat::P010 | PixelFormat::Rgb10a2 => { + PixelFormat::Nv12 | PixelFormat::P010 | PixelFormat::Rgb10a2 | PixelFormat::Yuv444 => { bail!("VAAPI CPU-input path supports packed RGB/BGR only; got {format:?}") } }) diff --git a/crates/punktfunk-host/src/encode/sw.rs b/crates/punktfunk-host/src/encode/sw.rs index c5db79e1..97ae6ff6 100644 --- a/crates/punktfunk-host/src/encode/sw.rs +++ b/crates/punktfunk-host/src/encode/sw.rs @@ -184,9 +184,9 @@ impl Encoder for OpenH264Encoder { } // NV12/P010 are GPU-resident video-processor outputs for the NVENC path; the software // encoder never receives them (it only gets CPU RGB frames). - PixelFormat::Nv12 | PixelFormat::P010 => { + PixelFormat::Nv12 | PixelFormat::P010 | PixelFormat::Yuv444 => { anyhow::bail!( - "software encoder cannot encode YUV GPU textures (NV12/P010 → NVENC only)" + "software encoder cannot encode YUV GPU frames (NV12/P010/YUV444 → NVENC only)" ) } }; diff --git a/crates/punktfunk-host/src/encode/windows/ffmpeg_win.rs b/crates/punktfunk-host/src/encode/windows/ffmpeg_win.rs index 68c8f5d9..dbf575de 100644 --- a/crates/punktfunk-host/src/encode/windows/ffmpeg_win.rs +++ b/crates/punktfunk-host/src/encode/windows/ffmpeg_win.rs @@ -138,7 +138,7 @@ fn sws_src(format: PixelFormat) -> Result { PixelFormat::Rgba => Pixel::RGBA, PixelFormat::Rgb => Pixel::RGB24, PixelFormat::Bgr => Pixel::BGR24, - PixelFormat::Nv12 | PixelFormat::P010 | PixelFormat::Rgb10a2 => { + PixelFormat::Nv12 | PixelFormat::P010 | PixelFormat::Rgb10a2 | PixelFormat::Yuv444 => { bail!("ffmpeg_win swscale path supports packed RGB/BGR only; got {format:?}") } }) diff --git a/crates/punktfunk-host/src/linux/zerocopy/client.rs b/crates/punktfunk-host/src/linux/zerocopy/client.rs index 2cbd5f3d..3e66cf3d 100644 --- a/crates/punktfunk-host/src/linux/zerocopy/client.rs +++ b/crates/punktfunk-host/src/linux/zerocopy/client.rs @@ -297,6 +297,19 @@ impl RemoteImporter { ) } + /// Mirror of [`super::egl::EglImporter::import_yuv444`] (tiled dmabuf → stacked 3-plane + /// YUV444 CUDA buffer — the 4:4:4 zero-copy path). + pub fn import_yuv444( + &mut self, + plane: &DmabufPlane, + width: u32, + height: u32, + fourcc: u32, + modifier: Option, + ) -> Result { + self.import_impl(plane, ImportKind::Tiled444, width, height, fourcc, modifier) + } + /// Mirror of [`super::egl::EglImporter::import_linear`] (LINEAR dmabuf → Vulkan bridge). pub fn import_linear( &mut self, @@ -394,6 +407,8 @@ impl RemoteImporter { m.width, m.height, m.uv, + // The wire carries no plane format — the buffer's layout is what WE requested. + kind == ImportKind::Tiled444, Box::new(move || { // Fire-and-forget recycle; a dead worker just means EPIPE, ignored. The // captured `shared` Arc is what keeps the mapping + socket alive until diff --git a/crates/punktfunk-host/src/linux/zerocopy/cuda.rs b/crates/punktfunk-host/src/linux/zerocopy/cuda.rs index 1c32b7ed..47d5c51b 100644 --- a/crates/punktfunk-host/src/linux/zerocopy/cuda.rs +++ b/crates/punktfunk-host/src/linux/zerocopy/cuda.rs @@ -620,6 +620,31 @@ fn alloc_pitched(width: u32, height: u32) -> Result<(CUdeviceptr, usize)> { Ok((ptr, pitch)) } +/// Allocate ONE pitched buffer holding the three stacked full-res 1-byte planes of a planar +/// YUV444 surface: `3·height` rows of `width` bytes at the driver's pitch, rows `[0, H)` = Y, +/// `[H, 2H)` = U, `[2H, 3H)` = V. A single allocation keeps the buffer single-plane on the +/// worker↔host wire (one IPC handle), like the RGB path. +fn alloc_pitched_yuv444(width: u32, height: u32) -> Result<(CUdeviceptr, usize)> { + let mut ptr: CUdeviceptr = 0; + let mut pitch: usize = 0; + // SAFETY: `cuMemAllocPitch_v2` allocates a pitched device buffer (wrapper → live table). + // `&mut ptr`/`&mut pitch` are live, distinct stack out-params outliving the synchronous call; + // width/height/element-size are by-value ints. No aliasing. + unsafe { + ck( + cuMemAllocPitch_v2( + &mut ptr, + &mut pitch, + width as usize, // 1 byte/px per plane + height as usize * 3, // Y + U + V stacked + 16, + ), + "cuMemAllocPitch_v2(YUV444)", + )?; + } + Ok((ptr, pitch)) +} + /// Allocate the two pitched planes of an NV12 surface (8-bit BT.709 4:2:0): a `width`-byte Y plane /// (W×H, 1 byte/px) and an interleaved chroma plane (W/2 × H/2 samples, 2 bytes/sample → W bytes /// wide). Both planes share the driver's Y pitch (the wider request), so the encoder's two-plane @@ -706,6 +731,8 @@ pub struct BufferPool { pitch: usize, /// NV12 pools carry a second (chroma) pitch; `Some` ⇒ buffers from this pool have a UV plane. uv_pitch: Option, + /// YUV444 pools: one allocation of 3·`height` stacked 1-byte planes (see [`alloc_pitched_yuv444`]). + yuv444: bool, } impl BufferPool { @@ -722,6 +749,7 @@ impl BufferPool { height, pitch, uv_pitch: None, + yuv444: false, }) } @@ -738,6 +766,25 @@ impl BufferPool { height, pitch: y_pitch, uv_pitch: Some(uv_pitch), + yuv444: false, + }) + } + + /// Create a pool of planar YUV444 surfaces: ONE pitched allocation per buffer holding the + /// three full-res 1-byte planes stacked as rows `[Y | U | V]` (3·height rows at the same + /// pitch), so the two-plane wire protocol and IPC path carry it like a single-plane buffer. + pub fn new_yuv444(width: u32, height: u32) -> Result { + let (ptr, pitch) = alloc_pitched_yuv444(width, height)?; + Ok(BufferPool { + inner: Arc::new(Mutex::new(PoolInner { + free: vec![ptr], + free_uv: Vec::new(), + })), + width, + height, + pitch, + uv_pitch: None, + yuv444: true, }) } @@ -772,6 +819,7 @@ impl BufferPool { width: self.width, height: self.height, uv: Some((uv_ptr, uv_pitch)), + yuv444: false, pool: Some(self.inner.clone()), remote_release: None, }); @@ -779,6 +827,7 @@ impl BufferPool { let reuse = self.inner.lock().unwrap().free.pop(); let ptr = match reuse { Some(p) => p, + None if self.yuv444 => alloc_pitched_yuv444(self.width, self.height)?.0, None => alloc_pitched(self.width, self.height)?.0, }; Ok(DeviceBuffer { @@ -787,6 +836,7 @@ impl BufferPool { width: self.width, height: self.height, uv: None, + yuv444: self.yuv444, pool: Some(self.inner.clone()), remote_release: None, }) @@ -804,6 +854,10 @@ pub struct DeviceBuffer { /// NV12 only: the interleaved chroma plane `(ptr, pitch)` paired with the Y plane in [`ptr`]. /// `None` for the default 4-byte RGB/BGRx path. When `Some`, [`ptr`] is the Y plane (1 byte/px). pub uv: Option<(CUdeviceptr, usize)>, + /// Planar YUV444: [`ptr`] is ONE allocation of 3·[`height`](Self::height) rows at + /// [`pitch`](Self::pitch) — the full-res 1-byte Y, U, V planes stacked in that order + /// (`uv` stays `None`; the single-plane wire/IPC path carries it unchanged). + pub yuv444: bool, pool: Option>>, /// Set for buffers whose device memory is owned by ANOTHER process (the zero-copy import /// worker, reached via CUDA IPC): drop runs this exactly once (telling the owner to recycle) @@ -821,6 +875,7 @@ impl DeviceBuffer { width, height, uv: None, + yuv444: false, pool: None, remote_release: None, }) @@ -836,6 +891,23 @@ impl DeviceBuffer { width, height, uv: Some((uv_ptr, uv_pitch)), + yuv444: false, + pool: None, + remote_release: None, + }) + } + + /// Allocate a standalone (un-pooled) planar-YUV444 stacked buffer. Prefer + /// [`BufferPool::new_yuv444`] on the hot path; used by the self-test. + pub fn alloc_yuv444(width: u32, height: u32) -> Result { + let (ptr, pitch) = alloc_pitched_yuv444(width, height)?; + Ok(DeviceBuffer { + ptr, + pitch, + width, + height, + uv: None, + yuv444: true, pool: None, remote_release: None, }) @@ -850,12 +922,15 @@ impl DeviceBuffer { /// buffer. `release` runs exactly once on drop — it tells the owning process to recycle the /// buffer; nothing is freed or pooled locally (the IPC mapping itself is closed by the cache /// that opened it, after the last remote buffer referencing it has dropped). + /// `yuv444` marks a stacked 3-plane YUV444 allocation (the wire carries no format — the host + /// knows what it REQUESTED, `ImportKind::Tiled444`). pub fn remote( ptr: CUdeviceptr, pitch: usize, width: u32, height: u32, uv: Option<(CUdeviceptr, usize)>, + yuv444: bool, release: Box, ) -> DeviceBuffer { DeviceBuffer { @@ -864,6 +939,7 @@ impl DeviceBuffer { width, height, uv, + yuv444, pool: None, remote_release: Some(release), } @@ -1045,6 +1121,24 @@ pub fn copy_mapped_nv12( uv_tex.copy_mapped_plane(uv_ptr, uv_pitch, (w / 2) * 2, h / 2) } +/// Copy the three YUV444 convert targets (registered full-res `R8` GL textures) into `dst`'s +/// stacked planes (`dst.yuv444`: rows `[0,H)`=Y, `[H,2H)`=U, `[2H,3H)`=V at `dst.pitch`). Each +/// copy syncs on our priority stream before returning, so the dmabuf is safe to recycle after. +pub fn copy_mapped_yuv444( + y_tex: &mut RegisteredTexture, + u_tex: &mut RegisteredTexture, + v_tex: &mut RegisteredTexture, + dst: &DeviceBuffer, +) -> Result<()> { + anyhow::ensure!(dst.yuv444, "copy_mapped_yuv444 on a non-YUV444 buffer"); + let w = dst.width as usize; + let h = dst.height as usize; + let plane = |i: usize| dst.ptr + (dst.pitch * h * i) as CUdeviceptr; + y_tex.copy_mapped_plane(plane(0), dst.pitch, w, h)?; + u_tex.copy_mapped_plane(plane(1), dst.pitch, w, h)?; + v_tex.copy_mapped_plane(plane(2), dst.pitch, w, h) +} + /// Copy a pitched device buffer into another device region (device→device), e.g. our imported /// [`DeviceBuffer`] into a pooled CUDA surface NVENC owns. Both are 4-byte (BGRx) pixels. /// The caller must have the shared context current on this thread (see [`make_current`]). @@ -1121,6 +1215,36 @@ pub fn copy_nv12_to_device( } } +/// Copy our imported stacked-YUV444 [`DeviceBuffer`] into NVENC's three-plane CUDA surface +/// (`av_hwframe_get_buffer`'s `data[0..3]` + `linesize[0..3]` for a `yuv444p` frames context). +/// Each plane is `width`×`height` bytes; the source planes sit at row offsets `0/H/2H` of the +/// single allocation. The caller must have the shared context current. +pub fn copy_yuv444_to_device(src: &DeviceBuffer, dsts: [(CUdeviceptr, usize); 3]) -> Result<()> { + anyhow::ensure!(src.yuv444, "copy_yuv444_to_device on a non-YUV444 buffer"); + let w = src.width as usize; + let h = src.height as usize; + for (i, (dst_ptr, dst_pitch)) in dsts.into_iter().enumerate() { + let copy = CUDA_MEMCPY2D { + srcMemoryType: CU_MEMORYTYPE_DEVICE, + srcDevice: src.ptr + (src.pitch * h * i) as CUdeviceptr, + srcPitch: src.pitch, + dstMemoryType: CU_MEMORYTYPE_DEVICE, + dstDevice: dst_ptr, + dstPitch: dst_pitch, + WidthInBytes: w, // 1 byte/px per plane + Height: h, + ..Default::default() + }; + // SAFETY: unsafe `copy_blocking` device→device copy; the caller must have the shared + // context current (documented). `©` is a live local outliving the synchronous call; + // `src.ptr + pitch·h·i` stays within the live 3·H-row stacked allocation (`yuv444` + // checked above), `dst_ptr`/`dst_pitch` is the caller's live NVENC plane; `w`×`h` fits + // both. Wrapper → live table. + unsafe { copy_blocking(©, "cuMemcpy2DAsync_v2(yuv444 plane dev->dev)")? }; + } + Ok(()) +} + impl RegisteredTexture { /// Unregister now (idempotent; the later `Drop` then no-ops). Teardown-order helper: the blit /// destructors call this to release the CUDA registration BEFORE deleting the GL texture it diff --git a/crates/punktfunk-host/src/linux/zerocopy/egl.rs b/crates/punktfunk-host/src/linux/zerocopy/egl.rs index d7fef799..8d74abb4 100644 --- a/crates/punktfunk-host/src/linux/zerocopy/egl.rs +++ b/crates/punktfunk-host/src/linux/zerocopy/egl.rs @@ -131,6 +131,30 @@ const FRAG_SRC: &[u8] = b"#version 330 core\nuniform sampler2D image;\nin vec2 v const FRAG_Y_SRC: &[u8] = b"#version 330 core\nuniform sampler2D image;\nin vec2 v_tex;\nout vec4 o_color;\nvoid main(){vec3 c=texture(image,v_tex).rgb;float Y=(16.0+219.0*(0.2126*c.r+0.7152*c.g+0.0722*c.b))/255.0;o_color=vec4(clamp(Y,0.0,1.0),0.0,0.0,1.0);}\n"; const FRAG_UV_SRC: &[u8] = b"#version 330 core\nuniform sampler2D image;\nin vec2 v_tex;\nout vec4 o_color;\nvoid main(){vec3 c=texture(image,v_tex).rgb;float U=(128.0+224.0*(-0.1146*c.r-0.3854*c.g+0.5000*c.b))/255.0;float V=(128.0+224.0*(0.5000*c.r-0.4542*c.g-0.0458*c.b))/255.0;o_color=vec4(clamp(U,0.0,1.0),clamp(V,0.0,1.0),0.0,1.0);}\n"; +/// The three planar-YUV444 convert shaders (full-res `R8` target each) — the [`Yuv444Blit`] +/// analogue of `FRAG_Y_SRC`/`FRAG_UV_SRC` with NO subsampling (4:4:4 keeps every chroma sample). +/// Same BT.709 coefficients; `full_range` flips the quantization from studio (16+219 / 128±112) +/// to the full 0..255 swing — the encoder flips the VUI (`PUNKTFUNK_444_FULLRANGE`, read by both +/// processes from the same inherited environment) in lockstep, so pixels and signaling agree. +fn yuv444_frag_sources(full_range: bool) -> (Vec, Vec, Vec) { + let (y_scale, y_off, c_scale) = if full_range { + ("255.0", "0.0", "255.0") + } else { + ("219.0", "16.0", "224.0") + }; + let head = "#version 330 core\nuniform sampler2D image;\nin vec2 v_tex;\nout vec4 o_color;\nvoid main(){vec3 c=texture(image,v_tex).rgb;"; + let y = format!( + "{head}float Y=({y_off}+{y_scale}*(0.2126*c.r+0.7152*c.g+0.0722*c.b))/255.0;o_color=vec4(clamp(Y,0.0,1.0),0.0,0.0,1.0);}}\n" + ); + let u = format!( + "{head}float U=(128.0+{c_scale}*(-0.1146*c.r-0.3854*c.g+0.5000*c.b))/255.0;o_color=vec4(clamp(U,0.0,1.0),0.0,0.0,1.0);}}\n" + ); + let v = format!( + "{head}float V=(128.0+{c_scale}*(0.5000*c.r-0.4542*c.g-0.0458*c.b))/255.0;o_color=vec4(clamp(V,0.0,1.0),0.0,0.0,1.0);}}\n" + ); + (y.into_bytes(), u.into_bytes(), v.into_bytes()) +} + unsafe fn compile_shader(kind: u32, src: &[u8]) -> Result { let sh = glCreateShader(kind); ensure!(sh != 0, "glCreateShader failed"); @@ -465,6 +489,155 @@ impl Drop for Nv12Blit { } } +/// Per-size GL machinery to convert a dmabuf EGLImage into planar **YUV444** (BT.709; studio or +/// full range per `PUNKTFUNK_444_FULLRANGE`) — the [`Nv12Blit`] analogue for a 4:4:4 session. +/// Three full-res passes share `src_tex`, each into its own CUDA-registrable `GL_R8` texture +/// (Y/U/V — no subsampling, so no half-res pass and no siting question). The pooled destination +/// is ONE stacked allocation (`BufferPool::new_yuv444`), which keeps the worker↔host wire +/// single-plane. This is what lets a 4:4:4 NVENC session stay zero-copy instead of falling to +/// the CPU swscale path. +struct Yuv444Blit { + programs: [u32; 3], + vao: u32, + fbos: [u32; 3], + /// CUDA-registrable full-res `GL_R8` targets: Y, U, V. + texs: [u32; 3], + /// Source texture re-targeted to each frame's EGLImage. + src_tex: u32, + width: u32, + height: u32, + registered: [cuda::RegisteredTexture; 3], + /// Recycled stacked-YUV444 device buffers handed to the encoder. + pool: cuda::BufferPool, +} + +impl Yuv444Blit { + unsafe fn new(width: u32, height: u32) -> Result { + ensure!( + width % 2 == 0 && height % 2 == 0, + "YUV444 convert needs even dimensions (got {width}x{height})" + ); + let full_range = std::env::var("PUNKTFUNK_444_FULLRANGE").is_ok_and(|v| v.trim() == "1"); + let (y_src, u_src, v_src) = yuv444_frag_sources(full_range); + let programs = [ + compile_program_with(&y_src)?, + compile_program_with(&u_src)?, + compile_program_with(&v_src)?, + ]; + let mut vao = 0u32; + glGenVertexArrays(1, &mut vao); + let mut fbos = [0u32; 3]; + glGenFramebuffers(3, fbos.as_mut_ptr()); + let mut texs = [0u32; 3]; + glGenTextures(3, texs.as_mut_ptr()); + for &tex in &texs { + glBindTexture(GL_TEXTURE_2D, tex); + glTexStorage2D(GL_TEXTURE_2D, 1, GL_R8, width as c_int, height as c_int); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + } + // Source: LINEAR is exact at the 1:1 mapping every pass uses (texel centres), matching + // the Nv12Blit source setup. + let mut src_tex = 0u32; + glGenTextures(1, &mut src_tex); + glBindTexture(GL_TEXTURE_2D, src_tex); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glBindTexture(GL_TEXTURE_2D, 0); + for (&fbo, &tex) in fbos.iter().zip(&texs) { + glBindFramebuffer(GL_FRAMEBUFFER, fbo); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, tex, 0); + let status = glCheckFramebufferStatus(GL_FRAMEBUFFER); + glBindFramebuffer(GL_FRAMEBUFFER, 0); + ensure!( + status == GL_FRAMEBUFFER_COMPLETE, + "YUV444 blit FBO incomplete ({status:#x}) — GL_R8 not renderable?" + ); + } + let registered = [ + cuda::RegisteredTexture::register_gl(texs[0])?, + cuda::RegisteredTexture::register_gl(texs[1])?, + cuda::RegisteredTexture::register_gl(texs[2])?, + ]; + let pool = cuda::BufferPool::new_yuv444(width, height)?; + if full_range { + tracing::info!("YUV444 zero-copy convert: FULL range (PUNKTFUNK_444_FULLRANGE=1)"); + } + Ok(Yuv444Blit { + programs, + vao, + fbos, + texs, + src_tex, + width, + height, + registered, + pool, + }) + } + + /// Bind `image` to the source texture and run the three plane passes. + /// + /// # Safety: the GL context is current on this thread; `image` is a valid `EGLImage`. + unsafe fn run(&self, egl_image_target: EglImageTargetFn, image: *mut c_void) -> Result<()> { + glBindTexture(GL_TEXTURE_2D, self.src_tex); + let _ = glGetError(); + egl_image_target(GL_TEXTURE_2D, image); + let e = glGetError(); + glBindTexture(GL_TEXTURE_2D, 0); + ensure!(e == 0, "glEGLImageTargetTexture2DOES failed ({e:#x})"); + glActiveTexture(GL_TEXTURE0); + glBindVertexArray(self.vao); + for (&fbo, &program) in self.fbos.iter().zip(&self.programs) { + glBindFramebuffer(GL_FRAMEBUFFER, fbo); + glViewport(0, 0, self.width as c_int, self.height as c_int); + glUseProgram(program); + glBindTexture(GL_TEXTURE_2D, self.src_tex); + glDrawArrays(GL_TRIANGLES, 0, 3); + } + glBindVertexArray(0); + glBindFramebuffer(GL_FRAMEBUFFER, 0); + glFlush(); // submit GL work before CUDA maps the textures + Ok(()) + } +} + +impl Drop for Yuv444Blit { + fn drop(&mut self) { + // Unregister the CUDA graphics resources BEFORE deleting the GL textures they wrap + // (same teardown-order hazard as `Nv12Blit::drop`). + for r in &mut self.registered { + r.release(); + } + // SAFETY: these GL names were all created by THIS `Yuv444Blit` in `new` on the current GL + // context (still current — the owning `EglImporter` drops on its single capture thread). + // Each `glDelete*` takes a count + a pointer to that many names; the arrays are live + // fields (or a live temporary for the whole call). Each name is deleted exactly once, + // after its CUDA registration was released above. + unsafe { + glDeleteTextures(3, self.texs.as_ptr()); + glDeleteTextures(1, &self.src_tex); + glDeleteFramebuffers(3, self.fbos.as_ptr()); + glDeleteVertexArrays(1, &self.vao); + for &p in &self.programs { + glDeleteProgram(p); + } + } + } +} + +/// Which GPU conversion `import_inner` runs on the de-tiled EGLImage — mirrors the three tiled +/// [`super::proto::ImportKind`] entry points. +#[derive(Clone, Copy)] +enum Convert { + /// BGRx swizzle only ([`GlBlit`]). + Rgb, + /// RGB → NV12, BT.709 limited ([`Nv12Blit`]). + Nv12, + /// RGB → planar YUV444, BT.709 ([`Yuv444Blit`]) — the 4:4:4 zero-copy path. + Yuv444, +} + /// One dmabuf plane as delivered by PipeWire (single-plane for BGRx). #[derive(Clone, Copy, Debug)] pub struct DmabufPlane { @@ -489,6 +662,8 @@ pub struct EglImporter { blit: Option, /// Lazily-created NV12 convert machinery (`PUNKTFUNK_NV12` path; recreated on size change). nv12_blit: Option, + /// Lazily-created planar-YUV444 convert machinery (4:4:4 sessions; recreated on size change). + yuv444_blit: Option, /// LINEAR-dmabuf path (gamescope): a Vulkan bridge (dmabuf → exportable OPAQUE_FD → CUDA), /// created lazily on the first LINEAR frame, + the destination pool. vk: Option, @@ -633,6 +808,7 @@ impl EglImporter { egl_image_target, blit: None, nv12_blit: None, + yuv444_blit: None, vk: None, linear_pool: None, gbm, @@ -756,7 +932,7 @@ impl EglImporter { fourcc: u32, modifier: Option, ) -> Result { - self.import_inner(plane, width, height, fourcc, modifier, false) + self.import_inner(plane, width, height, fourcc, modifier, Convert::Rgb) } /// Like [`import`](Self::import), but de-tiles **and converts** the dmabuf to NV12 (BT.709 @@ -771,7 +947,21 @@ impl EglImporter { fourcc: u32, modifier: Option, ) -> Result { - self.import_inner(plane, width, height, fourcc, modifier, true) + self.import_inner(plane, width, height, fourcc, modifier, Convert::Nv12) + } + + /// Like [`import_nv12`](Self::import_nv12), but converts to planar **YUV444** (full chroma — + /// a 4:4:4 session) into one stacked 3-plane [`DeviceBuffer`] (`DeviceBuffer::yuv444`). Only + /// the tiled EGL/GL path supports this. + pub fn import_yuv444( + &mut self, + plane: &DmabufPlane, + width: u32, + height: u32, + fourcc: u32, + modifier: Option, + ) -> Result { + self.import_inner(plane, width, height, fourcc, modifier, Convert::Yuv444) } fn import_inner( @@ -781,7 +971,7 @@ impl EglImporter { height: u32, fourcc: u32, modifier: Option, - nv12: bool, + convert: Convert, ) -> Result { let mut attrs: Vec = vec![ egl::WIDTH as egl::Attrib, @@ -822,13 +1012,14 @@ impl EglImporter { ) .context("eglCreateImage(EGL_LINUX_DMA_BUF_EXT) — modifier mismatch?")?; - // EGLImage → (sampled by a shader) → GL_RGBA8 texture (or NV12 R8+RG8 pair) → register - // *that* with CUDA → map → array → copy out. Registering the EGLImage texture directly - // fails (its layout isn't a CUDA-registrable format); the render targets are. - let result = if nv12 { - self.blit_and_copy_nv12(image.as_ptr(), width, height) - } else { - self.blit_and_copy(image.as_ptr(), width, height) + // EGLImage → (sampled by a shader) → GL_RGBA8 texture (or NV12 R8+RG8 pair, or the three + // YUV444 R8 planes) → register *that* with CUDA → map → array → copy out. Registering the + // EGLImage texture directly fails (its layout isn't a CUDA-registrable format); the + // render targets are. + let result = match convert { + Convert::Nv12 => self.blit_and_copy_nv12(image.as_ptr(), width, height), + Convert::Yuv444 => self.blit_and_copy_yuv444(image.as_ptr(), width, height), + Convert::Rgb => self.blit_and_copy(image.as_ptr(), width, height), }; let _ = self.egl.destroy_image(self.display, image); result @@ -899,6 +1090,39 @@ impl EglImporter { Ok(dst) } + /// Convert the dmabuf `image` to planar YUV444 (three full-res `R8` textures) and copy the + /// planes into a pooled stacked [`DeviceBuffer`]. (Re)creates the per-size convert machinery + /// as needed — the 4:4:4 analogue of [`blit_and_copy_nv12`](Self::blit_and_copy_nv12). + fn blit_and_copy_yuv444( + &mut self, + image: *mut c_void, + width: u32, + height: u32, + ) -> Result { + cuda::make_current()?; + if self.yuv444_blit.as_ref().map(|b| (b.width, b.height)) != Some((width, height)) { + // SAFETY: `Yuv444Blit::new` requires the GL context current on the calling thread and + // a current CUDA context. Both hold: this runs on the capture thread where + // `EglImporter::new` made the GL context current and never released it, and + // `cuda::make_current()?` ran at the top of this function. `width`/`height` are plain + // `Copy` frame dimensions. + self.yuv444_blit = Some(unsafe { Yuv444Blit::new(width, height)? }); + } + let egl_image_target = self.egl_image_target; + let blit = self.yuv444_blit.as_mut().unwrap(); + // SAFETY: `Yuv444Blit::run` requires a current GL context and a valid `EGLImage`. The GL + // context is current on this capture thread (made current in `EglImporter::new`, never + // released) and `cuda::make_current()` ran above; `egl_image_target` is the + // `glEGLImageTargetTexture2DOES` pointer loaded in `new`; `image` is the raw handle of the + // live `EGLImage` that `import_inner` created with `eglCreateImage` and destroys only + // AFTER this call returns, so it stays valid for the whole synchronous `run`. + unsafe { blit.run(egl_image_target, image)? }; + let dst = blit.pool.get()?; + let [y, u, v] = &mut blit.registered; + cuda::copy_mapped_yuv444(y, u, v, &dst)?; + Ok(dst) + } + /// Self-test entry: upload a packed `width`×`height` RGBA8 host pattern into a GL texture, run /// the NV12 convert passes on the GPU, and copy both planes into a pooled NV12 [`DeviceBuffer`]. /// Exercises the exact shaders + CUDA copy the live path uses, but sourced from an uploaded diff --git a/crates/punktfunk-host/src/linux/zerocopy/mod.rs b/crates/punktfunk-host/src/linux/zerocopy/mod.rs index 3171bb0d..366817a8 100644 --- a/crates/punktfunk-host/src/linux/zerocopy/mod.rs +++ b/crates/punktfunk-host/src/linux/zerocopy/mod.rs @@ -137,6 +137,22 @@ impl Importer { } } + /// Tiled dmabuf → planar-YUV444 GPU convert → one stacked 3-plane CUDA buffer (the 4:4:4 + /// zero-copy path). + pub fn import_yuv444( + &mut self, + plane: &DmabufPlane, + width: u32, + height: u32, + fourcc: u32, + modifier: Option, + ) -> anyhow::Result { + match self { + Importer::Remote(r) => r.import_yuv444(plane, width, height, fourcc, modifier), + Importer::InProc(i) => i.import_yuv444(plane, width, height, fourcc, modifier), + } + } + pub fn import_linear( &mut self, plane: &DmabufPlane, @@ -215,8 +231,9 @@ pub fn drm_fourcc(format: crate::capture::PixelFormat) -> Option { Rgbx => fourcc(b"XB24"), // DRM_FORMAT_XBGR8888 Rgba => fourcc(b"AB24"), // DRM_FORMAT_ABGR8888 // 24-bit packed RGB/BGR have no straightforward dmabuf import here; use the CPU path. - // Rgb10a2/Nv12/P010 are the Windows HDR / video-processor formats — never produced on Linux. - Rgb | Bgr | Rgb10a2 | Nv12 | P010 => return None, + // Rgb10a2/Nv12/P010 are the Windows HDR / video-processor formats — never produced on + // Linux; Yuv444 is OUR convert's OUTPUT, never a capture source format. + Rgb | Bgr | Rgb10a2 | Nv12 | P010 | Yuv444 => return None, }) } diff --git a/crates/punktfunk-host/src/linux/zerocopy/proto.rs b/crates/punktfunk-host/src/linux/zerocopy/proto.rs index c54ce7ea..375e83c3 100644 --- a/crates/punktfunk-host/src/linux/zerocopy/proto.rs +++ b/crates/punktfunk-host/src/linux/zerocopy/proto.rs @@ -27,7 +27,7 @@ pub const PROTO_VERSION: u32 = 1; /// below this). A message reported truncated at this size is a protocol error. pub const MAX_MSG: usize = 64 * 1024; -/// How a dmabuf should be imported — mirrors the three `EglImporter` entry points. +/// How a dmabuf should be imported — mirrors the `EglImporter` entry points. #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] pub enum ImportKind { /// Tiled dmabuf → EGL/GL de-tile blit → BGRx CUDA buffer. @@ -36,6 +36,11 @@ pub enum ImportKind { TiledNv12, /// LINEAR dmabuf → Vulkan bridge → BGRx CUDA buffer (gamescope's only offer). Linear, + /// Tiled dmabuf → EGL/GL planar-YUV444 convert → ONE stacked 3-plane CUDA buffer (a 4:4:4 + /// session). APPENDED last: the worker can outlive a replaced host binary, so the earlier + /// variants' wire tags must never shift — an old worker receiving this fails the decode and + /// the import-fail machinery handles it like any other worker error. + Tiled444, } /// host → worker. diff --git a/crates/punktfunk-host/src/linux/zerocopy/worker.rs b/crates/punktfunk-host/src/linux/zerocopy/worker.rs index b92881b0..c51de928 100644 --- a/crates/punktfunk-host/src/linux/zerocopy/worker.rs +++ b/crates/punktfunk-host/src/linux/zerocopy/worker.rs @@ -291,6 +291,13 @@ impl EglBackend { req.fourcc, req.modifier, )?, + ImportKind::Tiled444 => self.importer.import_yuv444( + &plane, + req.width, + req.height, + req.fourcc, + req.modifier, + )?, ImportKind::Linear => self.importer.import_linear(&plane, req.width, req.height)?, }; // Assign / look up the buffer's id and export its CUDA IPC identity on first delivery. diff --git a/crates/punktfunk-host/src/session_plan.rs b/crates/punktfunk-host/src/session_plan.rs index 775d04b9..c4168c7b 100644 --- a/crates/punktfunk-host/src/session_plan.rs +++ b/crates/punktfunk-host/src/session_plan.rs @@ -124,22 +124,26 @@ impl SessionPlan { pub fn output_format(&self) -> crate::capture::OutputFormat { let gpu = self.encoder.is_gpu(); // Linux NVENC 4:4:4: libavcodec `hevc_nvenc` only emits 4:4:4 from a YUV444 *input* frame — - // RGB-in is always subsampled to 4:2:0 (verified on the RTX 5070 Ti). So the encoder does an - // RGB→YUV444P swscale and needs CPU-resident RGB frames; force the zero-copy GPU capture off - // for a 4:4:4 NVENC session. (VAAPI 4:4:4, where the hardware supports it, keeps its dmabuf - // path via `scale_vaapi`; Windows NVENC ingests ARGB directly and stays GPU.) + // RGB-in is always subsampled to 4:2:0 (verified on the RTX 5070 Ti). With zero-copy + // enabled the import worker produces that input ON the GPU (`ImportKind::Tiled444` — the + // planar-YUV444 convert), so the session stays fully zero-copy at full chroma. Without + // zero-copy the encoder swscales CPU RGB → YUV444P, which needs CPU-resident frames — + // force the GPU capture off for that case only. (VAAPI 4:4:4, where the hardware supports + // it, keeps its dmabuf path via `scale_vaapi`; Windows NVENC ingests BGRA directly.) #[cfg(target_os = "linux")] let gpu = { - let force_cpu_for_nvenc_444 = - self.chroma.is_444() && !crate::encode::linux_zero_copy_is_vaapi(); + let force_cpu_for_nvenc_444 = self.chroma.is_444() + && !crate::encode::linux_zero_copy_is_vaapi() + && !crate::zerocopy::enabled(); if gpu && force_cpu_for_nvenc_444 { // Surface the trade loudly: this is the single biggest per-frame cost a 4:4:4 // session adds (full-res CPU readback + swscale RGB→YUV444P every frame), and // it looks like an unexplained fps ceiling if you don't know it happened. tracing::warn!( - "4:4:4 session on the NVENC path: zero-copy GPU capture DISABLED — every \ - frame is CPU RGB + swscale RGB→YUV444P; expect a lower fps ceiling than \ - 4:2:0 at this mode" + "4:4:4 session on the NVENC path without PUNKTFUNK_ZEROCOPY: zero-copy GPU \ + capture DISABLED — every frame is CPU RGB + swscale RGB→YUV444P; expect a \ + lower fps ceiling than 4:2:0 at this mode (set PUNKTFUNK_ZEROCOPY=1 for the \ + GPU 4:4:4 convert)" ); } gpu && !force_cpu_for_nvenc_444