perf(latency): T2.5b — NV12 compute CSC on the LINEAR/gamescope zero-copy path
apple / swift (push) Successful in 1m21s
apple / screenshots (push) Successful in 6m23s
ci / web (push) Successful in 54s
arch / build-publish (push) Successful in 10m52s
ci / docs-site (push) Successful in 1m7s
decky / build-publish (push) Successful in 35s
android / android (push) Successful in 13m2s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 16s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 16s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 11s
ci / bench (push) Successful in 6m8s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 7m56s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 9m44s
docker / deploy-docs (push) Successful in 24s
deb / build-publish (push) Successful in 11m52s
ci / rust (push) Successful in 27m4s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 20m26s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 20m20s
apple / swift (push) Successful in 1m21s
apple / screenshots (push) Successful in 6m23s
ci / web (push) Successful in 54s
arch / build-publish (push) Successful in 10m52s
ci / docs-site (push) Successful in 1m7s
decky / build-publish (push) Successful in 35s
android / android (push) Successful in 13m2s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 16s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 16s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 11s
ci / bench (push) Successful in 6m8s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 7m56s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 9m44s
docker / deploy-docs (push) Successful in 24s
deb / build-publish (push) Successful in 11m52s
ci / rust (push) Successful in 27m4s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 20m26s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 20m20s
design/latency-reduction-2026-07.md T2.5's Linux half: the LINEAR dmabuf path (gamescope's only offer) fed NVENC RGB, paying its internal RGB->YUV CSC on the SM the game is saturating — the exact contention §5.A removed everywhere else. The Vulkan bridge now carries a buffer-to-buffer RGB->NV12 compute shader (rgb2nv12_buf.comp, BT.709 limited, coefficient-identical to pf-encode's rgb2yuv.comp; whole-word writes so no 8-bit-storage feature is needed): import dmabuf -> dispatch CSC into the exportable buffer -> CUDA de-strides both planes into a pooled two-plane NV12 buffer. PUNKTFUNK_NV12 (default-on) now covers LINEAR; a CSC failure latches RGB for the stream (mid-frame fallback, no dropped frame); 4:4:4 LINEAR sessions stay RGB (never silently subsample). New ImportKind::LinearNv12 rides the existing worker IPC (appended last per the wire-tag rule); cursor stays downstream (blend_nv12). Validated: .21 clippy -D warnings (pf-zerocopy/pf-capture/host+nvenc) + 17 zero-copy tests. Owed: on-glass gamescope session (visual + dmon sm% check). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -748,6 +748,9 @@ mod pipewire {
|
|||||||
/// (`ImportKind::Tiled444`) and feed NVENC native full-chroma YUV — takes precedence over
|
/// (`ImportKind::Tiled444`) and feed NVENC native full-chroma YUV — takes precedence over
|
||||||
/// `nv12` (a 4:4:4 session must never subsample).
|
/// `nv12` (a 4:4:4 session must never subsample).
|
||||||
yuv444: bool,
|
yuv444: bool,
|
||||||
|
/// The LINEAR (gamescope) NV12 compute CSC failed once — RGB for the rest of the stream
|
||||||
|
/// (T2.5b's per-stream fallback latch; cleared by the next stream's fresh `Ud`).
|
||||||
|
linear_nv12_failed: bool,
|
||||||
/// Rate-limit counter for the latest-frame-only diagnostic log (see `.process`).
|
/// Rate-limit counter for the latest-frame-only diagnostic log (see `.process`).
|
||||||
dbg_log_n: u64,
|
dbg_log_n: u64,
|
||||||
/// Cursor-as-metadata state, composited into the CPU de-pad path (see `consume_frame`).
|
/// Cursor-as-metadata state, composited into the CPU de-pad path (see `consume_frame`).
|
||||||
@@ -1352,15 +1355,17 @@ mod pipewire {
|
|||||||
// sample LINEAR).
|
// sample LINEAR).
|
||||||
let modifier = (ud.modifier != 0).then_some(ud.modifier);
|
let modifier = (ud.modifier != 0).then_some(ud.modifier);
|
||||||
if let Some(fourcc) = pf_frame::drm_fourcc(fmt) {
|
if let Some(fourcc) = pf_frame::drm_fourcc(fmt) {
|
||||||
// GPU converts only on the tiled EGL/GL path (`modifier.is_some()`): a 4:4:4
|
// GPU converts: a 4:4:4 session gets the planar-YUV444 convert on the tiled
|
||||||
// session gets the planar-YUV444 convert (full chroma, takes precedence over
|
// EGL/GL path (full chroma, takes precedence over NV12 — 4:4:4 must never
|
||||||
// NV12 — 4:4:4 must never subsample), otherwise `PUNKTFUNK_NV12` gets NV12 —
|
// subsample), otherwise `PUNKTFUNK_NV12` gets NV12 — tiled via the EGL/GL
|
||||||
// both feed NVENC native YUV so it skips its internal RGB→YUV CSC. The
|
// blit, LINEAR/gamescope via the Vulkan bridge's compute CSC (latency plan
|
||||||
// LINEAR/Vulkan (gamescope) path stays RGB — its converts aren't wired here;
|
// T2.5b) — so NVENC encodes native YUV and skips its internal RGB→YUV CSC on
|
||||||
// a 4:4:4 session on LINEAR frames falls to the encoder's clear-error path
|
// the contended SM. A 4:4:4 session on LINEAR frames has no convert and
|
||||||
// (`want_444` with an RGB CUDA payload) rather than silently subsampling.
|
// stays RGB, falling to the encoder's clear-error path (`want_444` with an
|
||||||
|
// RGB CUDA payload) rather than silently subsampling. A LINEAR NV12 convert
|
||||||
|
// failure latches RGB for the stream (mid-frame fallback, no drop).
|
||||||
let yuv444 = ud.yuv444 && modifier.is_some();
|
let yuv444 = ud.yuv444 && modifier.is_some();
|
||||||
let nv12 = ud.nv12 && !yuv444 && modifier.is_some();
|
let mut nv12 = ud.nv12 && !ud.yuv444;
|
||||||
let imported = if let Some(m) = modifier {
|
let imported = if let Some(m) = modifier {
|
||||||
if yuv444 {
|
if yuv444 {
|
||||||
importer.import_yuv444(&plane, w as u32, h as u32, fourcc, Some(m))
|
importer.import_yuv444(&plane, w as u32, h as u32, fourcc, Some(m))
|
||||||
@@ -1369,7 +1374,20 @@ mod pipewire {
|
|||||||
} else {
|
} else {
|
||||||
importer.import(&plane, w as u32, h as u32, fourcc, Some(m))
|
importer.import(&plane, w as u32, h as u32, fourcc, Some(m))
|
||||||
}
|
}
|
||||||
|
} else if nv12 && !ud.linear_nv12_failed {
|
||||||
|
match importer.import_linear_nv12(&plane, w as u32, h as u32) {
|
||||||
|
Ok(buf) => Ok(buf),
|
||||||
|
Err(e) => {
|
||||||
|
ud.linear_nv12_failed = true;
|
||||||
|
nv12 = false;
|
||||||
|
tracing::warn!(error = %format!("{e:#}"),
|
||||||
|
"LINEAR NV12 compute CSC failed — RGB for the rest of this \
|
||||||
|
stream (NVENC does the CSC internally)");
|
||||||
|
importer.import_linear(&plane, w as u32, h as u32)
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
|
nv12 = false;
|
||||||
importer.import_linear(&plane, w as u32, h as u32)
|
importer.import_linear(&plane, w as u32, h as u32)
|
||||||
};
|
};
|
||||||
match imported {
|
match imported {
|
||||||
@@ -1742,6 +1760,7 @@ mod pipewire {
|
|||||||
vaapi_passthrough,
|
vaapi_passthrough,
|
||||||
nv12: pf_zerocopy::nv12_enabled(),
|
nv12: pf_zerocopy::nv12_enabled(),
|
||||||
yuv444: want_444,
|
yuv444: want_444,
|
||||||
|
linear_nv12_failed: false,
|
||||||
dbg_log_n: 0,
|
dbg_log_n: 0,
|
||||||
cursor: CursorState::default(),
|
cursor: CursorState::default(),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -320,6 +320,17 @@ impl RemoteImporter {
|
|||||||
self.import_impl(plane, ImportKind::Linear, width, height, 0, None)
|
self.import_impl(plane, ImportKind::Linear, width, height, 0, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Mirror of [`super::egl::EglImporter::import_linear_nv12`] (LINEAR dmabuf → Vulkan-bridge
|
||||||
|
/// compute CSC → two-plane NV12 buffer, latency plan T2.5b).
|
||||||
|
pub fn import_linear_nv12(
|
||||||
|
&mut self,
|
||||||
|
plane: &DmabufPlane,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
) -> Result<DeviceBuffer> {
|
||||||
|
self.import_impl(plane, ImportKind::LinearNv12, width, height, 0, None)
|
||||||
|
}
|
||||||
|
|
||||||
fn import_impl(
|
fn import_impl(
|
||||||
&mut self,
|
&mut self,
|
||||||
plane: &DmabufPlane,
|
plane: &DmabufPlane,
|
||||||
|
|||||||
@@ -1386,3 +1386,50 @@ pub fn copy_pitched_to_buffer(
|
|||||||
// within both. Wrapper → live table.
|
// within both. Wrapper → live table.
|
||||||
unsafe { copy_blocking(©, "cuMemcpy2DAsync_v2(ext->dev)") }
|
unsafe { copy_blocking(©, "cuMemcpy2DAsync_v2(ext->dev)") }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// De-stride an NV12 pair from an external mapping (the Vulkan bridge's exportable buffer after
|
||||||
|
/// its compute CSC — latency plan T2.5b) into a pooled two-plane NV12 [`DeviceBuffer`]: the Y
|
||||||
|
/// plane (`width` bytes × `height` rows) and the interleaved UV plane (`width` bytes × ⌈h/2⌉
|
||||||
|
/// rows), each de-strided from `src_pitch` to the pool's own plane pitches. Same contract as
|
||||||
|
/// [`copy_pitched_to_buffer`]: the shared context must be current.
|
||||||
|
pub fn copy_pitched_nv12_to_buffer(
|
||||||
|
y_src: CUdeviceptr,
|
||||||
|
uv_src: CUdeviceptr,
|
||||||
|
src_pitch: usize,
|
||||||
|
dst: &DeviceBuffer,
|
||||||
|
) -> Result<()> {
|
||||||
|
let Some((uv_ptr, uv_pitch)) = dst.uv else {
|
||||||
|
anyhow::bail!("copy_pitched_nv12_to_buffer: destination is not an NV12 buffer");
|
||||||
|
};
|
||||||
|
let y = CUDA_MEMCPY2D {
|
||||||
|
srcMemoryType: CU_MEMORYTYPE_DEVICE,
|
||||||
|
srcDevice: y_src,
|
||||||
|
srcPitch: src_pitch,
|
||||||
|
dstMemoryType: CU_MEMORYTYPE_DEVICE,
|
||||||
|
dstDevice: dst.ptr,
|
||||||
|
dstPitch: dst.pitch,
|
||||||
|
WidthInBytes: dst.width as usize,
|
||||||
|
Height: dst.height as usize,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let uv = CUDA_MEMCPY2D {
|
||||||
|
srcMemoryType: CU_MEMORYTYPE_DEVICE,
|
||||||
|
srcDevice: uv_src,
|
||||||
|
srcPitch: src_pitch,
|
||||||
|
dstMemoryType: CU_MEMORYTYPE_DEVICE,
|
||||||
|
dstDevice: uv_ptr,
|
||||||
|
dstPitch: uv_pitch,
|
||||||
|
// W/2 interleaved UV samples × 2 bytes = `width` bytes per row.
|
||||||
|
WidthInBytes: dst.width as usize,
|
||||||
|
Height: dst.height.div_ceil(2) as usize,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
// SAFETY: same contract as `copy_pitched_to_buffer` — the caller holds the shared context
|
||||||
|
// current; both `CUDA_MEMCPY2D`s are live locals describing spans inside the caller's live
|
||||||
|
// external mapping (`y_src`/`uv_src` at `src_pitch`) and `dst`'s live pooled planes; each
|
||||||
|
// `copy_blocking` synchronizes before returning.
|
||||||
|
unsafe {
|
||||||
|
copy_blocking(&y, "cuMemcpy2DAsync_v2(ext->dev nv12 Y)")?;
|
||||||
|
copy_blocking(&uv, "cuMemcpy2DAsync_v2(ext->dev nv12 UV)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -504,6 +504,9 @@ pub struct EglImporter {
|
|||||||
/// created lazily on the first LINEAR frame, + the destination pool.
|
/// created lazily on the first LINEAR frame, + the destination pool.
|
||||||
vk: Option<super::vulkan::VkBridge>,
|
vk: Option<super::vulkan::VkBridge>,
|
||||||
linear_pool: Option<cuda::BufferPool>,
|
linear_pool: Option<cuda::BufferPool>,
|
||||||
|
/// NV12 twin of [`linear_pool`](Self::linear_pool) for the bridge's compute-CSC output
|
||||||
|
/// (T2.5b) — separate pools because a session may fall back RGB mid-stream.
|
||||||
|
linear_nv12_pool: Option<cuda::BufferPool>,
|
||||||
gbm: *mut c_void,
|
gbm: *mut c_void,
|
||||||
render_fd: c_int,
|
render_fd: c_int,
|
||||||
}
|
}
|
||||||
@@ -647,6 +650,7 @@ impl EglImporter {
|
|||||||
yuv444_blit: None,
|
yuv444_blit: None,
|
||||||
vk: None,
|
vk: None,
|
||||||
linear_pool: None,
|
linear_pool: None,
|
||||||
|
linear_nv12_pool: None,
|
||||||
gbm,
|
gbm,
|
||||||
render_fd,
|
render_fd,
|
||||||
})
|
})
|
||||||
@@ -677,6 +681,38 @@ impl EglImporter {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Like [`import_linear`](Self::import_linear), but the bridge's compute CSC converts to a
|
||||||
|
/// two-plane **NV12** buffer (latency plan T2.5b) — the gamescope/LINEAR analogue of
|
||||||
|
/// [`import_nv12`](Self::import_nv12), so NVENC encodes native YUV on the dedicated-session
|
||||||
|
/// path too instead of paying its internal RGB→YUV CSC on the contended SM.
|
||||||
|
pub fn import_linear_nv12(
|
||||||
|
&mut self,
|
||||||
|
plane: &DmabufPlane,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
) -> Result<DeviceBuffer> {
|
||||||
|
cuda::make_current()?;
|
||||||
|
if self
|
||||||
|
.linear_nv12_pool
|
||||||
|
.as_ref()
|
||||||
|
.map(|p| (p.width(), p.height()))
|
||||||
|
!= Some((width, height))
|
||||||
|
{
|
||||||
|
self.linear_nv12_pool = Some(cuda::BufferPool::new_nv12(width, height)?);
|
||||||
|
}
|
||||||
|
if self.vk.is_none() {
|
||||||
|
self.vk = Some(super::vulkan::VkBridge::new()?);
|
||||||
|
}
|
||||||
|
self.vk.as_mut().unwrap().import_linear_nv12(
|
||||||
|
plane.fd,
|
||||||
|
plane.offset,
|
||||||
|
plane.stride,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
self.linear_nv12_pool.as_ref().unwrap(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Drop the Vulkan bridge's cached per-fd import (see [`super::vulkan::VkBridge::forget_fd`]).
|
/// Drop the Vulkan bridge's cached per-fd import (see [`super::vulkan::VkBridge::forget_fd`]).
|
||||||
/// No-op when the bridge hasn't been built (tiled-only captures).
|
/// No-op when the bridge hasn't been built (tiled-only captures).
|
||||||
pub fn forget_linear_fd(&mut self, fd: i32) {
|
pub fn forget_linear_fd(&mut self, fd: i32) {
|
||||||
|
|||||||
@@ -166,6 +166,20 @@ impl Importer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// LINEAR dmabuf → Vulkan-bridge compute CSC → two-plane NV12 buffer (latency plan T2.5b —
|
||||||
|
/// the gamescope analogue of [`import_nv12`](Self::import_nv12)).
|
||||||
|
pub fn import_linear_nv12(
|
||||||
|
&mut self,
|
||||||
|
plane: &DmabufPlane,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
) -> anyhow::Result<DeviceBuffer> {
|
||||||
|
match self {
|
||||||
|
Importer::Remote(r) => r.import_linear_nv12(plane, width, height),
|
||||||
|
Importer::InProc(i) => i.import_linear_nv12(plane, width, height),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// True once the worker process is gone/wedged (every further call fails fast). Always
|
/// True once the worker process is gone/wedged (every further call fails fast). Always
|
||||||
/// `false` in-process — an in-process driver fault doesn't return.
|
/// `false` in-process — an in-process driver fault doesn't return.
|
||||||
pub fn dead(&self) -> bool {
|
pub fn dead(&self) -> bool {
|
||||||
|
|||||||
@@ -41,6 +41,10 @@ pub enum ImportKind {
|
|||||||
/// variants' wire tags must never shift — an old worker receiving this fails the decode and
|
/// variants' wire tags must never shift — an old worker receiving this fails the decode and
|
||||||
/// the import-fail machinery handles it like any other worker error.
|
/// the import-fail machinery handles it like any other worker error.
|
||||||
Tiled444,
|
Tiled444,
|
||||||
|
/// LINEAR dmabuf → Vulkan-bridge compute CSC → two-plane NV12 CUDA buffer (latency plan
|
||||||
|
/// T2.5b — the gamescope analogue of [`TiledNv12`](Self::TiledNv12)). Appended last, same
|
||||||
|
/// wire-tag rule as [`Tiled444`](Self::Tiled444).
|
||||||
|
LinearNv12,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// host → worker.
|
/// host → worker.
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
#version 450
|
||||||
|
// LINEAR BGRx dmabuf bytes -> NV12 (BT.709 limited range), BUFFER to BUFFER — the Vulkan-bridge
|
||||||
|
// CSC for the gamescope path (latency plan T2.5b). The bridge deals in VkBuffers (NVIDIA's EGL
|
||||||
|
// can't sample LINEAR and CUDA rejects raw dmabuf fds), so unlike `pf-encode`'s image-based
|
||||||
|
// `rgb2yuv.comp` sibling this reads packed texels straight out of the imported dmabuf buffer and
|
||||||
|
// writes the two NV12 planes into the exportable buffer CUDA reads. Same BT.709 coefficients as
|
||||||
|
// the sibling shader — keep the two in sync.
|
||||||
|
//
|
||||||
|
// One invocation converts a 4x2 luma block: 4 Y bytes = exactly one u32 word per row, and its
|
||||||
|
// two chroma samples = one u32 word — so every store is a whole word and no two invocations
|
||||||
|
// touch the same word (an 8-bit-storage-free way to write byte planes race-free). All pitches
|
||||||
|
// and the UV offset are in WORDS and must be word-aligned (the Rust side sizes them so).
|
||||||
|
//
|
||||||
|
// Rebuild: glslc rgb2nv12_buf.comp -o rgb2nv12_buf.spv
|
||||||
|
layout(local_size_x = 8, local_size_y = 8) in;
|
||||||
|
|
||||||
|
layout(std430, binding = 0) readonly buffer Src {
|
||||||
|
uint spx[]; // packed 4-byte texels, byte order B,G,R,X (little-endian XRGB8888/ARGB8888)
|
||||||
|
};
|
||||||
|
layout(std430, binding = 1) writeonly buffer Dst {
|
||||||
|
uint dw[]; // [0, uv_off_w): Y rows at y_pitch_w; [uv_off_w, ...): interleaved UV rows
|
||||||
|
};
|
||||||
|
|
||||||
|
layout(push_constant) uniform Push {
|
||||||
|
uint width; // source pixels
|
||||||
|
uint height;
|
||||||
|
uint src_off_w; // dmabuf plane offset, in words (offset bytes / 4)
|
||||||
|
uint src_pitch_w; // dmabuf row stride, in words
|
||||||
|
uint y_pitch_w; // dst Y row pitch, in words
|
||||||
|
uint uv_off_w; // dst UV plane offset, in words
|
||||||
|
uint uv_pitch_w; // dst UV row pitch, in words
|
||||||
|
} pc;
|
||||||
|
|
||||||
|
// Edge-clamped fetch: alignment padding (x >= width from the 4-wide block, odd-height chroma)
|
||||||
|
// duplicates the last real texel instead of reading out of bounds.
|
||||||
|
vec3 texel(uint x, uint y) {
|
||||||
|
x = min(x, pc.width - 1u);
|
||||||
|
y = min(y, pc.height - 1u);
|
||||||
|
uint t = spx[pc.src_off_w + y * pc.src_pitch_w + x];
|
||||||
|
// B,G,R,X byte order: r = bits 16..23, g = 8..15, b = 0..7.
|
||||||
|
return vec3(float((t >> 16) & 0xFFu), float((t >> 8) & 0xFFu), float(t & 0xFFu)) / 255.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// BT.709 limited range — numerically identical to rgb2yuv.comp's lumaY/U/V, scaled to bytes.
|
||||||
|
uint lumaB(vec3 c) {
|
||||||
|
return uint(clamp(16.0 + 255.0 * (0.1826 * c.r + 0.6142 * c.g + 0.0620 * c.b) + 0.5, 0.0, 255.0));
|
||||||
|
}
|
||||||
|
|
||||||
|
void main() {
|
||||||
|
uint bx = gl_GlobalInvocationID.x; // 4-px column block
|
||||||
|
uint by = gl_GlobalInvocationID.y; // 2-row block
|
||||||
|
uint x0 = bx * 4u;
|
||||||
|
uint y0 = by * 2u;
|
||||||
|
if (x0 >= pc.width || y0 >= pc.height) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Two Y words (4 bytes each, rows y0 and y0+1).
|
||||||
|
for (uint row = 0u; row < 2u; row++) {
|
||||||
|
uint y = y0 + row;
|
||||||
|
uint w = lumaB(texel(x0, y))
|
||||||
|
| (lumaB(texel(x0 + 1u, y)) << 8)
|
||||||
|
| (lumaB(texel(x0 + 2u, y)) << 16)
|
||||||
|
| (lumaB(texel(x0 + 3u, y)) << 24);
|
||||||
|
dw[y * pc.y_pitch_w + bx] = w;
|
||||||
|
}
|
||||||
|
// One UV word: the block's two chroma samples (each a 2x2 average).
|
||||||
|
uint uvw = 0u;
|
||||||
|
for (uint s = 0u; s < 2u; s++) {
|
||||||
|
uint cx = x0 + s * 2u;
|
||||||
|
vec3 a = (texel(cx, y0) + texel(cx + 1u, y0) + texel(cx, y0 + 1u) + texel(cx + 1u, y0 + 1u))
|
||||||
|
* 0.25;
|
||||||
|
uint U = uint(clamp(128.0 + 255.0 * (-0.1006 * a.r - 0.3386 * a.g + 0.4392 * a.b) + 0.5, 0.0, 255.0));
|
||||||
|
uint V = uint(clamp(128.0 + 255.0 * (0.4392 * a.r - 0.3989 * a.g - 0.0403 * a.b) + 0.5, 0.0, 255.0));
|
||||||
|
uvw |= (U | (V << 8)) << (16u * s);
|
||||||
|
}
|
||||||
|
dw[pc.uv_off_w + by * pc.uv_pitch_w + bx] = uvw;
|
||||||
|
}
|
||||||
Binary file not shown.
@@ -40,6 +40,20 @@ struct DstBuf {
|
|||||||
cuda: cuda::ExternalDmabuf,
|
cuda: cuda::ExternalDmabuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The lazy compute-CSC pipeline (`rgb2nv12_buf.comp`) for [`VkBridge::import_linear_nv12`].
|
||||||
|
struct Csc {
|
||||||
|
module: vk::ShaderModule,
|
||||||
|
dset_layout: vk::DescriptorSetLayout,
|
||||||
|
playout: vk::PipelineLayout,
|
||||||
|
pipeline: vk::Pipeline,
|
||||||
|
dpool: vk::DescriptorPool,
|
||||||
|
dset: vk::DescriptorSet,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The buffer-to-buffer RGB→NV12 compute shader (see `rgb2nv12_buf.comp` beside this file;
|
||||||
|
/// rebuild with `glslc rgb2nv12_buf.comp -o rgb2nv12_buf.spv`).
|
||||||
|
const CSC_SPV: &[u8] = include_bytes!("rgb2nv12_buf.spv");
|
||||||
|
|
||||||
pub struct VkBridge {
|
pub struct VkBridge {
|
||||||
_entry: ash::Entry,
|
_entry: ash::Entry,
|
||||||
instance: ash::Instance,
|
instance: ash::Instance,
|
||||||
@@ -52,6 +66,9 @@ pub struct VkBridge {
|
|||||||
mem_props: vk::PhysicalDeviceMemoryProperties,
|
mem_props: vk::PhysicalDeviceMemoryProperties,
|
||||||
src_cache: HashMap<i32, SrcBuf>,
|
src_cache: HashMap<i32, SrcBuf>,
|
||||||
dst: Option<DstBuf>,
|
dst: Option<DstBuf>,
|
||||||
|
/// Built on the first [`import_linear_nv12`](Self::import_linear_nv12); RGB-only bridges
|
||||||
|
/// never pay for it.
|
||||||
|
csc: Option<Csc>,
|
||||||
}
|
}
|
||||||
|
|
||||||
// SAFETY: `VkBridge` owns ash Vulkan handles (instance/device/queue/command pool+buffer/fence), a
|
// SAFETY: `VkBridge` owns ash Vulkan handles (instance/device/queue/command pool+buffer/fence), a
|
||||||
@@ -94,18 +111,15 @@ impl VkBridge {
|
|||||||
.ok_or_else(|| anyhow!("no NVIDIA Vulkan device"))?;
|
.ok_or_else(|| anyhow!("no NVIDIA Vulkan device"))?;
|
||||||
let mem_props = instance.get_physical_device_memory_properties(phys);
|
let mem_props = instance.get_physical_device_memory_properties(phys);
|
||||||
|
|
||||||
// Any queue family supporting transfer (graphics/compute imply it).
|
// A COMPUTE-capable family (compute implies transfer): the copy path only needs
|
||||||
|
// transfer, but the NV12 CSC dispatch (T2.5b) needs compute — on every NVIDIA
|
||||||
|
// device family 0 is graphics+compute+transfer, so this picks the same family the
|
||||||
|
// old transfer-only predicate did.
|
||||||
let qf = instance
|
let qf = instance
|
||||||
.get_physical_device_queue_family_properties(phys)
|
.get_physical_device_queue_family_properties(phys)
|
||||||
.iter()
|
.iter()
|
||||||
.position(|q| {
|
.position(|q| q.queue_flags.contains(vk::QueueFlags::COMPUTE))
|
||||||
q.queue_flags.intersects(
|
.ok_or_else(|| anyhow!("no compute-capable queue family"))?
|
||||||
vk::QueueFlags::TRANSFER
|
|
||||||
| vk::QueueFlags::GRAPHICS
|
|
||||||
| vk::QueueFlags::COMPUTE,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.ok_or_else(|| anyhow!("no transfer-capable queue family"))?
|
|
||||||
as u32;
|
as u32;
|
||||||
|
|
||||||
let exts = [
|
let exts = [
|
||||||
@@ -161,6 +175,7 @@ impl VkBridge {
|
|||||||
mem_props,
|
mem_props,
|
||||||
src_cache: HashMap::new(),
|
src_cache: HashMap::new(),
|
||||||
dst: None,
|
dst: None,
|
||||||
|
csc: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -189,7 +204,11 @@ impl VkBridge {
|
|||||||
.create_buffer(
|
.create_buffer(
|
||||||
&vk::BufferCreateInfo::default()
|
&vk::BufferCreateInfo::default()
|
||||||
.size(size)
|
.size(size)
|
||||||
.usage(vk::BufferUsageFlags::TRANSFER_SRC)
|
// STORAGE so the NV12 compute CSC can read it as an SSBO (T2.5b); harmless
|
||||||
|
// for the plain copy path.
|
||||||
|
.usage(
|
||||||
|
vk::BufferUsageFlags::TRANSFER_SRC | vk::BufferUsageFlags::STORAGE_BUFFER,
|
||||||
|
)
|
||||||
.push_next(&mut ext_info),
|
.push_next(&mut ext_info),
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
@@ -256,7 +275,10 @@ impl VkBridge {
|
|||||||
.create_buffer(
|
.create_buffer(
|
||||||
&vk::BufferCreateInfo::default()
|
&vk::BufferCreateInfo::default()
|
||||||
.size(size)
|
.size(size)
|
||||||
.usage(vk::BufferUsageFlags::TRANSFER_DST)
|
// STORAGE so the NV12 compute CSC can write it as an SSBO (T2.5b).
|
||||||
|
.usage(
|
||||||
|
vk::BufferUsageFlags::TRANSFER_DST | vk::BufferUsageFlags::STORAGE_BUFFER,
|
||||||
|
)
|
||||||
.push_next(&mut ext_info),
|
.push_next(&mut ext_info),
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
@@ -302,6 +324,246 @@ impl VkBridge {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Build the RGB→NV12 compute pipeline once (T2.5b): two-SSBO descriptor set + a 28-byte
|
||||||
|
/// push-constant block matching `rgb2nv12_buf.comp`'s `Push`.
|
||||||
|
unsafe fn ensure_csc(&mut self) -> Result<()> {
|
||||||
|
if self.csc.is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let words: Vec<u32> = CSC_SPV
|
||||||
|
.chunks_exact(4)
|
||||||
|
.map(|c| u32::from_le_bytes(c.try_into().unwrap()))
|
||||||
|
.collect();
|
||||||
|
let module = self
|
||||||
|
.device
|
||||||
|
.create_shader_module(&vk::ShaderModuleCreateInfo::default().code(&words), None)
|
||||||
|
.context("create CSC shader module")?;
|
||||||
|
let bindings = [
|
||||||
|
vk::DescriptorSetLayoutBinding::default()
|
||||||
|
.binding(0)
|
||||||
|
.descriptor_type(vk::DescriptorType::STORAGE_BUFFER)
|
||||||
|
.descriptor_count(1)
|
||||||
|
.stage_flags(vk::ShaderStageFlags::COMPUTE),
|
||||||
|
vk::DescriptorSetLayoutBinding::default()
|
||||||
|
.binding(1)
|
||||||
|
.descriptor_type(vk::DescriptorType::STORAGE_BUFFER)
|
||||||
|
.descriptor_count(1)
|
||||||
|
.stage_flags(vk::ShaderStageFlags::COMPUTE),
|
||||||
|
];
|
||||||
|
let dset_layout = self
|
||||||
|
.device
|
||||||
|
.create_descriptor_set_layout(
|
||||||
|
&vk::DescriptorSetLayoutCreateInfo::default().bindings(&bindings),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.context("create CSC dset layout")?;
|
||||||
|
let pc = [vk::PushConstantRange::default()
|
||||||
|
.stage_flags(vk::ShaderStageFlags::COMPUTE)
|
||||||
|
.size(28)];
|
||||||
|
let layouts = [dset_layout];
|
||||||
|
let playout = self
|
||||||
|
.device
|
||||||
|
.create_pipeline_layout(
|
||||||
|
&vk::PipelineLayoutCreateInfo::default()
|
||||||
|
.set_layouts(&layouts)
|
||||||
|
.push_constant_ranges(&pc),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.context("create CSC pipeline layout")?;
|
||||||
|
let entry = c"main";
|
||||||
|
let stage = vk::PipelineShaderStageCreateInfo::default()
|
||||||
|
.stage(vk::ShaderStageFlags::COMPUTE)
|
||||||
|
.module(module)
|
||||||
|
.name(entry);
|
||||||
|
let pipeline = self
|
||||||
|
.device
|
||||||
|
.create_compute_pipelines(
|
||||||
|
vk::PipelineCache::null(),
|
||||||
|
&[vk::ComputePipelineCreateInfo::default()
|
||||||
|
.stage(stage)
|
||||||
|
.layout(playout)],
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.map_err(|(_, e)| anyhow!("create CSC pipeline: {e}"))?[0];
|
||||||
|
let sizes = [vk::DescriptorPoolSize::default()
|
||||||
|
.ty(vk::DescriptorType::STORAGE_BUFFER)
|
||||||
|
.descriptor_count(2)];
|
||||||
|
let dpool = self
|
||||||
|
.device
|
||||||
|
.create_descriptor_pool(
|
||||||
|
&vk::DescriptorPoolCreateInfo::default()
|
||||||
|
.max_sets(1)
|
||||||
|
.pool_sizes(&sizes),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.context("create CSC descriptor pool")?;
|
||||||
|
let dset = self
|
||||||
|
.device
|
||||||
|
.allocate_descriptor_sets(
|
||||||
|
&vk::DescriptorSetAllocateInfo::default()
|
||||||
|
.descriptor_pool(dpool)
|
||||||
|
.set_layouts(&layouts),
|
||||||
|
)
|
||||||
|
.context("allocate CSC descriptor set")?[0];
|
||||||
|
self.csc = Some(Csc {
|
||||||
|
module,
|
||||||
|
dset_layout,
|
||||||
|
playout,
|
||||||
|
pipeline,
|
||||||
|
dpool,
|
||||||
|
dset,
|
||||||
|
});
|
||||||
|
tracing::info!("Vulkan-bridge NV12 compute CSC ready (LINEAR path feeds NVENC native YUV)");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bridge one LINEAR dmabuf frame into a pooled NV12 CUDA buffer (latency plan T2.5b):
|
||||||
|
/// instead of the plain byte copy, the compute CSC reads the imported RGB texels and writes
|
||||||
|
/// both NV12 planes into the exportable buffer, so NVENC on the gamescope path encodes
|
||||||
|
/// native YUV (its internal RGB→YUV CSC on the contended SM disappears). `pool` must be an
|
||||||
|
/// NV12 pool ([`cuda::BufferPool::new_nv12`]).
|
||||||
|
pub fn import_linear_nv12(
|
||||||
|
&mut self,
|
||||||
|
fd: i32,
|
||||||
|
offset: u32,
|
||||||
|
stride: u32,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
pool: &cuda::BufferPool,
|
||||||
|
) -> Result<DeviceBuffer> {
|
||||||
|
anyhow::ensure!(
|
||||||
|
offset % 4 == 0 && stride % 4 == 0,
|
||||||
|
"LINEAR dmabuf offset/stride not word-aligned ({offset}/{stride})"
|
||||||
|
);
|
||||||
|
// Exportable-buffer NV12 layout the shader writes: 4-aligned Y pitch, UV plane (⌈h/2⌉
|
||||||
|
// rows at the same pitch) directly after the Y plane.
|
||||||
|
let y_pitch = (width as u64 + 3) & !3;
|
||||||
|
let uv_off = y_pitch * height as u64;
|
||||||
|
let dst_size = uv_off + y_pitch * height.div_ceil(2) as u64;
|
||||||
|
// SAFETY: same structure and proofs as `import_linear` — `fd` is the caller's live dmabuf
|
||||||
|
// (dup'd by `import_src`), sizes are checked (`import_src` asserts the fd covers
|
||||||
|
// `offset + stride*height`; `ensure_dst(dst_size)` makes the exportable buffer at least
|
||||||
|
// the shader's whole write range, whose last word is `dst_size - 4`). The descriptor
|
||||||
|
// update binds the live cached src buffer and the live dst buffer WHOLE_SIZE; every
|
||||||
|
// `*Info`/array is a local outliving its synchronous call; `cmd`/`queue`/`fence` are this
|
||||||
|
// bridge's own single-thread handles. The dispatch covers ⌈w/32⌉×⌈h/16⌉ groups of 8×8
|
||||||
|
// invocations, each writing only whole words inside the proven dst range (shader
|
||||||
|
// contract). The host `wait_for_fences` retires the compute pass (with a shader-write →
|
||||||
|
// memory barrier recorded before end) BEFORE CUDA reads the shared memory.
|
||||||
|
unsafe {
|
||||||
|
let span = offset as u64 + stride as u64 * height as u64;
|
||||||
|
if !self.src_cache.contains_key(&fd) {
|
||||||
|
let size = libc::lseek(fd, 0, libc::SEEK_END);
|
||||||
|
anyhow::ensure!(size > 0, "lseek(dmabuf)");
|
||||||
|
anyhow::ensure!(size as u64 >= span, "dmabuf smaller than frame span");
|
||||||
|
self.import_src(fd, size as u64)?;
|
||||||
|
}
|
||||||
|
let src_buffer = self.src_cache[&fd].buffer;
|
||||||
|
self.ensure_dst(dst_size)?;
|
||||||
|
self.ensure_csc()?;
|
||||||
|
let (dst_buffer, dst_cuda_ptr) = {
|
||||||
|
let d = self.dst.as_ref().unwrap();
|
||||||
|
(d.buffer, d.cuda.ptr)
|
||||||
|
};
|
||||||
|
let csc = self.csc.as_ref().unwrap();
|
||||||
|
|
||||||
|
let src_info = [vk::DescriptorBufferInfo::default()
|
||||||
|
.buffer(src_buffer)
|
||||||
|
.range(vk::WHOLE_SIZE)];
|
||||||
|
let dst_info = [vk::DescriptorBufferInfo::default()
|
||||||
|
.buffer(dst_buffer)
|
||||||
|
.range(vk::WHOLE_SIZE)];
|
||||||
|
let writes = [
|
||||||
|
vk::WriteDescriptorSet::default()
|
||||||
|
.dst_set(csc.dset)
|
||||||
|
.dst_binding(0)
|
||||||
|
.descriptor_type(vk::DescriptorType::STORAGE_BUFFER)
|
||||||
|
.buffer_info(&src_info),
|
||||||
|
vk::WriteDescriptorSet::default()
|
||||||
|
.dst_set(csc.dset)
|
||||||
|
.dst_binding(1)
|
||||||
|
.descriptor_type(vk::DescriptorType::STORAGE_BUFFER)
|
||||||
|
.buffer_info(&dst_info),
|
||||||
|
];
|
||||||
|
self.device.update_descriptor_sets(&writes, &[]);
|
||||||
|
|
||||||
|
self.device
|
||||||
|
.begin_command_buffer(
|
||||||
|
self.cmd,
|
||||||
|
&vk::CommandBufferBeginInfo::default()
|
||||||
|
.flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT),
|
||||||
|
)
|
||||||
|
.context("begin cmd")?;
|
||||||
|
self.device
|
||||||
|
.cmd_bind_pipeline(self.cmd, vk::PipelineBindPoint::COMPUTE, csc.pipeline);
|
||||||
|
self.device.cmd_bind_descriptor_sets(
|
||||||
|
self.cmd,
|
||||||
|
vk::PipelineBindPoint::COMPUTE,
|
||||||
|
csc.playout,
|
||||||
|
0,
|
||||||
|
&[csc.dset],
|
||||||
|
&[],
|
||||||
|
);
|
||||||
|
let push: [u32; 7] = [
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
offset / 4,
|
||||||
|
stride / 4,
|
||||||
|
(y_pitch / 4) as u32,
|
||||||
|
(uv_off / 4) as u32,
|
||||||
|
(y_pitch / 4) as u32,
|
||||||
|
];
|
||||||
|
let push_bytes: &[u8] = std::slice::from_raw_parts(push.as_ptr().cast(), 28);
|
||||||
|
self.device.cmd_push_constants(
|
||||||
|
self.cmd,
|
||||||
|
csc.playout,
|
||||||
|
vk::ShaderStageFlags::COMPUTE,
|
||||||
|
0,
|
||||||
|
push_bytes,
|
||||||
|
);
|
||||||
|
self.device
|
||||||
|
.cmd_dispatch(self.cmd, width.div_ceil(32), height.div_ceil(16), 1);
|
||||||
|
// Make the shader writes available before the external (CUDA) read.
|
||||||
|
let barrier = vk::MemoryBarrier::default()
|
||||||
|
.src_access_mask(vk::AccessFlags::SHADER_WRITE)
|
||||||
|
.dst_access_mask(vk::AccessFlags::MEMORY_READ);
|
||||||
|
self.device.cmd_pipeline_barrier(
|
||||||
|
self.cmd,
|
||||||
|
vk::PipelineStageFlags::COMPUTE_SHADER,
|
||||||
|
vk::PipelineStageFlags::BOTTOM_OF_PIPE,
|
||||||
|
vk::DependencyFlags::empty(),
|
||||||
|
&[barrier],
|
||||||
|
&[],
|
||||||
|
&[],
|
||||||
|
);
|
||||||
|
self.device
|
||||||
|
.end_command_buffer(self.cmd)
|
||||||
|
.context("end cmd")?;
|
||||||
|
let cmds = [self.cmd];
|
||||||
|
let submit = vk::SubmitInfo::default().command_buffers(&cmds);
|
||||||
|
self.device
|
||||||
|
.queue_submit(self.queue, &[submit], self.fence)
|
||||||
|
.context("queue submit")?;
|
||||||
|
self.device
|
||||||
|
.wait_for_fences(&[self.fence], true, 1_000_000_000)
|
||||||
|
.context("fence wait")?;
|
||||||
|
self.device
|
||||||
|
.reset_fences(&[self.fence])
|
||||||
|
.context("reset fence")?;
|
||||||
|
|
||||||
|
// De-stride both NV12 planes from the CUDA view into a pooled two-plane buffer.
|
||||||
|
cuda::make_current()?;
|
||||||
|
let out = pool.get()?;
|
||||||
|
cuda::copy_pitched_nv12_to_buffer(
|
||||||
|
dst_cuda_ptr,
|
||||||
|
dst_cuda_ptr + uv_off,
|
||||||
|
y_pitch as usize,
|
||||||
|
&out,
|
||||||
|
)?;
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Drop the cached import for `fd` (the PipeWire buffer it wrapped is gone — pool recycle /
|
/// Drop the cached import for `fd` (the PipeWire buffer it wrapped is gone — pool recycle /
|
||||||
/// renegotiation — or the caller is about to store a different dmabuf under the same slot).
|
/// renegotiation — or the caller is about to store a different dmabuf under the same slot).
|
||||||
/// Without this the cache could serve a stale imported buffer for a reused fd number, or
|
/// Without this the cache could serve a stale imported buffer for a reused fd number, or
|
||||||
@@ -414,6 +676,14 @@ impl Drop for VkBridge {
|
|||||||
self.device.destroy_buffer(d.buffer, None);
|
self.device.destroy_buffer(d.buffer, None);
|
||||||
self.device.free_memory(d.memory, None);
|
self.device.free_memory(d.memory, None);
|
||||||
}
|
}
|
||||||
|
if let Some(c) = self.csc.take() {
|
||||||
|
self.device.destroy_pipeline(c.pipeline, None);
|
||||||
|
self.device.destroy_pipeline_layout(c.playout, None);
|
||||||
|
self.device.destroy_descriptor_pool(c.dpool, None); // frees `c.dset` with it
|
||||||
|
self.device
|
||||||
|
.destroy_descriptor_set_layout(c.dset_layout, None);
|
||||||
|
self.device.destroy_shader_module(c.module, None);
|
||||||
|
}
|
||||||
self.device.destroy_fence(self.fence, None);
|
self.device.destroy_fence(self.fence, None);
|
||||||
self.device.destroy_command_pool(self.cmd_pool, None);
|
self.device.destroy_command_pool(self.cmd_pool, None);
|
||||||
self.device.destroy_device(None);
|
self.device.destroy_device(None);
|
||||||
|
|||||||
@@ -299,6 +299,9 @@ impl EglBackend {
|
|||||||
req.modifier,
|
req.modifier,
|
||||||
)?,
|
)?,
|
||||||
ImportKind::Linear => self.importer.import_linear(&plane, req.width, req.height)?,
|
ImportKind::Linear => self.importer.import_linear(&plane, req.width, req.height)?,
|
||||||
|
ImportKind::LinearNv12 => self
|
||||||
|
.importer
|
||||||
|
.import_linear_nv12(&plane, req.width, req.height)?,
|
||||||
};
|
};
|
||||||
// Assign / look up the buffer's id and export its CUDA IPC identity on first delivery.
|
// Assign / look up the buffer's id and export its CUDA IPC identity on first delivery.
|
||||||
cuda::make_current()?;
|
cuda::make_current()?;
|
||||||
|
|||||||
Reference in New Issue
Block a user