From a02d0a2e9f8ffd4f2fa0cb99573aed30fe3a066d Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Tue, 7 Jul 2026 18:12:16 +0200 Subject: [PATCH] =?UTF-8?q?feat(presenter):=20VAAPI=20dmabuf=20=E2=86=92?= =?UTF-8?q?=20Vulkan=20zero-copy=20import=20+=20CSC=20pass=20(phase=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decoder's NV12 dmabuf imports per-plane (R8 + GR88, explicit DRM format modifier, dedicated dup'd-fd import, FOREIGN→graphics acquire) and a fullscreen-triangle render pass converts it into the presenter's video image with the CICP-driven coefficients ported from video_gl.rs (same tests, plus a rows-vs-matrix agreement check). SPIR-V is committed (shaders/build.sh regenerates) so builds and CI need no toolchain. The import extension set is probed at device creation; unsupported boxes and 3-failure streaks demote the decoder to software via the existing force_software contract. The session binary now honors the Settings decoder preference instead of forcing software. Co-Authored-By: Claude Fable 5 --- clients/linux-session/README.md | 8 +- clients/linux-session/src/main.rs | 6 +- crates/pf-presenter/shaders/build.sh | 10 + crates/pf-presenter/shaders/fullscreen.vert | 14 + .../pf-presenter/shaders/fullscreen.vert.spv | Bin 0 -> 916 bytes crates/pf-presenter/shaders/nv12_csc.frag | 36 ++ crates/pf-presenter/shaders/nv12_csc.frag.spv | Bin 0 -> 1580 bytes crates/pf-presenter/src/csc.rs | 398 ++++++++++++++++++ crates/pf-presenter/src/dmabuf.rs | 287 +++++++++++++ crates/pf-presenter/src/lib.rs | 13 +- crates/pf-presenter/src/run.rs | 79 +++- crates/pf-presenter/src/vk.rs | 296 ++++++++++++- 12 files changed, 1091 insertions(+), 56 deletions(-) create mode 100755 crates/pf-presenter/shaders/build.sh create mode 100644 crates/pf-presenter/shaders/fullscreen.vert create mode 100644 crates/pf-presenter/shaders/fullscreen.vert.spv create mode 100644 crates/pf-presenter/shaders/nv12_csc.frag create mode 100644 crates/pf-presenter/shaders/nv12_csc.frag.spv create mode 100644 crates/pf-presenter/src/csc.rs create mode 100644 crates/pf-presenter/src/dmabuf.rs diff --git a/clients/linux-session/README.md b/clients/linux-session/README.md index 3b1d5194..44ef2937 100644 --- a/clients/linux-session/README.md +++ b/clients/linux-session/README.md @@ -23,6 +23,8 @@ In-stream keys match the desktop client: click captures input (Ctrl+Alt+Shift+Q releases), Ctrl+Alt+Shift+D disconnects, F11 toggles fullscreen; the controller escape chord (L1+R1+Start+Select, hold to disconnect) works the same. -Current phase (1): software decode presented via a transfer-only blit — request path -`--connect` only. The VAAPI dmabuf → Vulkan zero-copy import, HDR/P010, and the Skia -console UI (`--browse`) are later phases of the plan. +Decode follows the Settings preference: VAAPI frames import zero-copy into Vulkan +(per-plane dmabuf + the stream's CICP-driven CSC shader); boxes whose driver can't +import (NVIDIA proprietary by design) fall back to software decode automatically — +`PUNKTFUNK_DECODER=software|vaapi` overrides for bisects. HDR/P010 and the Skia console +UI (`--browse`) are later phases of the plan. diff --git a/clients/linux-session/src/main.rs b/clients/linux-session/src/main.rs index 750042eb..417313a8 100644 --- a/clients/linux-session/src/main.rs +++ b/clients/linux-session/src/main.rs @@ -180,10 +180,10 @@ mod session_main { audio_channels: settings.audio_channels, preferred_codec: settings.preferred_codec(), mic_enabled: settings.mic_enabled, - // Phase 1 presents the software path only (the Vulkan dmabuf import is - // Phase 2) — request it outright instead of demoting after one frame. + // The Settings preference (auto → VAAPI where it exists; the presenter + // demotes to software on boxes whose Vulkan can't import the dmabufs). // PUNKTFUNK_DECODER still overrides inside the decoder for bisects. - decoder: "software".into(), + decoder: settings.decoder.clone(), launch, pin: Some(pin), identity, diff --git a/crates/pf-presenter/shaders/build.sh b/crates/pf-presenter/shaders/build.sh new file mode 100755 index 00000000..b98e76c5 --- /dev/null +++ b/crates/pf-presenter/shaders/build.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Recompile the presenter's shaders to the committed .spv blobs. The blobs are committed +# so neither developer builds nor CI need a SPIR-V toolchain (glslc is a dev-time tool); +# rerun this after editing a .vert/.frag and commit both. +set -euo pipefail +cd "$(dirname "$0")" +for f in *.vert *.frag; do + glslc -O "$f" -o "$f.spv" + echo "compiled $f -> $f.spv" +done diff --git a/crates/pf-presenter/shaders/fullscreen.vert b/crates/pf-presenter/shaders/fullscreen.vert new file mode 100644 index 00000000..b5bf0066 --- /dev/null +++ b/crates/pf-presenter/shaders/fullscreen.vert @@ -0,0 +1,14 @@ +// The bufferless fullscreen triangle (same trick as the GL presenter's gl_VertexID +// geometry). Vulkan clip space: y=-1 is the TOP, so uv (0,0) lands on the top-left — +// which is exactly where the video's row 0 lives. No flip anywhere. +// +// Regenerate: shaders/build.sh (committed .spv, no build-time toolchain). +#version 450 + +layout(location = 0) out vec2 v_uv; + +void main() { + vec2 uv = vec2((gl_VertexIndex << 1) & 2, gl_VertexIndex & 2); + v_uv = uv; + gl_Position = vec4(uv * 2.0 - 1.0, 0.0, 1.0); +} diff --git a/crates/pf-presenter/shaders/fullscreen.vert.spv b/crates/pf-presenter/shaders/fullscreen.vert.spv new file mode 100644 index 0000000000000000000000000000000000000000..b4c1a0a0f7bf57441754d86e0204a5f6ef09d1ba GIT binary patch literal 916 zcmZ9J%Pxaa6opT#E>T6@DinWHg0{*4k%-%<1_s0vNK6bc-~lA&o=W1COeDVV?>UH* ztnR(mS^wVq><$#0!yyzxIaI=S=$~pRq6D;|JoYZGFS<|9x82>n9UbFgAUEon2qSr3 z;`jdM?m;pJ=0OH}p_KD7c`I`7mVu{7E%PEc4}(9teoxZ+^Em0R<~RDBCi%?OoM)P< zdKx&?&F9F5h#9Ei;_*56ZWcaD%z%9|xV6mH4ZWbQ82@J`rwg=i1&?JLiKTAI5-?vY!wnX_=3 z_*%|iP7>ef-{%R(z23PucXCDz_qGKx;QpOOuK$Y)2jClD?jYp= literal 0 HcmV?d00001 diff --git a/crates/pf-presenter/shaders/nv12_csc.frag b/crates/pf-presenter/shaders/nv12_csc.frag new file mode 100644 index 00000000..6df1239d --- /dev/null +++ b/crates/pf-presenter/shaders/nv12_csc.frag @@ -0,0 +1,36 @@ +// NV12 → RGBA with the stream's CICP signaling — the Vulkan port of the GL presenter's +// fragment shader (video_gl.rs). The YUV→RGB matrix + range expansion arrive as three +// push-constant rows precomputed on the CPU (csc.rs `csc_rows`): rgb[i] = dot(r_i.xyz, +// yuv) + r_i.w — one shader for BT.601/709/2020, full/limited. The chroma plane is +// half-res; the linear sampler interpolates, same as the GL path. +// +// Regenerate: shaders/build.sh (committed .spv, no build-time toolchain). +#version 450 + +layout(location = 0) in vec2 v_uv; +layout(location = 0) out vec4 frag; + +layout(set = 0, binding = 0) uniform sampler2D u_y; +layout(set = 0, binding = 1) uniform sampler2D u_c; + +layout(push_constant) uniform Csc { + vec4 r0; + vec4 r1; + vec4 r2; +} pc; + +void main() { + vec3 yuv = vec3(texture(u_y, v_uv).r, texture(u_c, v_uv).rg); + frag = vec4( + clamp( + vec3( + dot(pc.r0.xyz, yuv) + pc.r0.w, + dot(pc.r1.xyz, yuv) + pc.r1.w, + dot(pc.r2.xyz, yuv) + pc.r2.w + ), + 0.0, + 1.0 + ), + 1.0 + ); +} diff --git a/crates/pf-presenter/shaders/nv12_csc.frag.spv b/crates/pf-presenter/shaders/nv12_csc.frag.spv new file mode 100644 index 0000000000000000000000000000000000000000..4b82da407ba3625087e0098600641740071ff02f GIT binary patch literal 1580 zcmZ9MOHWim5QUo=0g<gw)0G#l;Fl!nrHnoRvPu-VjrNpK_edf5BW>wN$5xwE--N5xcX)hSz&HdLm>s$lw zC=H+J+QeFL_L7?^IBVo8Fvr=ATg>h$^)KR$uvTD!IXsWC`NcGr&gIzNrA54ia-4*^ ziu#ExlH19#cca;Vn$`a9t=?JfC9mGtyOL8s$Ex1Dp6Bm8YX!9E@7-zkt5}`gI`hUY zAh(?F&RoEnHt`r(0oE9=X3jmY;#Xh@tmk+QU$g!itM&FY?~J#Adz$aCn)e)0e-FRx zw?pimm{<1#Xc51xqn}SV$ofX_fSN$?$8c%`4>+@Zy+6&F{=f{g?!x*`Q@}Se_8rax z-{2!E&vXH(k$(^8d<#Hci$06^zD3k6;mehEmx<-I=;e1V=Y90^`L5u*hkEl}_Vu2x z1K(y1xUciw0P@G_Y<)L3GbgXv*E{%~XWzhA;~ri1^L=iE=2AL#)-lmGw# literal 0 HcmV?d00001 diff --git a/crates/pf-presenter/src/csc.rs b/crates/pf-presenter/src/csc.rs new file mode 100644 index 00000000..c21bd194 --- /dev/null +++ b/crates/pf-presenter/src/csc.rs @@ -0,0 +1,398 @@ +//! The NV12→RGBA color-space-conversion pass: coefficient rows from the stream's CICP +//! signaling (the GL presenter's `yuv_to_rgb`, folded into row form for the shader's +//! push constants) and the graphics pipeline that renders the two imported planes into +//! the presenter's video image. +//! +//! Deliberately NOT `VK_KHR_sampler_ycbcr_conversion`: the ext can't express the PQ path +//! coming with the HDR phase and its range handling varies by driver — these are the +//! proven, unit-tested coefficients (plan §5.2). + +use anyhow::{Context as _, Result}; +use ash::vk; +use pf_client_core::video::ColorDesc; + +/// The push-constant block: three vec4 rows, `rgb[i] = dot(r[i].xyz, yuv) + r[i].w`. +/// One layout for BT.601/709/2020 × full/limited; PQ transfer joins in the HDR phase. +pub fn csc_rows(desc: ColorDesc) -> [[f32; 4]; 3] { + // BT.601 (5/6), BT.2020 (9/10); everything else — incl. unspecified — is the host's + // BT.709 SDR default (mirrors the software path's swscale coefficient choice). + let (kr, kb) = match desc.matrix { + 5 | 6 => (0.299, 0.114), + 9 | 10 => (0.2627, 0.0593), + _ => (0.2126, 0.0722), + }; + let kg = 1.0 - kr - kb; + let (sy, oy, sc) = if desc.full_range { + (1.0f64, 0.0f64, 1.0f64) + } else { + (255.0 / 219.0, -16.0 / 255.0, 255.0 / 224.0) + }; + // rgb = M * (yuv + off) = M*yuv + M*off — rows of M with the offset dot folded into w. + let off = [oy, -0.5, -0.5]; + let m = [ + [sy, 0.0, 2.0 * (1.0 - kr) * sc], + [ + sy, + -2.0 * (1.0 - kb) * kb / kg * sc, + -2.0 * (1.0 - kr) * kr / kg * sc, + ], + [sy, 2.0 * (1.0 - kb) * sc, 0.0], + ]; + core::array::from_fn(|r| { + let w: f64 = (0..3).map(|c| m[r][c] * off[c]).sum(); + [m[r][0] as f32, m[r][1] as f32, m[r][2] as f32, w as f32] + }) +} + +/// The pass objects (everything except the per-video-size framebuffer, which lives with +/// the video image). Destroyed explicitly via [`CscPass::destroy`] from the presenter's +/// `Drop` — no device handle is stored here. +pub struct CscPass { + pub render_pass: vk::RenderPass, + pub set_layout: vk::DescriptorSetLayout, + pub pipeline_layout: vk::PipelineLayout, + pub pipeline: vk::Pipeline, + pub desc_pool: vk::DescriptorPool, + pub desc_set: vk::DescriptorSet, + pub sampler: vk::Sampler, +} + +impl CscPass { + pub fn new(device: &ash::Device) -> Result { + // One color attachment: the presenter's R8G8B8A8 video image. Content is fully + // overwritten (DONT_CARE load), and the pass ends in TRANSFER_SRC so the + // existing letterbox blit consumes it with no extra barrier. + let attachment = [vk::AttachmentDescription::default() + .format(vk::Format::R8G8B8A8_UNORM) + .samples(vk::SampleCountFlags::TYPE_1) + .load_op(vk::AttachmentLoadOp::DONT_CARE) + .store_op(vk::AttachmentStoreOp::STORE) + .initial_layout(vk::ImageLayout::UNDEFINED) + .final_layout(vk::ImageLayout::TRANSFER_SRC_OPTIMAL)]; + let color_ref = [vk::AttachmentReference::default() + .attachment(0) + .layout(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL)]; + let subpass = [vk::SubpassDescription::default() + .pipeline_bind_point(vk::PipelineBindPoint::GRAPHICS) + .color_attachments(&color_ref)]; + // Conservative scopes, matching the presenter's per-frame barrier granularity. + let deps = [ + vk::SubpassDependency::default() + .src_subpass(vk::SUBPASS_EXTERNAL) + .dst_subpass(0) + .src_stage_mask(vk::PipelineStageFlags::ALL_COMMANDS) + .src_access_mask(vk::AccessFlags::MEMORY_WRITE) + .dst_stage_mask(vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT) + .dst_access_mask(vk::AccessFlags::COLOR_ATTACHMENT_WRITE), + vk::SubpassDependency::default() + .src_subpass(0) + .dst_subpass(vk::SUBPASS_EXTERNAL) + .src_stage_mask(vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT) + .src_access_mask(vk::AccessFlags::COLOR_ATTACHMENT_WRITE) + .dst_stage_mask(vk::PipelineStageFlags::TRANSFER) + .dst_access_mask(vk::AccessFlags::TRANSFER_READ), + ]; + let render_pass = unsafe { + device.create_render_pass( + &vk::RenderPassCreateInfo::default() + .attachments(&attachment) + .subpasses(&subpass) + .dependencies(&deps), + None, + ) + } + .context("CSC render pass")?; + + let sampler = unsafe { + device.create_sampler( + &vk::SamplerCreateInfo::default() + .mag_filter(vk::Filter::LINEAR) + .min_filter(vk::Filter::LINEAR) + .address_mode_u(vk::SamplerAddressMode::CLAMP_TO_EDGE) + .address_mode_v(vk::SamplerAddressMode::CLAMP_TO_EDGE) + .address_mode_w(vk::SamplerAddressMode::CLAMP_TO_EDGE), + None, + ) + }?; + + let samplers = [sampler]; + let bindings = [ + vk::DescriptorSetLayoutBinding::default() + .binding(0) + .descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) + .descriptor_count(1) + .stage_flags(vk::ShaderStageFlags::FRAGMENT) + .immutable_samplers(&samplers), + vk::DescriptorSetLayoutBinding::default() + .binding(1) + .descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) + .descriptor_count(1) + .stage_flags(vk::ShaderStageFlags::FRAGMENT) + .immutable_samplers(&samplers), + ]; + let set_layout = unsafe { + device.create_descriptor_set_layout( + &vk::DescriptorSetLayoutCreateInfo::default().bindings(&bindings), + None, + ) + }?; + let set_layouts = [set_layout]; + let push = [vk::PushConstantRange::default() + .stage_flags(vk::ShaderStageFlags::FRAGMENT) + .size(48)]; // three vec4 rows + let pipeline_layout = unsafe { + device.create_pipeline_layout( + &vk::PipelineLayoutCreateInfo::default() + .set_layouts(&set_layouts) + .push_constant_ranges(&push), + None, + ) + }?; + + let pool_sizes = [vk::DescriptorPoolSize::default() + .ty(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) + .descriptor_count(2)]; + let desc_pool = unsafe { + device.create_descriptor_pool( + &vk::DescriptorPoolCreateInfo::default() + .max_sets(1) + .pool_sizes(&pool_sizes), + None, + ) + }?; + let desc_set = unsafe { + device.allocate_descriptor_sets( + &vk::DescriptorSetAllocateInfo::default() + .descriptor_pool(desc_pool) + .set_layouts(&set_layouts), + ) + }?[0]; + + let pipeline = build_pipeline(device, render_pass, pipeline_layout)?; + + Ok(CscPass { + render_pass, + set_layout, + pipeline_layout, + pipeline, + desc_pool, + desc_set, + sampler, + }) + } + + /// Point the descriptor set at this frame's plane views. Only safe while no + /// submitted command buffer references the set — the presenter's single in-flight + /// fence is waited before every record, which covers it. + pub fn bind_planes(&self, device: &ash::Device, luma: vk::ImageView, chroma: vk::ImageView) { + let infos = [luma, chroma].map(|view| { + [vk::DescriptorImageInfo::default() + .image_view(view) + .image_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL)] + }); + let writes = [ + vk::WriteDescriptorSet::default() + .dst_set(self.desc_set) + .dst_binding(0) + .descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) + .image_info(&infos[0]), + vk::WriteDescriptorSet::default() + .dst_set(self.desc_set) + .dst_binding(1) + .descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) + .image_info(&infos[1]), + ]; + unsafe { device.update_descriptor_sets(&writes, &[]) }; + } + + pub fn destroy(&self, device: &ash::Device) { + unsafe { + device.destroy_pipeline(self.pipeline, None); + device.destroy_pipeline_layout(self.pipeline_layout, None); + device.destroy_descriptor_pool(self.desc_pool, None); + device.destroy_descriptor_set_layout(self.set_layout, None); + device.destroy_sampler(self.sampler, None); + device.destroy_render_pass(self.render_pass, None); + } + } +} + +fn build_pipeline( + device: &ash::Device, + render_pass: vk::RenderPass, + layout: vk::PipelineLayout, +) -> Result { + // Committed SPIR-V (shaders/build.sh) — include_bytes! alignment is unspecified, so + // read_spv copies into aligned Vecs. + let vert = ash::util::read_spv(&mut std::io::Cursor::new( + &include_bytes!("../shaders/fullscreen.vert.spv")[..], + ))?; + let frag = ash::util::read_spv(&mut std::io::Cursor::new( + &include_bytes!("../shaders/nv12_csc.frag.spv")[..], + ))?; + let vert_mod = unsafe { + device.create_shader_module(&vk::ShaderModuleCreateInfo::default().code(&vert), None) + }?; + let frag_mod = unsafe { + device.create_shader_module(&vk::ShaderModuleCreateInfo::default().code(&frag), None) + }; + let frag_mod = match frag_mod { + Ok(m) => m, + Err(e) => { + unsafe { device.destroy_shader_module(vert_mod, None) }; + return Err(e).context("fragment shader module"); + } + }; + + let entry = c"main"; + let stages = [ + vk::PipelineShaderStageCreateInfo::default() + .stage(vk::ShaderStageFlags::VERTEX) + .module(vert_mod) + .name(entry), + vk::PipelineShaderStageCreateInfo::default() + .stage(vk::ShaderStageFlags::FRAGMENT) + .module(frag_mod) + .name(entry), + ]; + let vertex_input = vk::PipelineVertexInputStateCreateInfo::default(); // bufferless + let assembly = vk::PipelineInputAssemblyStateCreateInfo::default() + .topology(vk::PrimitiveTopology::TRIANGLE_LIST); + // Dynamic viewport/scissor: the video size changes with the stream mode, the + // pipeline must not bake one in. + let viewport = vk::PipelineViewportStateCreateInfo::default() + .viewport_count(1) + .scissor_count(1); + let dynamic = [vk::DynamicState::VIEWPORT, vk::DynamicState::SCISSOR]; + let dynamic_state = vk::PipelineDynamicStateCreateInfo::default().dynamic_states(&dynamic); + let raster = vk::PipelineRasterizationStateCreateInfo::default() + .polygon_mode(vk::PolygonMode::FILL) + .cull_mode(vk::CullModeFlags::NONE) + .line_width(1.0); + let multisample = vk::PipelineMultisampleStateCreateInfo::default() + .rasterization_samples(vk::SampleCountFlags::TYPE_1); + let blend_attachment = [vk::PipelineColorBlendAttachmentState::default() + .color_write_mask(vk::ColorComponentFlags::RGBA)]; + let blend = vk::PipelineColorBlendStateCreateInfo::default().attachments(&blend_attachment); + + let info = vk::GraphicsPipelineCreateInfo::default() + .stages(&stages) + .vertex_input_state(&vertex_input) + .input_assembly_state(&assembly) + .viewport_state(&viewport) + .rasterization_state(&raster) + .multisample_state(&multisample) + .color_blend_state(&blend) + .dynamic_state(&dynamic_state) + .layout(layout) + .render_pass(render_pass); + let pipeline = unsafe { + device.create_graphics_pipelines(vk::PipelineCache::null(), &[info], None) + } + .map_err(|(_, e)| e) + .context("CSC pipeline"); + unsafe { + device.destroy_shader_module(vert_mod, None); + device.destroy_shader_module(frag_mod, None); + } + Ok(pipeline?[0]) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn desc(matrix: u8, full_range: bool) -> ColorDesc { + ColorDesc { + primaries: 1, + transfer: 1, + matrix, + full_range, + } + } + + fn apply(rows: &[[f32; 4]; 3], yuv: [f32; 3]) -> [f32; 3] { + core::array::from_fn(|r| { + rows[r][0] * yuv[0] + rows[r][1] * yuv[1] + rows[r][2] * yuv[2] + rows[r][3] + }) + } + + /// Reference white (Y=235, U=V=128 limited) → RGB 1.0; reference black (Y=16) → 0.0 + /// — the GL presenter's test, in row form. + #[test] + fn bt709_limited_white_black() { + let rows = csc_rows(desc(1, false)); + let white = apply(&rows, [235.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0]); + let black = apply(&rows, [16.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0]); + for (w, b) in white.iter().zip(black) { + assert!((w - 1.0).abs() < 0.005, "white {white:?}"); + assert!(b.abs() < 0.005, "black {black:?}"); + } + } + + /// Full-range identity points + the 601-vs-709 red excursion (guards the + /// matrix-code dispatch), same as the GL presenter's test. + #[test] + fn full_range_and_red_excursion() { + let rows = csc_rows(desc(5, true)); + let white = apply(&rows, [1.0, 0.5, 0.5]); + assert!(white.iter().all(|v| (v - 1.0).abs() < 1e-5), "{white:?}"); + let red = apply(&rows, [0.0, 0.5, 1.0]); + assert!((red[0] - 2.0 * (1.0 - 0.299) * 0.5).abs() < 1e-4, "{red:?}"); + let rows709 = csc_rows(desc(1, true)); + let red709 = apply(&rows709, [0.0, 0.5, 1.0]); + assert!( + (red709[0] - 2.0 * (1.0 - 0.2126) * 0.5).abs() < 1e-4, + "{red709:?}" + ); + assert!((red[0] - red709[0]).abs() > 0.05); + } + + /// The row form must agree with the GL presenter's column-major `yuv_to_rgb` on a + /// grid of inputs — same math, different packing. + #[test] + fn rows_match_the_gl_matrix_form() { + for (matrix, full) in [(1u8, false), (1, true), (5, false), (9, false), (9, true)] { + let d = desc(matrix, full); + let rows = csc_rows(d); + // Reimplementation of video_gl::yuv_to_rgb's application for comparison. + let (kr, kb) = match matrix { + 5 | 6 => (0.299f32, 0.114f32), + 9 | 10 => (0.2627, 0.0593), + _ => (0.2126, 0.0722), + }; + let kg = 1.0 - kr - kb; + let (sy, oy, sc) = if full { + (1.0f32, 0.0f32, 1.0f32) + } else { + (255.0 / 219.0, -16.0 / 255.0, 255.0 / 224.0) + }; + let mat = [ + sy, + sy, + sy, + 0.0, + -2.0 * (1.0 - kb) * kb / kg * sc, + 2.0 * (1.0 - kb) * sc, + 2.0 * (1.0 - kr) * sc, + -2.0 * (1.0 - kr) * kr / kg * sc, + 0.0, + ]; + let off = [oy, -0.5, -0.5]; + for yuv in [ + [0.1f32, 0.3, 0.7], + [0.9, 0.5, 0.5], + [0.5, 0.2, 0.8], + [16.0 / 255.0, 0.5, 0.5], + ] { + let v = [yuv[0] + off[0], yuv[1] + off[1], yuv[2] + off[2]]; + let gl: [f32; 3] = + core::array::from_fn(|r| (0..3).map(|c| mat[c * 3 + r] * v[c]).sum()); + let ours = apply(&rows, yuv); + for (a, b) in gl.iter().zip(ours) { + assert!((a - b).abs() < 1e-5, "{matrix}/{full}: gl {gl:?} rows {ours:?}"); + } + } + } + } +} diff --git a/crates/pf-presenter/src/dmabuf.rs b/crates/pf-presenter/src/dmabuf.rs new file mode 100644 index 00000000..2818ab20 --- /dev/null +++ b/crates/pf-presenter/src/dmabuf.rs @@ -0,0 +1,287 @@ +//! VAAPI dmabuf → Vulkan import: per-plane `VkImage`s (R8 luma + GR88 chroma) with the +//! surface's explicit DRM format modifier — the same layer-wise import the EGL presenter +//! (`video_gl.rs`) proved on this hardware, minus the toolkit. Same-Mesa export/import +//! is the contract; anything a driver rejects surfaces as a clean error and the caller +//! demotes the decoder to software (never a black screen). + +use anyhow::{bail, Context as _, Result}; +use ash::vk; +use pf_client_core::video::{DmabufFrame, DrmFrameGuard}; +use std::os::fd::{BorrowedFd, IntoRawFd as _}; + +/// `fourcc('N','V','1','2')` — the only VAAPI decoder output today (8-bit 4:2:0). +const DRM_FORMAT_NV12: u32 = 0x3231_564e; +const DRM_FORMAT_MOD_INVALID: u64 = 0x00ff_ffff_ffff_ffff; +/// `DRM_FORMAT_MOD_LINEAR` — the fallback when the export carried no explicit modifier. +const DRM_FORMAT_MOD_LINEAR: u64 = 0; + +/// The four device extensions the import path needs; queried at device creation. All +/// Mesa drivers (RADV/ANV/radeonsi boxes) expose the set — NVIDIA proprietary has no +/// usable VAAPI anyway, so the software path owns that vendor by design. +pub const DEVICE_EXTENSIONS: [&std::ffi::CStr; 4] = [ + ash::ext::external_memory_dma_buf::NAME, + ash::khr::external_memory_fd::NAME, + ash::ext::image_drm_format_modifier::NAME, + ash::ext::queue_family_foreign::NAME, +]; + +/// One imported frame: both plane images + their memory, and the decoder surface guard. +/// GPU reads outlive the submit — the presenter parks this until the frame's fence has +/// signaled, then calls [`HwFrame::destroy`] (which finally drops the guard). +pub struct HwFrame { + pub luma_view: vk::ImageView, + pub chroma_view: vk::ImageView, + pub color: pf_client_core::video::ColorDesc, + pub width: u32, + pub height: u32, + images: [vk::Image; 2], + memories: [vk::DeviceMemory; 2], + views: [vk::ImageView; 2], + _guard: DrmFrameGuard, +} + +impl HwFrame { + /// The raw plane images — the presenter's foreign-acquire barriers need them. + pub fn luma_image(&self) -> vk::Image { + self.images[0] + } + + pub fn chroma_image(&self) -> vk::Image { + self.images[1] + } + + pub fn destroy(self, device: &ash::Device) { + unsafe { + for v in self.views { + device.destroy_image_view(v, None); + } + for i in self.images { + device.destroy_image(i, None); + } + for m in self.memories { + device.free_memory(m, None); + } + } + // _guard (the mapped AVFrame / VAAPI surface) drops here — after every GPU read. + } +} + +/// Import one frame's two planes. Fails cleanly (caller demotes) on anything the driver +/// rejects: unknown fourcc, unsupported modifier, import refusal. +pub fn import( + device: &ash::Device, + ext_mem_fd: &ash::khr::external_memory_fd::Device, + frame: DmabufFrame, +) -> Result { + if frame.fourcc != DRM_FORMAT_NV12 { + bail!("hw presenter handles NV12 only (got {:#x})", frame.fourcc); + } + if frame.planes.len() < 2 { + bail!("NV12 needs 2 planes (got {})", frame.planes.len()); + } + // EGL could leave an INVALID modifier to the driver's implied choice; explicit- + // modifier images can't — LINEAR is the only honest guess (debug-visible if wrong). + let modifier = if frame.modifier == DRM_FORMAT_MOD_INVALID { + tracing::debug!("dmabuf carried no explicit modifier — importing as LINEAR"); + DRM_FORMAT_MOD_LINEAR + } else { + frame.modifier + }; + + let y = &frame.planes[0]; + let c = &frame.planes[1]; + let (luma_img, luma_mem) = plane_image( + device, + ext_mem_fd, + frame.width, + frame.height, + vk::Format::R8_UNORM, + y.fd, + y.offset, + y.stride, + modifier, + ) + .context("luma plane")?; + let (chroma_img, chroma_mem) = match plane_image( + device, + ext_mem_fd, + frame.width.div_ceil(2), + frame.height.div_ceil(2), + vk::Format::R8G8_UNORM, + c.fd, + c.offset, + c.stride, + modifier, + ) + .context("chroma plane") + { + Ok(r) => r, + Err(e) => { + unsafe { + device.destroy_image(luma_img, None); + device.free_memory(luma_mem, None); + } + return Err(e); + } + }; + + let view = |image, format| { + unsafe { + device.create_image_view( + &vk::ImageViewCreateInfo::default() + .image(image) + .view_type(vk::ImageViewType::TYPE_2D) + .format(format) + .subresource_range( + vk::ImageSubresourceRange::default() + .aspect_mask(vk::ImageAspectFlags::COLOR) + .level_count(1) + .layer_count(1), + ), + None, + ) + } + .context("plane image view") + }; + let destroy_images = |views: &[vk::ImageView]| unsafe { + for v in views { + device.destroy_image_view(*v, None); + } + device.destroy_image(luma_img, None); + device.destroy_image(chroma_img, None); + device.free_memory(luma_mem, None); + device.free_memory(chroma_mem, None); + }; + let luma_view = match view(luma_img, vk::Format::R8_UNORM) { + Ok(v) => v, + Err(e) => { + destroy_images(&[]); + return Err(e); + } + }; + let chroma_view = match view(chroma_img, vk::Format::R8G8_UNORM) { + Ok(v) => v, + Err(e) => { + destroy_images(&[luma_view]); + return Err(e); + } + }; + + Ok(HwFrame { + luma_view, + chroma_view, + color: frame.color, + width: frame.width, + height: frame.height, + images: [luma_img, chroma_img], + memories: [luma_mem, chroma_mem], + views: [luma_view, chroma_view], + _guard: frame.guard, + }) +} + +/// One single-plane image over a dmabuf plane: explicit-modifier tiling with the plane's +/// (offset, pitch), external-memory dmabuf handle type, dedicated import of a dup'd fd +/// (Vulkan takes ownership of the fd it's given; the frame guard keeps owning the +/// original). +#[allow(clippy::too_many_arguments)] +fn plane_image( + device: &ash::Device, + ext_mem_fd: &ash::khr::external_memory_fd::Device, + width: u32, + height: u32, + format: vk::Format, + fd: std::os::fd::RawFd, + offset: u32, + stride: u32, + modifier: u64, +) -> Result<(vk::Image, vk::DeviceMemory)> { + let plane_layouts = [vk::SubresourceLayout { + offset: u64::from(offset), + size: 0, // must be 0 for imports (the driver derives it) + row_pitch: u64::from(stride), + array_pitch: 0, + depth_pitch: 0, + }]; + let mut modifier_info = vk::ImageDrmFormatModifierExplicitCreateInfoEXT::default() + .drm_format_modifier(modifier) + .plane_layouts(&plane_layouts); + let mut external_info = vk::ExternalMemoryImageCreateInfo::default() + .handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let image = unsafe { + device.create_image( + &vk::ImageCreateInfo::default() + .push_next(&mut modifier_info) + .push_next(&mut external_info) + .image_type(vk::ImageType::TYPE_2D) + .format(format) + .extent(vk::Extent3D { + width, + height, + depth: 1, + }) + .mip_levels(1) + .array_layers(1) + .samples(vk::SampleCountFlags::TYPE_1) + .tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT) + .usage(vk::ImageUsageFlags::SAMPLED) + .initial_layout(vk::ImageLayout::UNDEFINED), + None, + ) + } + .with_context(|| { + format!("create {width}x{height} {format:?} image (modifier {modifier:#018x})") + })?; + + let result = (|| { + // The fd's importable memory types, intersected with the image's requirement. + let mut fd_props = vk::MemoryFdPropertiesKHR::default(); + unsafe { + ext_mem_fd.get_memory_fd_properties( + vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT, + fd, + &mut fd_props, + ) + } + .context("vkGetMemoryFdPropertiesKHR")?; + let reqs = unsafe { device.get_image_memory_requirements(image) }; + let bits = reqs.memory_type_bits & fd_props.memory_type_bits; + let type_index = (0..32u32) + .find(|i| bits & (1 << i) != 0) + .context("no importable memory type for dmabuf")?; + + // Vulkan owns the fd it imports — dup so the decoder guard keeps the original. + let owned = unsafe { BorrowedFd::borrow_raw(fd) } + .try_clone_to_owned() + .context("dup dmabuf fd")?; + let mut import_info = vk::ImportMemoryFdInfoKHR::default() + .handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT) + .fd(owned.into_raw_fd()); + let mut dedicated = vk::MemoryDedicatedAllocateInfo::default().image(image); + let memory = unsafe { + device.allocate_memory( + &vk::MemoryAllocateInfo::default() + .push_next(&mut import_info) + .push_next(&mut dedicated) + .allocation_size(reqs.size) + .memory_type_index(type_index), + None, + ) + } + .context("import dmabuf memory")?; + // (On allocate_memory failure Vulkan still closed the dup'd fd — nothing leaks.) + if let Err(e) = unsafe { device.bind_image_memory(image, memory, 0) } { + unsafe { device.free_memory(memory, None) }; + return Err(e).context("bind imported memory"); + } + Ok(memory) + })(); + + match result { + Ok(memory) => Ok((image, memory)), + Err(e) => { + unsafe { device.destroy_image(image, None) }; + Err(e) + } + } +} diff --git a/crates/pf-presenter/src/lib.rs b/crates/pf-presenter/src/lib.rs index 16fac865..9ee5749a 100644 --- a/crates/pf-presenter/src/lib.rs +++ b/crates/pf-presenter/src/lib.rs @@ -3,11 +3,16 @@ //! decoded frames, captures input on the `ui_stream` state-machine contract, and reports //! the unified stats window on stdout. No UI toolkit anywhere in the dependency tree. //! -//! Phase 1 is the software path: `CpuFrame` RGBA uploads + a transfer-only letterbox -//! blit (no graphics pipeline, no shaders — those arrive with the Phase 2 dmabuf/CSC -//! pass). A hardware (dmabuf) frame slipping through demotes the decoder to software via -//! the session pump's `force_software` contract, same as the GTK presenter. +//! Two frame paths: software (`CpuFrame` RGBA staging upload) and hardware (the +//! decoder's NV12 dmabuf imported per-plane into Vulkan + the CICP-driven CSC pass — +//! `dmabuf.rs`/`csc.rs`), both composited by a letterboxed blit. Devices without the +//! import extensions, and any import/present failure streak, demote the decoder to +//! software via the session pump's `force_software` contract, same as the GTK presenter. +#[cfg(target_os = "linux")] +pub mod csc; +#[cfg(target_os = "linux")] +pub mod dmabuf; #[cfg(target_os = "linux")] pub mod input; #[cfg(target_os = "linux")] diff --git a/crates/pf-presenter/src/run.rs b/crates/pf-presenter/src/run.rs index 8fd81152..ba7f4d93 100644 --- a/crates/pf-presenter/src/run.rs +++ b/crates/pf-presenter/src/run.rs @@ -7,7 +7,7 @@ //! (Ctrl+Alt+Shift+S toggles). Logs go to stderr (the binary configures tracing so). use crate::input::Capture; -use crate::vk::Presenter; +use crate::vk::{FrameInput, Presenter}; use anyhow::{Context as _, Result}; use pf_client_core::gamepad::GamepadService; use pf_client_core::session::{self, SessionEvent, SessionParams, Stats}; @@ -68,7 +68,7 @@ where .map_err(|e| anyhow::anyhow!("vulkan instance extensions: {e}"))?; let mut presenter = Presenter::new(&window, &instance_exts).context("vulkan presenter")?; // A valid black frame immediately — the window is honest while the connect runs. - presenter.present(&window, None)?; + presenter.present(&window, FrameInput::Redraw)?; let gamepad_subsystem = sdl.gamepad().context("SDL gamepad")?; let (gamepad, mut pump) = GamepadService::pumped(gamepad_subsystem); @@ -115,9 +115,10 @@ where let mut win_disp_us: Vec = Vec::with_capacity(256); let mut win_start = Instant::now(); let mut presented = PresentedWindow::default(); - // Retained newest frame for expose/resize re-blits (the presenter keeps the GPU - // image; this keeps the pending upload if one arrives while minimized). + // Hardware-path health: a failure streak (or a device with no import support at all) + // demotes the decoder to software via the shared flag — once per session. let mut dmabuf_demoted = false; + let mut hw_fails = 0u32; let outcome = 'main: loop { // --- SDL events (input, window, gamepads) --------------------------------------- @@ -154,10 +155,10 @@ where } WindowEvent::PixelSizeChanged(..) | WindowEvent::Resized(..) => { presenter.recreate_swapchain(&window)?; - presenter.present(&window, None)?; + presenter.present(&window, FrameInput::Redraw)?; } WindowEvent::Exposed => { - presenter.present(&window, None)?; + presenter.present(&window, FrameInput::Redraw)?; } _ => {} }, @@ -325,35 +326,67 @@ where newest = Some(f); } if let Some(f) = newest { - match f.image { + let DecodedFrame { + pts_ns, + decoded_ns, + image, + } = f; + let did_present = match image { DecodedImage::Cpu(c) => { hdr = c.color.is_pq(); - presenter.present(&window, Some(&c))?; - let displayed_ns = session::now_ns(); - if opts.json_status && !ready_announced { - ready_announced = true; - println!("{{\"ready\":true}}"); + presenter.present(&window, FrameInput::Cpu(&c))? + } + DecodedImage::Dmabuf(d) if presenter.supports_dmabuf() && !dmabuf_demoted => { + hdr = d.color.is_pq(); + match presenter.present(&window, FrameInput::Dmabuf(d)) { + Ok(p) => { + hw_fails = 0; + p + } + // Import/CSC failure is survivable (the stream continues on the + // next frame) — but a streak means this box can't do the hw path: + // demote the decoder to software, same contract as the GTK + // presenter's GL-converter failures. + Err(e) => { + hw_fails += 1; + tracing::warn!(error = %format!("{e:#}"), fails = hw_fails, + "hardware present failed"); + if hw_fails >= 3 && !dmabuf_demoted { + dmabuf_demoted = true; + tracing::warn!("demoting the decoder to software"); + force_software.store(true, Ordering::Relaxed); + } + false + } } - // The `displayed` stamp (same clamp rules as the pump's windows). - let e2e = (displayed_ns as i128 + clock_offset_ns as i128 - f.pts_ns as i128) - .max(0) as u64; - if e2e > 0 && e2e < 10_000_000_000 { - win_e2e_us.push(e2e / 1000); - } - win_disp_us.push(displayed_ns.saturating_sub(f.decoded_ns) / 1000); } DecodedImage::Dmabuf(_) => { - // Phase 1 has no hardware present path — demote the decoder to - // software through the same contract the GTK presenter uses. Once. + // No import extensions on this device (or already demoted) — the + // pump rebuilds the decoder as software; frames flow again shortly. if !dmabuf_demoted { dmabuf_demoted = true; tracing::warn!( - "hardware (dmabuf) frame reached the phase-1 presenter — \ - demoting the decoder to software" + "no dmabuf import support on this device — demoting the \ + decoder to software" ); force_software.store(true, Ordering::Relaxed); } + false } + }; + if did_present { + let displayed_ns = session::now_ns(); + if opts.json_status && !ready_announced { + ready_announced = true; + println!("{{\"ready\":true}}"); + } + // The `displayed` stamp (same clamp rules as the pump's windows). + let e2e = + (displayed_ns as i128 + clock_offset_ns as i128 - pts_ns as i128).max(0) as u64; + if e2e > 0 && e2e < 10_000_000_000 { + win_e2e_us.push(e2e / 1000); + } + win_disp_us.push(displayed_ns.saturating_sub(decoded_ns) / 1000); } } diff --git a/crates/pf-presenter/src/vk.rs b/crates/pf-presenter/src/vk.rs index 94166eba..1e1b8f27 100644 --- a/crates/pf-presenter/src/vk.rs +++ b/crates/pf-presenter/src/vk.rs @@ -1,27 +1,47 @@ -//! The transfer-only Vulkan presenter: swapchain + staging upload + letterboxed blit. +//! The Vulkan presenter: swapchain + two frame paths into one device-local RGBA video +//! image, then a letterboxed `vkCmdBlitImage` composite. //! -//! Phase 1 deliberately has NO graphics pipeline: software-decoded RGBA uploads to a -//! device-local image (`copy_buffer_to_image`, the row stride handled by -//! `buffer_row_length`), and each present clears the swapchain image and -//! `vkCmdBlitImage`s the video into the Contain-fit letterbox rect (linear filter, -//! format conversion by the blit). No render pass, no framebuffers, no image views, no -//! SPIR-V toolchain — all of that arrives with the Phase 2 dmabuf/CSC pass, which is the -//! first thing that actually needs a shader. +//! * **Software** (`FrameInput::Cpu`): staging upload + `copy_buffer_to_image` (row +//! stride via `buffer_row_length`) — transfer-only, runs on every GPU. +//! * **Hardware** (`FrameInput::Dmabuf`): the decoder's NV12 dmabuf imported per-plane +//! (`dmabuf.rs`) and converted by the CSC render pass (`csc.rs`) — zero-copy, gated on +//! the four import extensions at device creation; boxes without them (NVIDIA +//! proprietary by design) report `supports_dmabuf() == false` and the caller keeps the +//! decoder on software. //! //! Pacing: one frame in flight (the submit fence is waited before each record), FIFO by //! default (`PUNKTFUNK_PRESENT_MODE=mailbox|immediate` if available). Present is -//! arrival-paced by the caller: `present(Some(frame))` on each decoded frame, -//! `present(None)` re-blits the retained video image (expose/resize redraws). +//! arrival-paced by the caller: a frame input on each decoded frame, +//! `FrameInput::Redraw` re-blits the retained video image (expose/resize redraws). +use crate::csc::{csc_rows, CscPass}; +use crate::dmabuf::{self, HwFrame}; use anyhow::{anyhow, bail, Context as _, Result}; use ash::vk; -use pf_client_core::video::CpuFrame; +use pf_client_core::video::{CpuFrame, DmabufFrame}; use std::ffi::CString; +/// One presenter iteration's video input. +pub enum FrameInput<'a> { + /// No new frame — re-composite the retained video image (expose/resize). + Redraw, + Cpu(&'a CpuFrame), + Dmabuf(DmabufFrame), +} + +/// The dmabuf/CSC machinery, present only when the device carries the import extensions. +struct HwCtx { + ext_mem_fd: ash::khr::external_memory_fd::Device, + csc: CscPass, +} + /// The one video image (device-local RGBA the size of the decoded stream) + its staging. +/// `view`/`framebuffer` exist only on hw-capable devices (the CSC pass renders into it). struct VideoImage { image: vk::Image, memory: vk::DeviceMemory, + view: vk::ImageView, + framebuffer: vk::Framebuffer, width: u32, height: u32, } @@ -44,6 +64,12 @@ pub struct Presenter { device: ash::Device, swap_d: ash::khr::swapchain::Device, queue: vk::Queue, + qfi: u32, + /// Dmabuf import + CSC — `None` when the device lacks the import extensions. + hw: Option, + /// The submitted hw frame (plane images + decoder-surface guard): its GPU reads end + /// with the in-flight fence, so it's destroyed right after the next fence wait. + retired_hw: Option, format: vk::SurfaceFormatKHR, present_mode: vk::PresentModeKHR, swapchain: vk::SwapchainKHR, @@ -107,9 +133,24 @@ impl Presenter { let queue_info = [vk::DeviceQueueCreateInfo::default() .queue_family_index(qfi) .queue_priorities(&[1.0])]; - // Phase 2 (dmabuf import) adds VK_EXT_external_memory_dma_buf + - // VK_KHR_external_memory_fd + VK_EXT_image_drm_format_modifier here. - let dev_exts = [ash::khr::swapchain::NAME.as_ptr()]; + // The dmabuf import set is optional: enabled when the device offers all four, + // else the presenter is software-only (`supports_dmabuf() == false`). + let available = unsafe { instance.enumerate_device_extension_properties(pdev) }?; + let has = |name: &std::ffi::CStr| { + available + .iter() + .any(|e| e.extension_name_as_c_str() == Ok(name)) + }; + let hw_capable = dmabuf::DEVICE_EXTENSIONS.iter().all(|n| has(n)); + let mut dev_exts = vec![ash::khr::swapchain::NAME.as_ptr()]; + if hw_capable { + dev_exts.extend(dmabuf::DEVICE_EXTENSIONS.iter().map(|n| n.as_ptr())); + } else { + tracing::info!( + "device lacks the dmabuf import extensions — hardware frames unavailable \ + (software decode only)" + ); + } let device = unsafe { instance.create_device( pdev, @@ -122,6 +163,14 @@ impl Presenter { .context("vkCreateDevice")?; let swap_d = ash::khr::swapchain::Device::new(&instance, &device); let queue = unsafe { device.get_device_queue(qfi, 0) }; + let hw = if hw_capable { + Some(HwCtx { + ext_mem_fd: ash::khr::external_memory_fd::Device::new(&instance, &device), + csc: CscPass::new(&device)?, + }) + } else { + None + }; let format = pick_format(&surface_i, pdev, surface)?; let present_mode = pick_present_mode(&surface_i, pdev, surface)?; @@ -162,6 +211,9 @@ impl Presenter { device, swap_d, queue, + qfi, + hw, + retired_hw: None, format, present_mode, swapchain: vk::SwapchainKHR::null(), @@ -252,15 +304,38 @@ impl Presenter { Ok(()) } - /// Present one frame: upload `frame` if given (else re-blit the retained video - /// image), clear, letterbox-blit, present. Returns false when the swapchain was out - /// of date — the caller recreates (with current window state) and may retry. - pub fn present(&mut self, window: &sdl3::video::Window, frame: Option<&CpuFrame>) -> Result { + /// Whether the hardware (dmabuf) path exists on this device — callers keep the + /// decoder on software when it doesn't. + pub fn supports_dmabuf(&self) -> bool { + self.hw.is_some() + } + + /// Present one frame: route `input` into the video image (staging upload or dmabuf + /// import + CSC pass; `Redraw` re-blits what's retained), clear, letterbox-blit, + /// present. Returns false when the swapchain was out of date — the caller recreates + /// (with current window state) and may retry. + pub fn present(&mut self, window: &sdl3::video::Window, input: FrameInput) -> Result { if self.extent.width == 0 || self.extent.height == 0 { return Ok(true); // minimized — nothing to do } - // One frame in flight: the fence covers the command buffer AND the staging - // buffer, so waiting here makes both safe to reuse. + // A dmabuf frame imports before anything touches the queue: an import the driver + // rejects must fail out here, before this present consumed the acquire semaphore. + let mut hw_frame: Option = None; + let cpu_frame = match input { + FrameInput::Redraw => None, + FrameInput::Cpu(f) => Some(f), + FrameInput::Dmabuf(d) => { + let hw = self + .hw + .as_ref() + .context("hardware frame without dmabuf support")?; + hw_frame = Some(dmabuf::import(&self.device, &hw.ext_mem_fd, d)?); + None + } + }; + + // One frame in flight: the fence covers the command buffer, the staging buffer + // AND the previously submitted hw frame — waiting makes all three reusable. unsafe { if self.submitted { self.device.wait_for_fences(&[self.fence], true, u64::MAX)?; @@ -268,10 +343,26 @@ impl Presenter { } self.device.reset_fences(&[self.fence])?; } + if let Some(old) = self.retired_hw.take() { + old.destroy(&self.device); + } - if let Some(f) = frame { + if let Some(f) = cpu_frame { self.stage_frame(f)?; } + if let Some(f) = &hw_frame { + if self + .video + .as_ref() + .is_none_or(|v| v.width != f.width || v.height != f.height) + { + self.rebuild_video_image(f.width, f.height)?; + tracing::info!(width = f.width, height = f.height, "video image (re)built"); + } + // Safe while nothing in flight references the set — the fence wait above. + let hw = self.hw.as_ref().unwrap(); + hw.csc.bind_planes(&self.device, f.luma_view, f.chroma_view); + } let (index, _suboptimal) = match unsafe { self.swap_d.acquire_next_image( @@ -283,6 +374,10 @@ impl Presenter { } { Ok(r) => r, Err(vk::Result::ERROR_OUT_OF_DATE_KHR) => { + // Never submitted — the import (if any) dies here, GPU never saw it. + if let Some(f) = hw_frame { + f.destroy(&self.device); + } self.recreate_swapchain(window)?; return Ok(false); } @@ -297,8 +392,76 @@ impl Presenter { .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), )?; + // Hardware frame: acquire the foreign planes, then the CSC pass renders + // NV12→RGBA into the video image (render pass ends it in TRANSFER_SRC for + // the blit below). + if let (Some(f), Some(hw), Some(v)) = (&hw_frame, &self.hw, &self.video) { + for view_image in [f.luma_image(), f.chroma_image()] { + foreign_acquire_barrier(&self.device, self.cmd_buf, view_image, self.qfi); + } + let extent = vk::Extent2D { + width: v.width, + height: v.height, + }; + self.device.cmd_begin_render_pass( + self.cmd_buf, + &vk::RenderPassBeginInfo::default() + .render_pass(hw.csc.render_pass) + .framebuffer(v.framebuffer) + .render_area(vk::Rect2D { + offset: vk::Offset2D { x: 0, y: 0 }, + extent, + }), + vk::SubpassContents::INLINE, + ); + self.device.cmd_bind_pipeline( + self.cmd_buf, + vk::PipelineBindPoint::GRAPHICS, + hw.csc.pipeline, + ); + self.device.cmd_set_viewport( + self.cmd_buf, + 0, + &[vk::Viewport { + x: 0.0, + y: 0.0, + width: extent.width as f32, + height: extent.height as f32, + min_depth: 0.0, + max_depth: 1.0, + }], + ); + self.device.cmd_set_scissor( + self.cmd_buf, + 0, + &[vk::Rect2D { + offset: vk::Offset2D { x: 0, y: 0 }, + extent, + }], + ); + self.device.cmd_bind_descriptor_sets( + self.cmd_buf, + vk::PipelineBindPoint::GRAPHICS, + hw.csc.pipeline_layout, + 0, + &[hw.csc.desc_set], + &[], + ); + let rows = csc_rows(f.color); + let bytes = std::slice::from_raw_parts(rows.as_ptr().cast::(), 48); + self.device.cmd_push_constants( + self.cmd_buf, + hw.csc.pipeline_layout, + vk::ShaderStageFlags::FRAGMENT, + 0, + bytes, + ); + self.device.cmd_draw(self.cmd_buf, 3, 1, 0, 0); + self.device.cmd_end_render_pass(self.cmd_buf); + } + // New frame: staging → video image (stride carried by buffer_row_length). - if let (Some(f), Some(v), Some(s)) = (frame, &self.video, &self.staging) { + if let (Some(f), Some(v), Some(s)) = (cpu_frame, &self.video, &self.staging) { barrier( &self.device, self.cmd_buf, @@ -396,6 +559,9 @@ impl Presenter { self.fence, )?; self.submitted = true; + // The hw frame is on the GPU now — park it until the fence proves the reads + // done (destroyed at the next present's fence wait, or in Drop). + self.retired_hw = hw_frame.take(); let swapchains = [self.swapchain]; let indices = [index]; @@ -449,10 +615,17 @@ impl Presenter { self.submitted = false; if let Some(v) = self.video.take() { unsafe { + if v.framebuffer != vk::Framebuffer::null() { + self.device.destroy_framebuffer(v.framebuffer, None); + } + if v.view != vk::ImageView::null() { + self.device.destroy_image_view(v.view, None); + } self.device.destroy_image(v.image, None); self.device.free_memory(v.memory, None); } } + // COLOR_ATTACHMENT is the CSC pass's render target; harmless where hw is absent. let image = unsafe { self.device.create_image( &vk::ImageCreateInfo::default() @@ -467,7 +640,11 @@ impl Presenter { .array_layers(1) .samples(vk::SampleCountFlags::TYPE_1) .tiling(vk::ImageTiling::OPTIMAL) - .usage(vk::ImageUsageFlags::TRANSFER_DST | vk::ImageUsageFlags::TRANSFER_SRC) + .usage( + vk::ImageUsageFlags::TRANSFER_DST + | vk::ImageUsageFlags::TRANSFER_SRC + | vk::ImageUsageFlags::COLOR_ATTACHMENT, + ) .initial_layout(vk::ImageLayout::UNDEFINED), None, ) @@ -475,9 +652,39 @@ impl Presenter { let reqs = unsafe { self.device.get_image_memory_requirements(image) }; let memory = self.allocate(reqs, vk::MemoryPropertyFlags::DEVICE_LOCAL)?; unsafe { self.device.bind_image_memory(image, memory, 0) }?; + // The CSC pass renders into it — view + framebuffer, hw-capable devices only. + let (view, framebuffer) = if let Some(hw) = &self.hw { + let view = unsafe { + self.device.create_image_view( + &vk::ImageViewCreateInfo::default() + .image(image) + .view_type(vk::ImageViewType::TYPE_2D) + .format(vk::Format::R8G8B8A8_UNORM) + .subresource_range(subresource_range()), + None, + ) + }?; + let attachments = [view]; + let framebuffer = unsafe { + self.device.create_framebuffer( + &vk::FramebufferCreateInfo::default() + .render_pass(hw.csc.render_pass) + .attachments(&attachments) + .width(width) + .height(height) + .layers(1), + None, + ) + }?; + (view, framebuffer) + } else { + (vk::ImageView::null(), vk::Framebuffer::null()) + }; self.video = Some(VideoImage { image, memory, + view, + framebuffer, width, height, }); @@ -551,15 +758,27 @@ impl Drop for Presenter { fn drop(&mut self) { unsafe { self.device.device_wait_idle().ok(); + if let Some(f) = self.retired_hw.take() { + f.destroy(&self.device); // idle above — the GPU reads are done + } if let Some(s) = self.staging.take() { self.device.unmap_memory(s.memory); self.device.destroy_buffer(s.buffer, None); self.device.free_memory(s.memory, None); } if let Some(v) = self.video.take() { + if v.framebuffer != vk::Framebuffer::null() { + self.device.destroy_framebuffer(v.framebuffer, None); + } + if v.view != vk::ImageView::null() { + self.device.destroy_image_view(v.view, None); + } self.device.destroy_image(v.image, None); self.device.free_memory(v.memory, None); } + if let Some(hw) = self.hw.take() { + hw.csc.destroy(&self.device); + } for s in self.render_sems.drain(..) { self.device.destroy_semaphore(s, None); } @@ -680,6 +899,37 @@ fn subresource_range() -> vk::ImageSubresourceRange { .layer_count(1) } +/// Acquire a dmabuf plane image from its foreign owner (the VAAPI decoder): queue-family +/// transfer FOREIGN → ours, UNDEFINED → SHADER_READ_ONLY (content is preserved across +/// the transfer regardless of the UNDEFINED old-layout, per the external-memory rules). +fn foreign_acquire_barrier( + device: &ash::Device, + cmd: vk::CommandBuffer, + image: vk::Image, + qfi: u32, +) { + let b = vk::ImageMemoryBarrier::default() + .src_access_mask(vk::AccessFlags::empty()) + .dst_access_mask(vk::AccessFlags::SHADER_READ) + .old_layout(vk::ImageLayout::UNDEFINED) + .new_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) + .src_queue_family_index(vk::QUEUE_FAMILY_FOREIGN_EXT) + .dst_queue_family_index(qfi) + .image(image) + .subresource_range(subresource_range()); + unsafe { + device.cmd_pipeline_barrier( + cmd, + vk::PipelineStageFlags::TOP_OF_PIPE, + vk::PipelineStageFlags::FRAGMENT_SHADER, + vk::DependencyFlags::empty(), + &[], + &[], + &[b], + ); + } +} + /// A full-subresource layout transition with the conservative ALL_COMMANDS/TRANSFER /// scopes this transfer-only pipeline needs (per-frame granularity, not per-stage). fn barrier(