feat(presenter): VAAPI dmabuf → Vulkan zero-copy import + CSC pass (phase 2)
The decoder's NV12 dmabuf imports per-plane (R8 + GR88, explicit DRM format modifier, dedicated dup'd-fd import, FOREIGN→graphics acquire) and a fullscreen-triangle render pass converts it into the presenter's video image with the CICP-driven coefficients ported from video_gl.rs (same tests, plus a rows-vs-matrix agreement check). SPIR-V is committed (shaders/build.sh regenerates) so builds and CI need no toolchain. The import extension set is probed at device creation; unsupported boxes and 3-failure streaks demote the decoder to software via the existing force_software contract. The session binary now honors the Settings decoder preference instead of forcing software. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,398 @@
|
||||
//! The NV12→RGBA color-space-conversion pass: coefficient rows from the stream's CICP
|
||||
//! signaling (the GL presenter's `yuv_to_rgb`, folded into row form for the shader's
|
||||
//! push constants) and the graphics pipeline that renders the two imported planes into
|
||||
//! the presenter's video image.
|
||||
//!
|
||||
//! Deliberately NOT `VK_KHR_sampler_ycbcr_conversion`: the ext can't express the PQ path
|
||||
//! coming with the HDR phase and its range handling varies by driver — these are the
|
||||
//! proven, unit-tested coefficients (plan §5.2).
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use ash::vk;
|
||||
use pf_client_core::video::ColorDesc;
|
||||
|
||||
/// The push-constant block: three vec4 rows, `rgb[i] = dot(r[i].xyz, yuv) + r[i].w`.
|
||||
/// One layout for BT.601/709/2020 × full/limited; PQ transfer joins in the HDR phase.
|
||||
pub fn csc_rows(desc: ColorDesc) -> [[f32; 4]; 3] {
|
||||
// BT.601 (5/6), BT.2020 (9/10); everything else — incl. unspecified — is the host's
|
||||
// BT.709 SDR default (mirrors the software path's swscale coefficient choice).
|
||||
let (kr, kb) = match desc.matrix {
|
||||
5 | 6 => (0.299, 0.114),
|
||||
9 | 10 => (0.2627, 0.0593),
|
||||
_ => (0.2126, 0.0722),
|
||||
};
|
||||
let kg = 1.0 - kr - kb;
|
||||
let (sy, oy, sc) = if desc.full_range {
|
||||
(1.0f64, 0.0f64, 1.0f64)
|
||||
} else {
|
||||
(255.0 / 219.0, -16.0 / 255.0, 255.0 / 224.0)
|
||||
};
|
||||
// rgb = M * (yuv + off) = M*yuv + M*off — rows of M with the offset dot folded into w.
|
||||
let off = [oy, -0.5, -0.5];
|
||||
let m = [
|
||||
[sy, 0.0, 2.0 * (1.0 - kr) * sc],
|
||||
[
|
||||
sy,
|
||||
-2.0 * (1.0 - kb) * kb / kg * sc,
|
||||
-2.0 * (1.0 - kr) * kr / kg * sc,
|
||||
],
|
||||
[sy, 2.0 * (1.0 - kb) * sc, 0.0],
|
||||
];
|
||||
core::array::from_fn(|r| {
|
||||
let w: f64 = (0..3).map(|c| m[r][c] * off[c]).sum();
|
||||
[m[r][0] as f32, m[r][1] as f32, m[r][2] as f32, w as f32]
|
||||
})
|
||||
}
|
||||
|
||||
/// The pass objects (everything except the per-video-size framebuffer, which lives with
|
||||
/// the video image). Destroyed explicitly via [`CscPass::destroy`] from the presenter's
|
||||
/// `Drop` — no device handle is stored here.
|
||||
pub struct CscPass {
|
||||
pub render_pass: vk::RenderPass,
|
||||
pub set_layout: vk::DescriptorSetLayout,
|
||||
pub pipeline_layout: vk::PipelineLayout,
|
||||
pub pipeline: vk::Pipeline,
|
||||
pub desc_pool: vk::DescriptorPool,
|
||||
pub desc_set: vk::DescriptorSet,
|
||||
pub sampler: vk::Sampler,
|
||||
}
|
||||
|
||||
impl CscPass {
|
||||
pub fn new(device: &ash::Device) -> Result<CscPass> {
|
||||
// One color attachment: the presenter's R8G8B8A8 video image. Content is fully
|
||||
// overwritten (DONT_CARE load), and the pass ends in TRANSFER_SRC so the
|
||||
// existing letterbox blit consumes it with no extra barrier.
|
||||
let attachment = [vk::AttachmentDescription::default()
|
||||
.format(vk::Format::R8G8B8A8_UNORM)
|
||||
.samples(vk::SampleCountFlags::TYPE_1)
|
||||
.load_op(vk::AttachmentLoadOp::DONT_CARE)
|
||||
.store_op(vk::AttachmentStoreOp::STORE)
|
||||
.initial_layout(vk::ImageLayout::UNDEFINED)
|
||||
.final_layout(vk::ImageLayout::TRANSFER_SRC_OPTIMAL)];
|
||||
let color_ref = [vk::AttachmentReference::default()
|
||||
.attachment(0)
|
||||
.layout(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL)];
|
||||
let subpass = [vk::SubpassDescription::default()
|
||||
.pipeline_bind_point(vk::PipelineBindPoint::GRAPHICS)
|
||||
.color_attachments(&color_ref)];
|
||||
// Conservative scopes, matching the presenter's per-frame barrier granularity.
|
||||
let deps = [
|
||||
vk::SubpassDependency::default()
|
||||
.src_subpass(vk::SUBPASS_EXTERNAL)
|
||||
.dst_subpass(0)
|
||||
.src_stage_mask(vk::PipelineStageFlags::ALL_COMMANDS)
|
||||
.src_access_mask(vk::AccessFlags::MEMORY_WRITE)
|
||||
.dst_stage_mask(vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT)
|
||||
.dst_access_mask(vk::AccessFlags::COLOR_ATTACHMENT_WRITE),
|
||||
vk::SubpassDependency::default()
|
||||
.src_subpass(0)
|
||||
.dst_subpass(vk::SUBPASS_EXTERNAL)
|
||||
.src_stage_mask(vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT)
|
||||
.src_access_mask(vk::AccessFlags::COLOR_ATTACHMENT_WRITE)
|
||||
.dst_stage_mask(vk::PipelineStageFlags::TRANSFER)
|
||||
.dst_access_mask(vk::AccessFlags::TRANSFER_READ),
|
||||
];
|
||||
let render_pass = unsafe {
|
||||
device.create_render_pass(
|
||||
&vk::RenderPassCreateInfo::default()
|
||||
.attachments(&attachment)
|
||||
.subpasses(&subpass)
|
||||
.dependencies(&deps),
|
||||
None,
|
||||
)
|
||||
}
|
||||
.context("CSC render pass")?;
|
||||
|
||||
let sampler = unsafe {
|
||||
device.create_sampler(
|
||||
&vk::SamplerCreateInfo::default()
|
||||
.mag_filter(vk::Filter::LINEAR)
|
||||
.min_filter(vk::Filter::LINEAR)
|
||||
.address_mode_u(vk::SamplerAddressMode::CLAMP_TO_EDGE)
|
||||
.address_mode_v(vk::SamplerAddressMode::CLAMP_TO_EDGE)
|
||||
.address_mode_w(vk::SamplerAddressMode::CLAMP_TO_EDGE),
|
||||
None,
|
||||
)
|
||||
}?;
|
||||
|
||||
let samplers = [sampler];
|
||||
let bindings = [
|
||||
vk::DescriptorSetLayoutBinding::default()
|
||||
.binding(0)
|
||||
.descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER)
|
||||
.descriptor_count(1)
|
||||
.stage_flags(vk::ShaderStageFlags::FRAGMENT)
|
||||
.immutable_samplers(&samplers),
|
||||
vk::DescriptorSetLayoutBinding::default()
|
||||
.binding(1)
|
||||
.descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER)
|
||||
.descriptor_count(1)
|
||||
.stage_flags(vk::ShaderStageFlags::FRAGMENT)
|
||||
.immutable_samplers(&samplers),
|
||||
];
|
||||
let set_layout = unsafe {
|
||||
device.create_descriptor_set_layout(
|
||||
&vk::DescriptorSetLayoutCreateInfo::default().bindings(&bindings),
|
||||
None,
|
||||
)
|
||||
}?;
|
||||
let set_layouts = [set_layout];
|
||||
let push = [vk::PushConstantRange::default()
|
||||
.stage_flags(vk::ShaderStageFlags::FRAGMENT)
|
||||
.size(48)]; // three vec4 rows
|
||||
let pipeline_layout = unsafe {
|
||||
device.create_pipeline_layout(
|
||||
&vk::PipelineLayoutCreateInfo::default()
|
||||
.set_layouts(&set_layouts)
|
||||
.push_constant_ranges(&push),
|
||||
None,
|
||||
)
|
||||
}?;
|
||||
|
||||
let pool_sizes = [vk::DescriptorPoolSize::default()
|
||||
.ty(vk::DescriptorType::COMBINED_IMAGE_SAMPLER)
|
||||
.descriptor_count(2)];
|
||||
let desc_pool = unsafe {
|
||||
device.create_descriptor_pool(
|
||||
&vk::DescriptorPoolCreateInfo::default()
|
||||
.max_sets(1)
|
||||
.pool_sizes(&pool_sizes),
|
||||
None,
|
||||
)
|
||||
}?;
|
||||
let desc_set = unsafe {
|
||||
device.allocate_descriptor_sets(
|
||||
&vk::DescriptorSetAllocateInfo::default()
|
||||
.descriptor_pool(desc_pool)
|
||||
.set_layouts(&set_layouts),
|
||||
)
|
||||
}?[0];
|
||||
|
||||
let pipeline = build_pipeline(device, render_pass, pipeline_layout)?;
|
||||
|
||||
Ok(CscPass {
|
||||
render_pass,
|
||||
set_layout,
|
||||
pipeline_layout,
|
||||
pipeline,
|
||||
desc_pool,
|
||||
desc_set,
|
||||
sampler,
|
||||
})
|
||||
}
|
||||
|
||||
/// Point the descriptor set at this frame's plane views. Only safe while no
|
||||
/// submitted command buffer references the set — the presenter's single in-flight
|
||||
/// fence is waited before every record, which covers it.
|
||||
pub fn bind_planes(&self, device: &ash::Device, luma: vk::ImageView, chroma: vk::ImageView) {
|
||||
let infos = [luma, chroma].map(|view| {
|
||||
[vk::DescriptorImageInfo::default()
|
||||
.image_view(view)
|
||||
.image_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL)]
|
||||
});
|
||||
let writes = [
|
||||
vk::WriteDescriptorSet::default()
|
||||
.dst_set(self.desc_set)
|
||||
.dst_binding(0)
|
||||
.descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER)
|
||||
.image_info(&infos[0]),
|
||||
vk::WriteDescriptorSet::default()
|
||||
.dst_set(self.desc_set)
|
||||
.dst_binding(1)
|
||||
.descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER)
|
||||
.image_info(&infos[1]),
|
||||
];
|
||||
unsafe { device.update_descriptor_sets(&writes, &[]) };
|
||||
}
|
||||
|
||||
pub fn destroy(&self, device: &ash::Device) {
|
||||
unsafe {
|
||||
device.destroy_pipeline(self.pipeline, None);
|
||||
device.destroy_pipeline_layout(self.pipeline_layout, None);
|
||||
device.destroy_descriptor_pool(self.desc_pool, None);
|
||||
device.destroy_descriptor_set_layout(self.set_layout, None);
|
||||
device.destroy_sampler(self.sampler, None);
|
||||
device.destroy_render_pass(self.render_pass, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_pipeline(
|
||||
device: &ash::Device,
|
||||
render_pass: vk::RenderPass,
|
||||
layout: vk::PipelineLayout,
|
||||
) -> Result<vk::Pipeline> {
|
||||
// Committed SPIR-V (shaders/build.sh) — include_bytes! alignment is unspecified, so
|
||||
// read_spv copies into aligned Vec<u32>s.
|
||||
let vert = ash::util::read_spv(&mut std::io::Cursor::new(
|
||||
&include_bytes!("../shaders/fullscreen.vert.spv")[..],
|
||||
))?;
|
||||
let frag = ash::util::read_spv(&mut std::io::Cursor::new(
|
||||
&include_bytes!("../shaders/nv12_csc.frag.spv")[..],
|
||||
))?;
|
||||
let vert_mod = unsafe {
|
||||
device.create_shader_module(&vk::ShaderModuleCreateInfo::default().code(&vert), None)
|
||||
}?;
|
||||
let frag_mod = unsafe {
|
||||
device.create_shader_module(&vk::ShaderModuleCreateInfo::default().code(&frag), None)
|
||||
};
|
||||
let frag_mod = match frag_mod {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
unsafe { device.destroy_shader_module(vert_mod, None) };
|
||||
return Err(e).context("fragment shader module");
|
||||
}
|
||||
};
|
||||
|
||||
let entry = c"main";
|
||||
let stages = [
|
||||
vk::PipelineShaderStageCreateInfo::default()
|
||||
.stage(vk::ShaderStageFlags::VERTEX)
|
||||
.module(vert_mod)
|
||||
.name(entry),
|
||||
vk::PipelineShaderStageCreateInfo::default()
|
||||
.stage(vk::ShaderStageFlags::FRAGMENT)
|
||||
.module(frag_mod)
|
||||
.name(entry),
|
||||
];
|
||||
let vertex_input = vk::PipelineVertexInputStateCreateInfo::default(); // bufferless
|
||||
let assembly = vk::PipelineInputAssemblyStateCreateInfo::default()
|
||||
.topology(vk::PrimitiveTopology::TRIANGLE_LIST);
|
||||
// Dynamic viewport/scissor: the video size changes with the stream mode, the
|
||||
// pipeline must not bake one in.
|
||||
let viewport = vk::PipelineViewportStateCreateInfo::default()
|
||||
.viewport_count(1)
|
||||
.scissor_count(1);
|
||||
let dynamic = [vk::DynamicState::VIEWPORT, vk::DynamicState::SCISSOR];
|
||||
let dynamic_state = vk::PipelineDynamicStateCreateInfo::default().dynamic_states(&dynamic);
|
||||
let raster = vk::PipelineRasterizationStateCreateInfo::default()
|
||||
.polygon_mode(vk::PolygonMode::FILL)
|
||||
.cull_mode(vk::CullModeFlags::NONE)
|
||||
.line_width(1.0);
|
||||
let multisample = vk::PipelineMultisampleStateCreateInfo::default()
|
||||
.rasterization_samples(vk::SampleCountFlags::TYPE_1);
|
||||
let blend_attachment = [vk::PipelineColorBlendAttachmentState::default()
|
||||
.color_write_mask(vk::ColorComponentFlags::RGBA)];
|
||||
let blend = vk::PipelineColorBlendStateCreateInfo::default().attachments(&blend_attachment);
|
||||
|
||||
let info = vk::GraphicsPipelineCreateInfo::default()
|
||||
.stages(&stages)
|
||||
.vertex_input_state(&vertex_input)
|
||||
.input_assembly_state(&assembly)
|
||||
.viewport_state(&viewport)
|
||||
.rasterization_state(&raster)
|
||||
.multisample_state(&multisample)
|
||||
.color_blend_state(&blend)
|
||||
.dynamic_state(&dynamic_state)
|
||||
.layout(layout)
|
||||
.render_pass(render_pass);
|
||||
let pipeline = unsafe {
|
||||
device.create_graphics_pipelines(vk::PipelineCache::null(), &[info], None)
|
||||
}
|
||||
.map_err(|(_, e)| e)
|
||||
.context("CSC pipeline");
|
||||
unsafe {
|
||||
device.destroy_shader_module(vert_mod, None);
|
||||
device.destroy_shader_module(frag_mod, None);
|
||||
}
|
||||
Ok(pipeline?[0])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn desc(matrix: u8, full_range: bool) -> ColorDesc {
|
||||
ColorDesc {
|
||||
primaries: 1,
|
||||
transfer: 1,
|
||||
matrix,
|
||||
full_range,
|
||||
}
|
||||
}
|
||||
|
||||
fn apply(rows: &[[f32; 4]; 3], yuv: [f32; 3]) -> [f32; 3] {
|
||||
core::array::from_fn(|r| {
|
||||
rows[r][0] * yuv[0] + rows[r][1] * yuv[1] + rows[r][2] * yuv[2] + rows[r][3]
|
||||
})
|
||||
}
|
||||
|
||||
/// Reference white (Y=235, U=V=128 limited) → RGB 1.0; reference black (Y=16) → 0.0
|
||||
/// — the GL presenter's test, in row form.
|
||||
#[test]
|
||||
fn bt709_limited_white_black() {
|
||||
let rows = csc_rows(desc(1, false));
|
||||
let white = apply(&rows, [235.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0]);
|
||||
let black = apply(&rows, [16.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0]);
|
||||
for (w, b) in white.iter().zip(black) {
|
||||
assert!((w - 1.0).abs() < 0.005, "white {white:?}");
|
||||
assert!(b.abs() < 0.005, "black {black:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Full-range identity points + the 601-vs-709 red excursion (guards the
|
||||
/// matrix-code dispatch), same as the GL presenter's test.
|
||||
#[test]
|
||||
fn full_range_and_red_excursion() {
|
||||
let rows = csc_rows(desc(5, true));
|
||||
let white = apply(&rows, [1.0, 0.5, 0.5]);
|
||||
assert!(white.iter().all(|v| (v - 1.0).abs() < 1e-5), "{white:?}");
|
||||
let red = apply(&rows, [0.0, 0.5, 1.0]);
|
||||
assert!((red[0] - 2.0 * (1.0 - 0.299) * 0.5).abs() < 1e-4, "{red:?}");
|
||||
let rows709 = csc_rows(desc(1, true));
|
||||
let red709 = apply(&rows709, [0.0, 0.5, 1.0]);
|
||||
assert!(
|
||||
(red709[0] - 2.0 * (1.0 - 0.2126) * 0.5).abs() < 1e-4,
|
||||
"{red709:?}"
|
||||
);
|
||||
assert!((red[0] - red709[0]).abs() > 0.05);
|
||||
}
|
||||
|
||||
/// The row form must agree with the GL presenter's column-major `yuv_to_rgb` on a
|
||||
/// grid of inputs — same math, different packing.
|
||||
#[test]
|
||||
fn rows_match_the_gl_matrix_form() {
|
||||
for (matrix, full) in [(1u8, false), (1, true), (5, false), (9, false), (9, true)] {
|
||||
let d = desc(matrix, full);
|
||||
let rows = csc_rows(d);
|
||||
// Reimplementation of video_gl::yuv_to_rgb's application for comparison.
|
||||
let (kr, kb) = match matrix {
|
||||
5 | 6 => (0.299f32, 0.114f32),
|
||||
9 | 10 => (0.2627, 0.0593),
|
||||
_ => (0.2126, 0.0722),
|
||||
};
|
||||
let kg = 1.0 - kr - kb;
|
||||
let (sy, oy, sc) = if full {
|
||||
(1.0f32, 0.0f32, 1.0f32)
|
||||
} else {
|
||||
(255.0 / 219.0, -16.0 / 255.0, 255.0 / 224.0)
|
||||
};
|
||||
let mat = [
|
||||
sy,
|
||||
sy,
|
||||
sy,
|
||||
0.0,
|
||||
-2.0 * (1.0 - kb) * kb / kg * sc,
|
||||
2.0 * (1.0 - kb) * sc,
|
||||
2.0 * (1.0 - kr) * sc,
|
||||
-2.0 * (1.0 - kr) * kr / kg * sc,
|
||||
0.0,
|
||||
];
|
||||
let off = [oy, -0.5, -0.5];
|
||||
for yuv in [
|
||||
[0.1f32, 0.3, 0.7],
|
||||
[0.9, 0.5, 0.5],
|
||||
[0.5, 0.2, 0.8],
|
||||
[16.0 / 255.0, 0.5, 0.5],
|
||||
] {
|
||||
let v = [yuv[0] + off[0], yuv[1] + off[1], yuv[2] + off[2]];
|
||||
let gl: [f32; 3] =
|
||||
core::array::from_fn(|r| (0..3).map(|c| mat[c * 3 + r] * v[c]).sum());
|
||||
let ours = apply(&rows, yuv);
|
||||
for (a, b) in gl.iter().zip(ours) {
|
||||
assert!((a - b).abs() < 1e-5, "{matrix}/{full}: gl {gl:?} rows {ours:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user