From 570ff504ad3e8f625d53901f08a0b0d77d59752b Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 17 Jul 2026 14:06:57 +0200 Subject: [PATCH] refactor(client-core/W8): split video.rs into flat decoder-backend siblings Break the 1974-line pf-client-core/src/video.rs into flat sibling modules (matching the crate's video_d3d11.rs / video_pyrowave.rs convention), leaving video.rs as the contract + Decoder dispatch facade: - video_color.rs : ColorDesc + csc_rows (the Y'CbCr->RGB matrix) - video_software.rs : the libavcodec/swscale SoftwareDecoder - video_vaapi.rs : the Linux-only VAAPI/DRM-PRIME backend (mod is cfg(linux)) - video_vulkan.rs : the FFmpeg Vulkan Video backend Every crate::video::X / video::X path stays byte-stable (ColorDesc + csc_rows re-exported from video.rs; frame POD, VulkanDecodeDevice, QueueLock, Decoder, decodable_codecs*, ffmpeg_codec_id, fourcc/drm_fourcc_for all stay in video.rs). Code-driven placements: averr, AVERROR_EAGAIN, frame_is_keyframe stay in video.rs (shared by all three decoders); DrmFrameGuard's field + drm_fourcc_for + Software/Vaapi/VulkanDecoder ctors/decode became pub(crate) (sibling access); the test module split three ways (software tests need private decoder internals). Pure move; no behavior change. Verified on Linux (home-worker-5): cargo clippy -p pf-client-core (default [pyrowave] + --no-default-features, --all-targets -D warnings) + cargo test. Windows verify BLOCKED environmentally: pf-client-core -> sdl3 build-from-source -> CMake/CL.exe fails on winbox's non-ASCII home path (fails the baseline too, independent of this split); the split's Windows surface (facade cfg(windows) bits + video_d3d11) is verbatim-preserved. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/pf-client-core/src/lib.rs | 8 + crates/pf-client-core/src/video.rs | 1111 +------------------ crates/pf-client-core/src/video_color.rs | 210 ++++ crates/pf-client-core/src/video_software.rs | 264 +++++ crates/pf-client-core/src/video_vaapi.rs | 243 ++++ crates/pf-client-core/src/video_vulkan.rs | 419 +++++++ 6 files changed, 1154 insertions(+), 1101 deletions(-) create mode 100644 crates/pf-client-core/src/video_color.rs create mode 100644 crates/pf-client-core/src/video_software.rs create mode 100644 crates/pf-client-core/src/video_vaapi.rs create mode 100644 crates/pf-client-core/src/video_vulkan.rs diff --git a/crates/pf-client-core/src/lib.rs b/crates/pf-client-core/src/lib.rs index f406bc8e..53c3f0a9 100644 --- a/crates/pf-client-core/src/lib.rs +++ b/crates/pf-client-core/src/lib.rs @@ -33,6 +33,14 @@ pub mod session; pub mod trust; #[cfg(any(target_os = "linux", windows))] pub mod video; +#[cfg(any(target_os = "linux", windows))] +mod video_color; +#[cfg(any(target_os = "linux", windows))] +mod video_software; +#[cfg(target_os = "linux")] +mod video_vaapi; +#[cfg(any(target_os = "linux", windows))] +mod video_vulkan; // PyroWave decode — Linux + `pyrowave` feature only (plan §4.5; the Windows client's // present-path decision and the Apple Metal port are their own phases). #[cfg(windows)] diff --git a/crates/pf-client-core/src/video.rs b/crates/pf-client-core/src/video.rs index 260d3014..50ac8915 100644 --- a/crates/pf-client-core/src/video.rs +++ b/crates/pf-client-core/src/video.rs @@ -32,13 +32,15 @@ #![allow(clippy::unnecessary_cast)] use anyhow::{anyhow, bail, Context as _, Result}; -use ffmpeg::format::Pixel; -use ffmpeg::software::scaling; -use ffmpeg::util::frame::Video as AvFrame; use ffmpeg_next as ffmpeg; #[cfg(target_os = "linux")] use std::os::fd::RawFd; -use std::ptr; + +pub use crate::video_color::{csc_rows, ColorDesc}; +use crate::video_software::SoftwareDecoder; +#[cfg(target_os = "linux")] +use crate::video_vaapi::VaapiDecoder; +use crate::video_vulkan::VulkanDecoder; /// One decoded frame headed for the presenter, carrying the host capture timestamp so the /// UI can measure capture→displayed latency at the moment it presents. @@ -115,45 +117,6 @@ pub struct VkVideoFrame { pub guard: DrmFrameGuard, } -/// The stream's colour signaling, read PER-FRAME from the decoder (HEVC VUI → the -/// `AVFrame` CICP fields). The Windows host switches an HDR desktop to Main10 BT.2020 PQ -/// **in-band** (the Welcome still says SDR — clients are expected to follow the VUI, as -/// the Windows/Apple/Android clients do), so rendering must follow the frames, not the -/// handshake — else PQ content drawn as BT.709 comes out washed out and desaturated. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub struct ColorDesc { - /// H.273 code points as signaled (2 = unspecified → the renderer picks the SDR default). - pub primaries: u8, - pub transfer: u8, - pub matrix: u8, - pub full_range: bool, -} - -impl ColorDesc { - /// Read the CICP fields off a raw decoded frame. Public: the Windows client's raw-FFI - /// D3D11VA/software decoders build their per-frame `ColorDesc` with it too (same - /// `ffmpeg-next` major, so the `AVFrame` type unifies across the workspace). - /// - /// # Safety - /// `frame` must point to a valid `AVFrame` (alive for the duration of the call). - pub unsafe fn from_raw(frame: *const ffmpeg::ffi::AVFrame) -> ColorDesc { - // SAFETY: caller guarantees a live AVFrame; these are plain enum field reads. - unsafe { - ColorDesc { - primaries: (*frame).color_primaries as u32 as u8, - transfer: (*frame).color_trc as u32 as u8, - matrix: (*frame).colorspace as u32 as u8, - full_range: (*frame).color_range == ffmpeg::ffi::AVColorRange::AVCOL_RANGE_JPEG, - } - } - } - - /// PQ (SMPTE ST.2084) transfer — the HDR10 signal. - pub fn is_pq(&self) -> bool { - self.transfer == 16 - } -} - /// True if the decoder tagged this frame as a full IDR keyframe — a guaranteed clean re-anchor /// after which the picture is loss-free, so the pump can lift a post-loss display freeze here. /// @@ -210,57 +173,6 @@ impl DecodedImage { } } -/// The Y′CbCr→RGB conversion as three vec4 rows for a shader constant buffer / push-constant -/// block: `rgb[i] = dot(r[i].xyz, yuv) + r[i].w` — bit-depth exact. The ONE coefficient -/// implementation every presenter derives its CSC from (Vulkan push constants, the Windows -/// client's D3D11 constant buffer), so a stream's signaled matrix/range is honored identically -/// everywhere; the Apple client ports this function (and its tests) to Swift. -/// -/// `depth` picks the limited-range code points (8-bit: 16/235/240 over 255; 10-bit: -/// 64/940/960 over 1023 — NOT the same normalized values, the difference is ~half a -/// code). `msb_packed` folds in the P010/X6 packing factor: 10 significant bits live in -/// the MSBs of 16, so a UNORM16 sample reads `code·64/65535` — multiplying by -/// `65535/65472` recovers exact `code/1023`. -pub fn csc_rows(desc: ColorDesc, depth: u8, msb_packed: bool) -> [[f32; 4]; 3] { - // BT.601 (5/6), BT.2020 (9/10); everything else — incl. unspecified — is the host's - // BT.709 SDR default (mirrors the software path's swscale coefficient choice). - let (kr, kb) = match desc.matrix { - 5 | 6 => (0.299, 0.114), - 9 | 10 => (0.2627, 0.0593), - _ => (0.2126, 0.0722), - }; - let kg = 1.0 - kr - kb; - let max = f64::from((1u32 << depth) - 1); // 255 / 1023 - let step = f64::from(1u32 << (depth - 8)); // code points per 8-bit step: 1 / 4 - let pack = if msb_packed { 65535.0 / 65472.0 } else { 1.0 }; - let (sy, oy, sc) = if desc.full_range { - (pack, 0.0f64, pack) - } else { - ( - pack * max / (219.0 * step), - -(16.0 * step) / max, - pack * max / (224.0 * step), - ) - }; - // rgb = M * (yuv + off) = M*yuv + M*off — rows of M with the offset dot folded into - // w. `yuv` is the SAMPLED (packed) value, so the offsets divide by the packing - // factor to land on the same scale. - let off = [oy / pack, -0.5 / pack, -0.5 / pack]; - let m = [ - [sy, 0.0, 2.0 * (1.0 - kr) * sc], - [ - sy, - -2.0 * (1.0 - kb) * kb / kg * sc, - -2.0 * (1.0 - kr) * kr / kg * sc, - ], - [sy, 2.0 * (1.0 - kb) * sc, 0.0], - ]; - core::array::from_fn(|r| { - let w: f64 = (0..3).map(|c| m[r][c] * off[c]).sum(); - [m[r][0] as f32, m[r][1] as f32, m[r][2] as f32, w as f32] - }) -} - /// RGBA pixels for `GdkMemoryTexture` (which takes a stride). pub struct CpuFrame { pub width: u32, @@ -305,7 +217,7 @@ pub struct DmabufPlane { /// Owns the mapped DRM-PRIME `AVFrame` (which in turn references the VAAPI surface). /// Dropping it releases the surface back to the decoder pool and closes the fds. -pub struct DrmFrameGuard(*mut ffmpeg::ffi::AVFrame); +pub struct DrmFrameGuard(pub(crate) *mut ffmpeg::ffi::AVFrame); // An AVFrame is plain refcounted data; freeing it from the GTK main thread is fine. unsafe impl Send for DrmFrameGuard {} @@ -704,348 +616,13 @@ impl Decoder { } } -// --- software backend --------------------------------------------------------------- - -struct SoftwareDecoder { - decoder: ffmpeg::decoder::Video, - /// Rebuilt whenever the decoded format/size — or the colour signaling (a mid-stream - /// SDR↔HDR flip) — changes. - sws: Option<(scaling::Context, Pixel, u32, u32, ColorDesc)>, -} - -impl SoftwareDecoder { - fn new(codec_id: ffmpeg::codec::Id) -> Result { - let codec = ffmpeg::decoder::find(codec_id) - .ok_or_else(|| anyhow!("no {codec_id:?} decoder in libavcodec"))?; - let mut ctx = ffmpeg::codec::Context::new_with_codec(codec); - unsafe { - let raw = ctx.as_mut_ptr(); - (*raw).flags |= ffmpeg::ffi::AV_CODEC_FLAG_LOW_DELAY as i32; - // Slice threading adds no frame delay (frame threading adds thread_count-1). - (*raw).thread_type = ffmpeg::ffi::FF_THREAD_SLICE; - (*raw).thread_count = 0; // auto - } - let decoder = ctx.decoder().video().context("open video decoder")?; - Ok(SoftwareDecoder { decoder, sws: None }) - } - - fn decode(&mut self, au: &[u8]) -> Result> { - let packet = ffmpeg::Packet::copy(au); - self.decoder - .send_packet(&packet) - .map_err(|e| anyhow!("send_packet: {e}"))?; - let mut frame = AvFrame::empty(); - let mut out = None; - while self.decoder.receive_frame(&mut frame).is_ok() { - out = Some(self.convert_rgba(&frame)?); - } - Ok(out) - } - - fn convert_rgba(&mut self, frame: &AvFrame) -> Result { - let (fmt, w, h) = (frame.format(), frame.width(), frame.height()); - // SAFETY: `frame.as_ptr()` is the decoder-owned live AVFrame for this call. - let color = unsafe { ColorDesc::from_raw(frame.as_ptr()) }; - let rebuild = !matches!(&self.sws, - Some((_, f, sw, sh, c)) if *f == fmt && *sw == w && *sh == h && *c == color); - if rebuild { - let mut ctx = - scaling::Context::get(fmt, w, h, Pixel::RGBA, w, h, scaling::Flags::POINT) - .context("swscale context")?; - // swscale defaults to BT.601 coefficients — set them from the FRAME's signaling - // (unspecified → BT.709 limited, the host's SDR default; a Windows HDR desktop - // streams BT.2020 in-band). Without this, YUV→RGB decodes with the wrong matrix - // and colours shift. Destination = full-range RGB; the transfer function stays - // baked in (the presenter tags PQ textures so GTK applies the EOTF). - const SWS_CS_ITU709: i32 = 1; - const SWS_CS_ITU601: i32 = 5; - const SWS_CS_BT2020: i32 = 9; - let cs = match color.matrix { - 9 | 10 => SWS_CS_BT2020, - 5 | 6 => SWS_CS_ITU601, - _ => SWS_CS_ITU709, - }; - unsafe { - let coeffs = ffmpeg::ffi::sws_getCoefficients(cs); - ffmpeg::ffi::sws_setColorspaceDetails( - ctx.as_mut_ptr(), - coeffs, // inv_table: source (YUV) coefficients per the VUI - color.full_range as i32, // srcRange: 0 = limited/studio (MPEG) - coeffs, // table: destination coefficients (ignored for RGB output) - 1, // dstRange: 1 = full-range RGB - 0, - 1 << 16, - 1 << 16, // brightness, contrast, saturation (defaults) - ); - } - self.sws = Some((ctx, fmt, w, h, color)); - } - let (sws, ..) = self.sws.as_mut().unwrap(); - // Single-pass conversion: swscale writes straight into the Vec the texture will - // wrap. (The old path scaled into a scratch AVFrame and then copied `data(0)` out - // — a second full-frame pass per frame.) 64-byte row alignment keeps swscale on - // aligned SIMD stores; `GdkMemoryTexture` takes the resulting stride explicitly. - const ALIGN: i32 = 64; - use ffmpeg::ffi; - let dst_fmt = ffi::AVPixelFormat::AV_PIX_FMT_RGBA; - // SAFETY: pure size computation from format/dimensions; no pointers involved. - let size = unsafe { ffi::av_image_get_buffer_size(dst_fmt, w as i32, h as i32, ALIGN) }; - if size < 0 { - return Err(averr("av_image_get_buffer_size", size)); - } - let rgba = vec![0u8; size as usize]; - let mut dst_data: [*mut u8; 4] = [ptr::null_mut(); 4]; - let mut dst_linesize: [i32; 4] = [0; 4]; - // SAFETY: fill_arrays only derives plane pointers/strides into `rgba` (sized by - // av_image_get_buffer_size above, same format/align) — no allocation, no - // ownership transfer; `rgba` outlives the scale below. - let r = unsafe { - ffi::av_image_fill_arrays( - dst_data.as_mut_ptr(), - dst_linesize.as_mut_ptr(), - rgba.as_ptr(), - dst_fmt, - w as i32, - h as i32, - ALIGN, - ) - }; - if r < 0 { - return Err(averr("av_image_fill_arrays", r)); - } - // SAFETY: src pointers/strides belong to the decoder-owned `frame` (alive for the - // call); dst pointers were just filled over `rgba`, and sws_scale writes rows - // [0, h) only — exactly the buffer fill_arrays sized. - let r = unsafe { - ffi::sws_scale( - sws.as_mut_ptr(), - (*frame.as_ptr()).data.as_ptr() as *const *const u8, - (*frame.as_ptr()).linesize.as_ptr(), - 0, - h as i32, - dst_data.as_ptr(), - dst_linesize.as_ptr(), - ) - }; - if r < 0 { - return Err(averr("sws_scale", r)); - } - Ok(CpuFrame { - width: w, - height: h, - stride: dst_linesize[0] as usize, - rgba, - color, - // `is_key()` reads the same intra flag `frame_is_keyframe` derives from pict_type - // for the hardware paths; ffmpeg-next handles the FFmpeg-version binding split. - keyframe: frame.is_key(), - }) - } -} - -// --- VAAPI backend -------------------------------------------------------------------- -// -// Raw FFI: ffmpeg-next has no hwaccel wrappers. All pointers are owned here and freed in -// Drop; decoded surfaces transfer out through DrmFrameGuard. - // -EAGAIN. FFmpeg uses POSIX errno values on both our targets (MinGW's EAGAIN is 11 too). -const AVERROR_EAGAIN: i32 = -11; +pub(crate) const AVERROR_EAGAIN: i32 = -11; -fn averr(what: &str, code: i32) -> anyhow::Error { +pub(crate) fn averr(what: &str, code: i32) -> anyhow::Error { anyhow!("{what}: {}", ffmpeg::Error::from(code)) } -/// libavcodec offers the formats it can decode into; pick the VAAPI hw surface. Falling -/// back to the first (software) entry would silently decode on the CPU *and* break our -/// dmabuf mapping — return NONE instead so the error surfaces and the session demotes -/// to the software backend explicitly. -#[cfg(target_os = "linux")] -unsafe extern "C" fn pick_vaapi( - _ctx: *mut ffmpeg::ffi::AVCodecContext, - mut list: *const ffmpeg::ffi::AVPixelFormat, -) -> ffmpeg::ffi::AVPixelFormat { - unsafe { - while *list != ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE { - if *list == ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI { - return ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI; - } - list = list.add(1); - } - } - ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE -} - -#[cfg(target_os = "linux")] -struct VaapiDecoder { - ctx: *mut ffmpeg::ffi::AVCodecContext, - hw_device: *mut ffmpeg::ffi::AVBufferRef, - packet: *mut ffmpeg::ffi::AVPacket, - frame: *mut ffmpeg::ffi::AVFrame, -} - -// Single-owner pointers, only touched from the session pump thread. -#[cfg(target_os = "linux")] -unsafe impl Send for VaapiDecoder {} - -#[cfg(target_os = "linux")] -impl VaapiDecoder { - fn new(codec_id: ffmpeg::codec::Id) -> Result { - use ffmpeg::ffi; - unsafe { - let mut hw_device: *mut ffi::AVBufferRef = ptr::null_mut(); - let r = ffi::av_hwdevice_ctx_create( - &mut hw_device, - ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI, - ptr::null(), - ptr::null_mut(), - 0, - ); - if r < 0 { - bail!("no VAAPI device ({})", ffmpeg::Error::from(r)); - } - // The negotiated codec's decoder id (av_codec_id maps 1:1 from ffmpeg::codec::Id). - let codec = ffi::avcodec_find_decoder(codec_id.into()); - if codec.is_null() { - ffi::av_buffer_unref(&mut hw_device); - bail!("no {codec_id:?} decoder"); - } - let ctx = ffi::avcodec_alloc_context3(codec); - (*ctx).hw_device_ctx = ffi::av_buffer_ref(hw_device); - (*ctx).get_format = Some(pick_vaapi); - (*ctx).flags |= ffi::AV_CODEC_FLAG_LOW_DELAY as i32; - (*ctx).thread_count = 1; // hwaccel: threads only add latency - - // The presenter holds mapped surfaces PAST receive_frame (the paintable's - // current texture + the newest frame in flight each pin one until GDK's - // release func) — surfaces libavcodec doesn't know are missing from its - // fixed-size VAAPI pool. Without headroom the decoder can recycle a surface - // the renderer is still sampling (intermittent block corruption) or fail - // allocation under scheduling jitter. - (*ctx).extra_hw_frames = 4; - let r = ffi::avcodec_open2(ctx, codec, ptr::null_mut()); - if r < 0 { - let mut ctx = ctx; - ffi::avcodec_free_context(&mut ctx); - let mut hw_device = hw_device; - ffi::av_buffer_unref(&mut hw_device); - bail!("avcodec_open2: {}", ffmpeg::Error::from(r)); - } - Ok(VaapiDecoder { - ctx, - hw_device, - packet: ffi::av_packet_alloc(), - frame: ffi::av_frame_alloc(), - }) - } - } - - fn decode(&mut self, au: &[u8]) -> Result> { - use ffmpeg::ffi; - unsafe { - let r = ffi::av_new_packet(self.packet, au.len() as i32); - if r < 0 { - return Err(averr("av_new_packet", r)); - } - ptr::copy_nonoverlapping(au.as_ptr(), (*self.packet).data, au.len()); - let r = ffi::avcodec_send_packet(self.ctx, self.packet); - ffi::av_packet_unref(self.packet); - if r < 0 { - return Err(averr("send_packet", r)); - } - let mut out = None; - loop { - let r = ffi::avcodec_receive_frame(self.ctx, self.frame); - if r == AVERROR_EAGAIN { - break; - } - if r < 0 { - return Err(averr("receive_frame", r)); - } - out = Some(self.map_dmabuf()?); // newest wins; older guards drop here - ffi::av_frame_unref(self.frame); - } - Ok(out) - } - } - - /// Map the VAAPI surface to DRM PRIME (zero copy) and lift the descriptor into a - /// `DmabufFrame`. The mapped frame keeps the surface alive via its buffer refs. - /// - /// FFmpeg's VAAPI export uses `VA_EXPORT_SURFACE_SEPARATE_LAYERS`, so an NV12 surface - /// comes back as TWO layers (`R8` luma + `GR88` chroma), each one plane — NOT a single - /// `NV12` layer. The previous code took `layers[0]` only: GTK then saw an `R8` - /// single-plane texture with the chroma dropped, painting the screen green. The fix: - /// derive the COMBINED fourcc from the decoder's software pixel format (NV12 → - /// `DRM_FORMAT_NV12`) and flatten every plane across every layer in order (Y then UV). - unsafe fn map_dmabuf(&mut self) -> Result { - use ffmpeg::ffi; - unsafe { - if (*self.frame).format != ffi::AVPixelFormat::AV_PIX_FMT_VAAPI as i32 { - bail!("decoder returned a software frame (no VAAPI surface)"); - } - // The real pixel layout lives on the hardware frames context, not the - // DRM-PRIME layer formats (those are the per-plane R8/GR88 component formats). - let sw_format = { - let hwfc = (*self.frame).hw_frames_ctx; - if hwfc.is_null() { - bail!("VAAPI frame without a hardware frames context"); - } - (*((*hwfc).data as *const ffi::AVHWFramesContext)).sw_format - }; - let fourcc = drm_fourcc_for(sw_format) - .ok_or_else(|| anyhow!("unsupported VAAPI output format {sw_format:?}"))?; - - let drm = ffi::av_frame_alloc(); - (*drm).format = ffi::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32; - let r = ffi::av_hwframe_map(drm, self.frame, ffi::AV_HWFRAME_MAP_READ as i32); - if r < 0 { - let mut drm = drm; - ffi::av_frame_free(&mut drm); - return Err(averr("av_hwframe_map", r)); - } - let desc = (*drm).data[0] as *const ffi::AVDRMFrameDescriptor; - let guard = DrmFrameGuard(drm); - let d = &*desc; - if d.nb_layers < 1 || d.nb_objects < 1 { - bail!("DRM descriptor without layers/objects"); - } - - // Flatten planes across ALL layers, in declared order — the combined fourcc's - // plane order (Y, then UV for NV12) matches the layer order FFmpeg emits. - let mut planes = Vec::new(); - for layer in &d.layers[..d.nb_layers as usize] { - for p in &layer.planes[..layer.nb_planes as usize] { - let obj = &d.objects[p.object_index as usize]; - planes.push(DmabufPlane { - fd: obj.fd, - offset: p.offset as u32, - stride: p.pitch as u32, - }); - } - } - - // The whole surface shares one tiling modifier (one BO on radeonsi); GTK takes - // a single modifier for the texture. - let modifier = d.objects[0].format_modifier; - - log_descriptor_once(d, sw_format, fourcc, modifier); - - Ok(DmabufFrame { - width: (*self.frame).width as u32, - height: (*self.frame).height as u32, - fourcc, - modifier, - planes, - // SAFETY: `self.frame` is the live decoded AVFrame (unref'd only after - // this returns); plain CICP field reads. - color: ColorDesc::from_raw(self.frame), - keyframe: frame_is_keyframe(self.frame), - guard, - }) - } - } -} - /// Guard-less mutex serializing every `vkQueueSubmit`/`vkQueuePresentKHR`/ /// `vkQueueWaitIdle` on the device the presenter shares with FFmpeg. /// @@ -1222,7 +799,7 @@ const fn fourcc(a: u8, b: u8, c: u8, d: u8) -> u32 { // Only the (Linux-gated) VAAPI path calls this outside tests; the constants are worth // locking on every platform, so it stays compiled rather than cfg-gated with its caller. #[cfg_attr(windows, allow(dead_code))] -fn drm_fourcc_for(sw: ffmpeg_next::ffi::AVPixelFormat) -> Option { +pub(crate) fn drm_fourcc_for(sw: ffmpeg_next::ffi::AVPixelFormat) -> Option { use ffmpeg_next::ffi::AVPixelFormat::*; Some(match sw { AV_PIX_FMT_NV12 => fourcc(b'N', b'V', b'1', b'2'), @@ -1231,453 +808,6 @@ fn drm_fourcc_for(sw: ffmpeg_next::ffi::AVPixelFormat) -> Option { }) } -/// One-time dump of the DRM descriptor layout (objects, layers, planes, modifier) — so a -/// new client/driver combination's real layout is visible in the logs without a debugger. -#[cfg(target_os = "linux")] -fn log_descriptor_once( - d: &ffmpeg_next::ffi::AVDRMFrameDescriptor, - sw: ffmpeg_next::ffi::AVPixelFormat, - fourcc: u32, - modifier: u64, -) { - use std::sync::atomic::{AtomicBool, Ordering}; - static ONCE: AtomicBool = AtomicBool::new(true); - if !ONCE.swap(false, Ordering::Relaxed) { - return; - } - let layers: Vec<(u32, i32)> = d.layers[..d.nb_layers.max(0) as usize] - .iter() - .map(|l| (l.format, l.nb_planes)) - .collect(); - tracing::info!( - sw_format = ?sw, - chosen_fourcc = format_args!("{:#010x}", fourcc), - nb_objects = d.nb_objects, - nb_layers = d.nb_layers, - ?layers, - modifier = format_args!("{:#018x}", modifier), - "VAAPI dmabuf descriptor layout (first frame)" - ); -} - -#[cfg(target_os = "linux")] -impl Drop for VaapiDecoder { - fn drop(&mut self) { - use ffmpeg::ffi; - unsafe { - ffi::av_packet_free(&mut self.packet); - ffi::av_frame_free(&mut self.frame); - ffi::avcodec_free_context(&mut self.ctx); - ffi::av_buffer_unref(&mut self.hw_device); - } - } -} - -// --- Vulkan Video backend ------------------------------------------------------------- - -/// FFmpeg's Vulkan Video decoder over the PRESENTER's device: the hwdevice context is -/// built from [`VulkanDecodeDevice`]'s handles (not `av_hwdevice_ctx_create`, which -/// would make FFmpeg create its own device the presenter can't sample from). Output -/// frames are `AVVkFrame`s whose VkImage the presenter feeds straight to its CSC pass. -struct VulkanDecoder { - ctx: *mut ffmpeg::ffi::AVCodecContext, - hw_device: *mut ffmpeg::ffi::AVBufferRef, - packet: *mut ffmpeg::ffi::AVPacket, - frame: *mut ffmpeg::ffi::AVFrame, - /// `vkWaitSemaphores` on the shared device — the decode-complete measurement - /// (resolved through the same get_proc_addr chain FFmpeg uses). - wait_semaphores: pf_ffvk::PFN_vkWaitSemaphores, - vk_device: pf_ffvk::VkDevice, - /// Storage `AVVulkanDeviceContext` points into (extension string arrays + the - /// feature chain) — FFmpeg reads the extension lists past init (frames-context - /// setup keys code paths off them), so this lives exactly as long as `hw_device`. - _ctx_storage: Box, -} - -// Single-owner pointers, only touched from the session pump thread. -unsafe impl Send for VulkanDecoder {} - -struct VkCtxStorage { - _inst: Vec, - inst_ptrs: Vec<*const std::os::raw::c_char>, - _dev: Vec, - dev_ptrs: Vec<*const std::os::raw::c_char>, - f11: pf_ffvk::VkPhysicalDeviceVulkan11Features, - f12: pf_ffvk::VkPhysicalDeviceVulkan12Features, - f13: pf_ffvk::VkPhysicalDeviceVulkan13Features, - /// Keeps the shared queue lock alive for `AVHWDeviceContext.user_opaque` — the - /// `lock_queue`/`unlock_queue` trampolines below dereference it for as long as the - /// hw device context can fire them. - _queue_lock: std::sync::Arc, -} - -/// FFmpeg `AVVulkanDeviceContext.lock_queue` trampoline: take the device's shared -/// [`QueueLock`] (stashed in `AVHWDeviceContext.user_opaque`; owned by -/// [`VkCtxStorage`], which outlives the context). Replaces FFmpeg's internal default, -/// which only serializes FFmpeg against itself — the presenter submits to the same -/// graphics queue from another thread and holds this same lock around its calls. -unsafe extern "C" fn ffvk_lock_queue( - ctx: *mut pf_ffvk::AVHWDeviceContext, - _queue_family: u32, - _index: u32, -) { - let dev = ctx as *mut ffmpeg::ffi::AVHWDeviceContext; - let lock = (*dev).user_opaque as *const QueueLock; - (*lock).lock(); -} - -/// The matching `unlock_queue` trampoline — see [`ffvk_lock_queue`]. -unsafe extern "C" fn ffvk_unlock_queue( - ctx: *mut pf_ffvk::AVHWDeviceContext, - _queue_family: u32, - _index: u32, -) { - let dev = ctx as *mut ffmpeg::ffi::AVHWDeviceContext; - let lock = (*dev).user_opaque as *const QueueLock; - (*lock).unlock(); -} - -impl VulkanDecoder { - fn new(codec_id: ffmpeg::codec::Id, vk: &VulkanDecodeDevice) -> Result { - use ffmpeg::ffi; - unsafe { - let mut hw_device = - ffi::av_hwdevice_ctx_alloc(ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VULKAN); - if hw_device.is_null() { - bail!("av_hwdevice_ctx_alloc(VULKAN) failed (FFmpeg built without Vulkan?)"); - } - let devctx = (*hw_device).data as *mut ffi::AVHWDeviceContext; - let hwctx = (*devctx).hwctx as *mut pf_ffvk::AVVulkanDeviceContext; - - // Pinned storage for everything the context points into. - let mut store = Box::new(VkCtxStorage { - _inst: vk.instance_extensions.clone(), - inst_ptrs: Vec::new(), - _dev: vk.device_extensions.clone(), - dev_ptrs: Vec::new(), - f11: std::mem::zeroed(), - f12: std::mem::zeroed(), - f13: std::mem::zeroed(), - _queue_lock: vk.queue_lock.clone(), - }); - store.inst_ptrs = store._inst.iter().map(|c| c.as_ptr()).collect(); - store.dev_ptrs = store._dev.iter().map(|c| c.as_ptr()).collect(); - // The features enabled at device creation, as the 1.1/1.2/1.3 chain FFmpeg - // walks to learn what it may use (sType values are vulkan.h constants). - store.f11.sType = - pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES; - store.f11.samplerYcbcrConversion = vk.f_sampler_ycbcr as u32; - store.f12.sType = - pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; - store.f12.timelineSemaphore = vk.f_timeline_semaphore as u32; - store.f13.sType = - pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES; - store.f13.synchronization2 = vk.f_synchronization2 as u32; - store.f11.pNext = &mut store.f12 as *mut _ as *mut std::ffi::c_void; - store.f12.pNext = &mut store.f13 as *mut _ as *mut std::ffi::c_void; - - (*hwctx).get_proc_addr = std::mem::transmute::( - vk.get_instance_proc_addr, - ); - (*hwctx).inst = vk.instance as pf_ffvk::VkInstance; - (*hwctx).phys_dev = vk.physical_device as pf_ffvk::VkPhysicalDevice; - (*hwctx).act_dev = vk.device as pf_ffvk::VkDevice; - (*hwctx).device_features.sType = - pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; - (*hwctx).device_features.pNext = &mut store.f11 as *mut _ as *mut std::ffi::c_void; - (*hwctx).enabled_inst_extensions = store.inst_ptrs.as_ptr(); - (*hwctx).nb_enabled_inst_extensions = store.inst_ptrs.len() as i32; - (*hwctx).enabled_dev_extensions = store.dev_ptrs.as_ptr(); - (*hwctx).nb_enabled_dev_extensions = store.dev_ptrs.len() as i32; - - // Queue map: the deprecated per-role indices (tx/comp are "Required") plus - // the qf[] list, which per the header must also carry every family named - // above. One merged entry when decode shares the graphics family. - let g = vk.graphics_qf as i32; - let d = vk.decode_qf as i32; - (*hwctx).queue_family_index = g; - (*hwctx).nb_graphics_queues = 1; - (*hwctx).queue_family_tx_index = g; - (*hwctx).nb_tx_queues = 1; - (*hwctx).queue_family_comp_index = g; - (*hwctx).nb_comp_queues = 1; - (*hwctx).queue_family_encode_index = -1; - (*hwctx).nb_encode_queues = 0; - (*hwctx).queue_family_decode_index = d; - (*hwctx).nb_decode_queues = 1; - const VIDEO_DECODE_BIT: u32 = 0x20; // VK_QUEUE_VIDEO_DECODE_BIT_KHR - // `flags`/`video_caps` are bindgen enum types: i32 under MSVC, u32 under - // Linux clang — the `as _` casts absorb the difference. - if g == d { - (*hwctx).qf[0] = pf_ffvk::AVVulkanDeviceQueueFamily { - idx: g, - num: 1, - flags: (vk.graphics_queue_flags | VIDEO_DECODE_BIT) as _, - video_caps: vk.decode_video_caps as _, - }; - (*hwctx).nb_qf = 1; - } else { - (*hwctx).qf[0] = pf_ffvk::AVVulkanDeviceQueueFamily { - idx: g, - num: 1, - flags: vk.graphics_queue_flags as _, - video_caps: 0, - }; - (*hwctx).qf[1] = pf_ffvk::AVVulkanDeviceQueueFamily { - idx: d, - num: 1, - flags: VIDEO_DECODE_BIT as _, - video_caps: vk.decode_video_caps as _, - }; - (*hwctx).nb_qf = 2; - } - - // Shared-queue external sync (see [`QueueLock`]): FFmpeg must take the - // same lock the presenter holds around its own submits/presents — set - // BEFORE init so FFmpeg never installs its internal defaults (which only - // serialize FFmpeg against itself; the cross-thread race with the - // presenter's queue was an intermittent VK_ERROR_DEVICE_LOST). - (*devctx).user_opaque = - std::sync::Arc::as_ptr(&store._queue_lock) as *mut std::ffi::c_void; - (*hwctx).lock_queue = Some(ffvk_lock_queue); - (*hwctx).unlock_queue = Some(ffvk_unlock_queue); - - let r = ffi::av_hwdevice_ctx_init(hw_device); - if r < 0 { - ffi::av_buffer_unref(&mut hw_device); - return Err(averr("av_hwdevice_ctx_init(VULKAN)", r)); - } - - // vkWaitSemaphores for the pump's decode-complete stat: loader → - // vkGetDeviceProcAddr → device fn (core 1.2, guaranteed by our gate). - let gipa = (*hwctx) - .get_proc_addr - .expect("get_proc_addr was just set above"); - let gdpa: pf_ffvk::PFN_vkGetDeviceProcAddr = - std::mem::transmute(gipa((*hwctx).inst, c"vkGetDeviceProcAddr".as_ptr())); - let wait_semaphores: pf_ffvk::PFN_vkWaitSemaphores = std::mem::transmute(gdpa - .expect("vkGetDeviceProcAddr resolvable")( - (*hwctx).act_dev, - c"vkWaitSemaphores".as_ptr(), - )); - if wait_semaphores.is_none() { - ffi::av_buffer_unref(&mut hw_device); - bail!("vkWaitSemaphores unresolvable on this device"); - } - let vk_device = (*hwctx).act_dev; - - let codec = ffi::avcodec_find_decoder(codec_id.into()); - if codec.is_null() { - ffi::av_buffer_unref(&mut hw_device); - bail!("no {codec_id:?} decoder"); - } - let ctx = ffi::avcodec_alloc_context3(codec); - (*ctx).hw_device_ctx = ffi::av_buffer_ref(hw_device); - (*ctx).get_format = Some(pick_vulkan); - (*ctx).flags |= ffi::AV_CODEC_FLAG_LOW_DELAY as i32; - (*ctx).thread_count = 1; // hwaccel: threads only add latency - // Same pool headroom rationale as VAAPI: the presenter pins the on-screen - // frame + the newest in flight past receive_frame. - (*ctx).extra_hw_frames = 4; - let r = ffi::avcodec_open2(ctx, codec, ptr::null_mut()); - if r < 0 { - let mut ctx = ctx; - ffi::avcodec_free_context(&mut ctx); - ffi::av_buffer_unref(&mut hw_device); - return Err(averr("avcodec_open2 (vulkan)", r)); - } - Ok(VulkanDecoder { - ctx, - hw_device, - packet: ffi::av_packet_alloc(), - frame: ffi::av_frame_alloc(), - wait_semaphores, - vk_device, - _ctx_storage: store, - }) - } - } - - fn decode(&mut self, au: &[u8]) -> Result> { - use ffmpeg::ffi; - unsafe { - let r = ffi::av_new_packet(self.packet, au.len() as i32); - if r < 0 { - return Err(averr("av_new_packet", r)); - } - ptr::copy_nonoverlapping(au.as_ptr(), (*self.packet).data, au.len()); - let r = ffi::avcodec_send_packet(self.ctx, self.packet); - ffi::av_packet_unref(self.packet); - if r < 0 { - return Err(averr("send_packet", r)); - } - let mut out = None; - loop { - let r = ffi::avcodec_receive_frame(self.ctx, self.frame); - if r == AVERROR_EAGAIN { - break; - } - if r < 0 { - return Err(averr("receive_frame", r)); - } - out = Some(self.extract()?); // newest wins; older guards drop here - ffi::av_frame_unref(self.frame); - } - Ok(out) - } - } - - /// Block until the timeline semaphore reaches `value` (GPU decode complete) or the - /// timeout passes. Pure measurement — the presenter's own GPU wait is what gates - /// sampling, so a timeout here only degrades the stat, never the picture. - fn wait_timeline(&self, sem: u64, value: u64, timeout_ns: u64) -> bool { - let sems = [sem as pf_ffvk::VkSemaphore]; - let values = [value]; - let info = pf_ffvk::VkSemaphoreWaitInfo { - sType: pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, - pNext: std::ptr::null(), - flags: 0, - semaphoreCount: 1, - pSemaphores: sems.as_ptr(), - pValues: values.as_ptr(), - }; - // SAFETY: resolved from this device at init; handles outlive the decoder. - let r = unsafe { - self.wait_semaphores.expect("checked at init")(self.vk_device, &info, timeout_ns) - }; - r == 0 // VK_SUCCESS (VK_TIMEOUT = 2) - } - - /// Lift the decoded `AVVkFrame` into a [`VkVideoFrame`]: clone the AVFrame (the - /// guard — keeps the image + frames context alive through present) and ship the - /// POINTERS; the presenter reads the live sync state under the frames-context lock - /// at its own submit time. - unsafe fn extract(&mut self) -> Result { - use ffmpeg::ffi; - unsafe { - if (*self.frame).format != ffi::AVPixelFormat::AV_PIX_FMT_VULKAN as i32 { - bail!("decoder returned a non-Vulkan frame"); - } - let hwfc_ref = (*self.frame).hw_frames_ctx; - if hwfc_ref.is_null() { - bail!("Vulkan frame without a hardware frames context"); - } - let fc = (*hwfc_ref).data as *mut ffi::AVHWFramesContext; - let sw = (*fc).sw_format; - if sw != ffi::AVPixelFormat::AV_PIX_FMT_NV12 - && sw != ffi::AVPixelFormat::AV_PIX_FMT_P010LE - { - bail!("Vulkan decode output {sw:?} unsupported (NV12/P010 only)"); - } - let vkfc = (*fc).hwctx as *const pf_ffvk::AVVulkanFramesContext; - let vk_format = (*vkfc).format[0] as i32; - let lock_frame = (*vkfc).lock_frame.map_or(0, |f| f as usize); - let unlock_frame = (*vkfc).unlock_frame.map_or(0, |f| f as usize); - if lock_frame == 0 || unlock_frame == 0 { - bail!("Vulkan frames context without lock functions"); - } - - let clone = ffi::av_frame_clone(self.frame); - if clone.is_null() { - bail!("av_frame_clone failed"); - } - let vkf = (*clone).data[0] as *mut pf_ffvk::AVVkFrame; - // v1 handles the (default) single multiplanar image; a disjoint/multi-image - // pool would need per-plane images — bail so the session demotes cleanly. - if !(*vkf).img[1].is_null() { - let mut clone = clone; - ffi::av_frame_free(&mut clone); - bail!("multi-image Vulkan frames unsupported (disjoint pool)"); - } - // Safe without the frames lock: the handle is creation-constant and - // sem_value was last written by the decode submission on THIS thread. - let timeline_sem = (*vkf).sem[0] as u64; - let decode_done_value = (*vkf).sem_value[0]; - Ok(VkVideoFrame { - vkframe: vkf as usize, - frames_ctx: fc as usize, - lock_frame, - unlock_frame, - vk_format, - timeline_sem, - decode_done_value, - width: (*self.frame).width as u32, - height: (*self.frame).height as u32, - color: ColorDesc::from_raw(self.frame), - keyframe: frame_is_keyframe(self.frame), - guard: DrmFrameGuard(clone), - }) - } - } -} - -impl Drop for VulkanDecoder { - fn drop(&mut self) { - use ffmpeg::ffi; - unsafe { - ffi::av_packet_free(&mut self.packet); - ffi::av_frame_free(&mut self.frame); - ffi::avcodec_free_context(&mut self.ctx); - ffi::av_buffer_unref(&mut self.hw_device); - } - } -} - -/// libavcodec offers the formats it can decode into; pick the Vulkan hw surface and -/// hand the decoder OUR frames context — the default one lacks -/// `VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT`, without which the presenter can't create the -/// per-plane views its CSC pass samples. Returning NONE (over the software entry) keeps -/// failures loud: the session demotes explicitly instead of silently CPU-decoding. -unsafe extern "C" fn pick_vulkan( - ctx: *mut ffmpeg::ffi::AVCodecContext, - mut list: *const ffmpeg::ffi::AVPixelFormat, -) -> ffmpeg::ffi::AVPixelFormat { - use ffmpeg::ffi; - unsafe { - let mut offered = false; - while *list != ffi::AVPixelFormat::AV_PIX_FMT_NONE { - if *list == ffi::AVPixelFormat::AV_PIX_FMT_VULKAN { - offered = true; - break; - } - list = list.add(1); - } - if !offered { - return ffi::AVPixelFormat::AV_PIX_FMT_NONE; - } - let mut fr: *mut ffi::AVBufferRef = ptr::null_mut(); - let r = ffi::avcodec_get_hw_frames_parameters( - ctx, - (*ctx).hw_device_ctx, - ffi::AVPixelFormat::AV_PIX_FMT_VULKAN, - &mut fr, - ); - if r < 0 || fr.is_null() { - tracing::warn!(code = r, "avcodec_get_hw_frames_parameters(VULKAN) failed"); - return ffi::AVPixelFormat::AV_PIX_FMT_NONE; - } - let fc = (*fr).data as *mut ffi::AVHWFramesContext; - let vkfc = (*fc).hwctx as *mut pf_ffvk::AVVulkanFramesContext; - // MUTABLE_FORMAT: per-plane views (spec requirement); ALIAS is FFmpeg's default. - // (`as _`: the FlagBits constants are i32 under MSVC, the img_flags field u32.) - (*vkfc).img_flags = (pf_ffvk::VkImageCreateFlagBits_VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT - | pf_ffvk::VkImageCreateFlagBits_VK_IMAGE_CREATE_ALIAS_BIT) - as _; - let r = ffi::av_hwframe_ctx_init(fr); - if r < 0 { - tracing::warn!(code = r, "av_hwframe_ctx_init(VULKAN) failed"); - let mut fr = fr; - ffi::av_buffer_unref(&mut fr); - return ffi::AVPixelFormat::AV_PIX_FMT_NONE; - } - if !(*ctx).hw_frames_ctx.is_null() { - ffi::av_buffer_unref(&mut (*ctx).hw_frames_ctx); - } - (*ctx).hw_frames_ctx = fr; // the codec owns our ref now - ffi::AVPixelFormat::AV_PIX_FMT_VULKAN - } -} - #[cfg(test)] mod tests { use super::*; @@ -1735,117 +865,6 @@ mod tests { ); } - fn desc(matrix: u8, full_range: bool) -> ColorDesc { - ColorDesc { - primaries: 1, - transfer: 1, - matrix, - full_range, - } - } - - fn apply(rows: &[[f32; 4]; 3], yuv: [f32; 3]) -> [f32; 3] { - core::array::from_fn(|r| { - rows[r][0] * yuv[0] + rows[r][1] * yuv[1] + rows[r][2] * yuv[2] + rows[r][3] - }) - } - - /// 10-bit limited MSB-packed (P010/X6): reference white Y=940, black Y=64, neutral - /// chroma 512 — sampled as UNORM16 of `code << 6`. - #[test] - fn bt2020_10bit_limited_white_black() { - let rows = csc_rows(desc(9, false), 10, true); - let s = |code: u32| ((code << 6) as f32) / 65535.0; - let white = apply(&rows, [s(940), s(512), s(512)]); - let black = apply(&rows, [s(64), s(512), s(512)]); - for (w, b) in white.iter().zip(black) { - assert!((w - 1.0).abs() < 0.002, "white {white:?}"); - assert!(b.abs() < 0.002, "black {black:?}"); - } - } - - /// Reference white (Y=235, U=V=128 limited) → RGB 1.0; reference black (Y=16) → 0.0 - /// — the GL presenter's test, in row form. - #[test] - fn bt709_limited_white_black() { - let rows = csc_rows(desc(1, false), 8, false); - let white = apply(&rows, [235.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0]); - let black = apply(&rows, [16.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0]); - for (w, b) in white.iter().zip(black) { - assert!((w - 1.0).abs() < 0.005, "white {white:?}"); - assert!(b.abs() < 0.005, "black {black:?}"); - } - } - - /// Full-range identity points + the 601-vs-709 red excursion (guards the - /// matrix-code dispatch), same as the GL presenter's test. - #[test] - fn full_range_and_red_excursion() { - let rows = csc_rows(desc(5, true), 8, false); - let white = apply(&rows, [1.0, 0.5, 0.5]); - assert!(white.iter().all(|v| (v - 1.0).abs() < 1e-5), "{white:?}"); - let red = apply(&rows, [0.0, 0.5, 1.0]); - assert!((red[0] - 2.0 * (1.0 - 0.299) * 0.5).abs() < 1e-4, "{red:?}"); - let rows709 = csc_rows(desc(1, true), 8, false); - let red709 = apply(&rows709, [0.0, 0.5, 1.0]); - assert!( - (red709[0] - 2.0 * (1.0 - 0.2126) * 0.5).abs() < 1e-4, - "{red709:?}" - ); - assert!((red[0] - red709[0]).abs() > 0.05); - } - - /// The row form must agree with the GL presenter's column-major `yuv_to_rgb` on a - /// grid of inputs — same math, different packing. - #[test] - fn rows_match_the_gl_matrix_form() { - for (matrix, full) in [(1u8, false), (1, true), (5, false), (9, false), (9, true)] { - let d = desc(matrix, full); - let rows = csc_rows(d, 8, false); - // Reimplementation of video_gl::yuv_to_rgb's application for comparison. - let (kr, kb) = match matrix { - 5 | 6 => (0.299f32, 0.114f32), - 9 | 10 => (0.2627, 0.0593), - _ => (0.2126, 0.0722), - }; - let kg = 1.0 - kr - kb; - let (sy, oy, sc) = if full { - (1.0f32, 0.0f32, 1.0f32) - } else { - (255.0 / 219.0, -16.0 / 255.0, 255.0 / 224.0) - }; - let mat = [ - sy, - sy, - sy, - 0.0, - -2.0 * (1.0 - kb) * kb / kg * sc, - 2.0 * (1.0 - kb) * sc, - 2.0 * (1.0 - kr) * sc, - -2.0 * (1.0 - kr) * kr / kg * sc, - 0.0, - ]; - let off = [oy, -0.5, -0.5]; - for yuv in [ - [0.1f32, 0.3, 0.7], - [0.9, 0.5, 0.5], - [0.5, 0.2, 0.8], - [16.0 / 255.0, 0.5, 0.5], - ] { - let v = [yuv[0] + off[0], yuv[1] + off[1], yuv[2] + off[2]]; - let gl: [f32; 3] = - core::array::from_fn(|r| (0..3).map(|c| mat[c * 3 + r] * v[c]).sum()); - let ours = apply(&rows, yuv); - for (a, b) in gl.iter().zip(ours) { - assert!( - (a - b).abs() < 1e-5, - "{matrix}/{full}: gl {gl:?} rows {ours:?}" - ); - } - } - } - } - /// Lock the DRM FourCC magic numbers against typos — these are the exact values /// `` defines, and a wrong one is what painted the Steam Deck green. #[test] @@ -1861,114 +880,4 @@ mod tests { None ); } - - /// The wire → `ColorDesc` plumbing: an HDR10 stream's VUI (BT.2020 primaries, PQ - /// transfer, BT.2020-NCL matrix, limited range) must arrive on the decoded frame — - /// this is what the Windows host emits in-band for an HDR desktop, and mis-rendering - /// it as BT.709 is the washed-out-colors bug. Fixture: one 64×64 Main10 IDR - /// (`tests/pq-frame.h265`, x265 with explicit VUI). - #[test] - fn software_decode_carries_pq_signaling() { - let au = include_bytes!("../tests/pq-frame.h265"); - let mut dec = SoftwareDecoder::new(ffmpeg::codec::Id::HEVC).expect("hevc decoder"); - let mut got = dec.decode(au).expect("decode"); - if got.is_none() { - // Low-delay decoders may still hold the frame until a flush — send EOF. - dec.decoder.send_eof().ok(); - let mut frame = AvFrame::empty(); - if dec.decoder.receive_frame(&mut frame).is_ok() { - got = Some(dec.convert_rgba(&frame).expect("convert")); - } - } - let f = got.expect("no frame decoded from the PQ fixture"); - assert_eq!( - f.color, - ColorDesc { - primaries: 9, - transfer: 16, - matrix: 9, - full_range: false - } - ); - assert!(f.color.is_pq()); - assert_eq!((f.width, f.height), (64, 64)); - } - - /// Golden colour fixtures: one 256×64 LOSSLESS x265 IDR of 8 fully-saturated colour bars per - /// signaling variant (generated offline with ffmpeg/libx265; the RGB→YUV conversion matched - /// to the VUI each fixture declares, so the original RGB is recoverable ±1 code). Decoding - /// through the real CPU path (`SoftwareDecoder` → per-frame `ColorDesc` → swscale with the - /// signaled matrix/range) must reproduce the bars — the end-to-end guard for the - /// signaling-driven CSC across BT.601/709 × limited/full. A hardcoded-709 regression fails - /// the 601 fixture by tens of code points; a range mix-up fails the full-range one. - #[test] - fn software_decode_reproduces_golden_bars() { - const BARS: [(u8, u8, u8); 8] = [ - (255, 255, 255), - (255, 255, 0), - (0, 255, 255), - (0, 255, 0), - (255, 0, 255), - (255, 0, 0), - (0, 0, 255), - (0, 0, 0), - ]; - let fixtures: [(&str, &[u8], ColorDesc); 3] = [ - ( - "601-limited", - include_bytes!("../tests/bars-601-limited.h265"), - ColorDesc { - primaries: 1, - transfer: 1, - matrix: 5, // BT.470BG — what a Linux host's RGB-input NVENC signals - full_range: false, - }, - ), - ( - "709-limited", - include_bytes!("../tests/bars-709-limited.h265"), - ColorDesc { - primaries: 1, - transfer: 1, - matrix: 1, - full_range: false, - }, - ), - ( - "709-full", - include_bytes!("../tests/bars-709-full.h265"), - ColorDesc { - primaries: 1, - transfer: 1, - matrix: 1, - full_range: true, // the PUNKTFUNK_444_FULLRANGE experiment's signaling - }, - ), - ]; - for (name, au, want_color) in fixtures { - let mut dec = SoftwareDecoder::new(ffmpeg::codec::Id::HEVC).expect("hevc decoder"); - let mut got = dec.decode(au).expect("decode"); - if got.is_none() { - dec.decoder.send_eof().ok(); - let mut frame = AvFrame::empty(); - if dec.decoder.receive_frame(&mut frame).is_ok() { - got = Some(dec.convert_rgba(&frame).expect("convert")); - } - } - let f = got.unwrap_or_else(|| panic!("{name}: no frame decoded")); - assert_eq!(f.color, want_color, "{name}: signaling"); - assert_eq!((f.width, f.height), (256, 64), "{name}: dims"); - for (i, (r, g, b)) in BARS.iter().enumerate() { - let (cx, cy) = (i * 32 + 16, 32usize); - let o = cy * f.stride + cx * 4; - let px = &f.rgba[o..o + 3]; - for (got, want) in px.iter().zip([r, g, b]) { - assert!( - got.abs_diff(*want) <= 3, - "{name} bar {i}: got {px:?}, want ({r},{g},{b})" - ); - } - } - } - } } diff --git a/crates/pf-client-core/src/video_color.rs b/crates/pf-client-core/src/video_color.rs new file mode 100644 index 00000000..87be7439 --- /dev/null +++ b/crates/pf-client-core/src/video_color.rs @@ -0,0 +1,210 @@ +//! The stream's per-frame colour signalling (`ColorDesc`) + the Y′CbCr→RGB CSC matrix (`csc_rows`). +#![allow(clippy::unnecessary_cast)] + +use ffmpeg_next as ffmpeg; + +/// The stream's colour signaling, read PER-FRAME from the decoder (HEVC VUI → the +/// `AVFrame` CICP fields). The Windows host switches an HDR desktop to Main10 BT.2020 PQ +/// **in-band** (the Welcome still says SDR — clients are expected to follow the VUI, as +/// the Windows/Apple/Android clients do), so rendering must follow the frames, not the +/// handshake — else PQ content drawn as BT.709 comes out washed out and desaturated. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct ColorDesc { + /// H.273 code points as signaled (2 = unspecified → the renderer picks the SDR default). + pub primaries: u8, + pub transfer: u8, + pub matrix: u8, + pub full_range: bool, +} + +impl ColorDesc { + /// Read the CICP fields off a raw decoded frame. Public: the Windows client's raw-FFI + /// D3D11VA/software decoders build their per-frame `ColorDesc` with it too (same + /// `ffmpeg-next` major, so the `AVFrame` type unifies across the workspace). + /// + /// # Safety + /// `frame` must point to a valid `AVFrame` (alive for the duration of the call). + pub unsafe fn from_raw(frame: *const ffmpeg::ffi::AVFrame) -> ColorDesc { + // SAFETY: caller guarantees a live AVFrame; these are plain enum field reads. + unsafe { + ColorDesc { + primaries: (*frame).color_primaries as u32 as u8, + transfer: (*frame).color_trc as u32 as u8, + matrix: (*frame).colorspace as u32 as u8, + full_range: (*frame).color_range == ffmpeg::ffi::AVColorRange::AVCOL_RANGE_JPEG, + } + } + } + + /// PQ (SMPTE ST.2084) transfer — the HDR10 signal. + pub fn is_pq(&self) -> bool { + self.transfer == 16 + } +} + +/// The Y′CbCr→RGB conversion as three vec4 rows for a shader constant buffer / push-constant +/// block: `rgb[i] = dot(r[i].xyz, yuv) + r[i].w` — bit-depth exact. The ONE coefficient +/// implementation every presenter derives its CSC from (Vulkan push constants, the Windows +/// client's D3D11 constant buffer), so a stream's signaled matrix/range is honored identically +/// everywhere; the Apple client ports this function (and its tests) to Swift. +/// +/// `depth` picks the limited-range code points (8-bit: 16/235/240 over 255; 10-bit: +/// 64/940/960 over 1023 — NOT the same normalized values, the difference is ~half a +/// code). `msb_packed` folds in the P010/X6 packing factor: 10 significant bits live in +/// the MSBs of 16, so a UNORM16 sample reads `code·64/65535` — multiplying by +/// `65535/65472` recovers exact `code/1023`. +pub fn csc_rows(desc: ColorDesc, depth: u8, msb_packed: bool) -> [[f32; 4]; 3] { + // BT.601 (5/6), BT.2020 (9/10); everything else — incl. unspecified — is the host's + // BT.709 SDR default (mirrors the software path's swscale coefficient choice). + let (kr, kb) = match desc.matrix { + 5 | 6 => (0.299, 0.114), + 9 | 10 => (0.2627, 0.0593), + _ => (0.2126, 0.0722), + }; + let kg = 1.0 - kr - kb; + let max = f64::from((1u32 << depth) - 1); // 255 / 1023 + let step = f64::from(1u32 << (depth - 8)); // code points per 8-bit step: 1 / 4 + let pack = if msb_packed { 65535.0 / 65472.0 } else { 1.0 }; + let (sy, oy, sc) = if desc.full_range { + (pack, 0.0f64, pack) + } else { + ( + pack * max / (219.0 * step), + -(16.0 * step) / max, + pack * max / (224.0 * step), + ) + }; + // rgb = M * (yuv + off) = M*yuv + M*off — rows of M with the offset dot folded into + // w. `yuv` is the SAMPLED (packed) value, so the offsets divide by the packing + // factor to land on the same scale. + let off = [oy / pack, -0.5 / pack, -0.5 / pack]; + let m = [ + [sy, 0.0, 2.0 * (1.0 - kr) * sc], + [ + sy, + -2.0 * (1.0 - kb) * kb / kg * sc, + -2.0 * (1.0 - kr) * kr / kg * sc, + ], + [sy, 2.0 * (1.0 - kb) * sc, 0.0], + ]; + core::array::from_fn(|r| { + let w: f64 = (0..3).map(|c| m[r][c] * off[c]).sum(); + [m[r][0] as f32, m[r][1] as f32, m[r][2] as f32, w as f32] + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn desc(matrix: u8, full_range: bool) -> ColorDesc { + ColorDesc { + primaries: 1, + transfer: 1, + matrix, + full_range, + } + } + + fn apply(rows: &[[f32; 4]; 3], yuv: [f32; 3]) -> [f32; 3] { + core::array::from_fn(|r| { + rows[r][0] * yuv[0] + rows[r][1] * yuv[1] + rows[r][2] * yuv[2] + rows[r][3] + }) + } + + /// 10-bit limited MSB-packed (P010/X6): reference white Y=940, black Y=64, neutral + /// chroma 512 — sampled as UNORM16 of `code << 6`. + #[test] + fn bt2020_10bit_limited_white_black() { + let rows = csc_rows(desc(9, false), 10, true); + let s = |code: u32| ((code << 6) as f32) / 65535.0; + let white = apply(&rows, [s(940), s(512), s(512)]); + let black = apply(&rows, [s(64), s(512), s(512)]); + for (w, b) in white.iter().zip(black) { + assert!((w - 1.0).abs() < 0.002, "white {white:?}"); + assert!(b.abs() < 0.002, "black {black:?}"); + } + } + + /// Reference white (Y=235, U=V=128 limited) → RGB 1.0; reference black (Y=16) → 0.0 + /// — the GL presenter's test, in row form. + #[test] + fn bt709_limited_white_black() { + let rows = csc_rows(desc(1, false), 8, false); + let white = apply(&rows, [235.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0]); + let black = apply(&rows, [16.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0]); + for (w, b) in white.iter().zip(black) { + assert!((w - 1.0).abs() < 0.005, "white {white:?}"); + assert!(b.abs() < 0.005, "black {black:?}"); + } + } + + /// Full-range identity points + the 601-vs-709 red excursion (guards the + /// matrix-code dispatch), same as the GL presenter's test. + #[test] + fn full_range_and_red_excursion() { + let rows = csc_rows(desc(5, true), 8, false); + let white = apply(&rows, [1.0, 0.5, 0.5]); + assert!(white.iter().all(|v| (v - 1.0).abs() < 1e-5), "{white:?}"); + let red = apply(&rows, [0.0, 0.5, 1.0]); + assert!((red[0] - 2.0 * (1.0 - 0.299) * 0.5).abs() < 1e-4, "{red:?}"); + let rows709 = csc_rows(desc(1, true), 8, false); + let red709 = apply(&rows709, [0.0, 0.5, 1.0]); + assert!( + (red709[0] - 2.0 * (1.0 - 0.2126) * 0.5).abs() < 1e-4, + "{red709:?}" + ); + assert!((red[0] - red709[0]).abs() > 0.05); + } + + /// The row form must agree with the GL presenter's column-major `yuv_to_rgb` on a + /// grid of inputs — same math, different packing. + #[test] + fn rows_match_the_gl_matrix_form() { + for (matrix, full) in [(1u8, false), (1, true), (5, false), (9, false), (9, true)] { + let d = desc(matrix, full); + let rows = csc_rows(d, 8, false); + // Reimplementation of video_gl::yuv_to_rgb's application for comparison. + let (kr, kb) = match matrix { + 5 | 6 => (0.299f32, 0.114f32), + 9 | 10 => (0.2627, 0.0593), + _ => (0.2126, 0.0722), + }; + let kg = 1.0 - kr - kb; + let (sy, oy, sc) = if full { + (1.0f32, 0.0f32, 1.0f32) + } else { + (255.0 / 219.0, -16.0 / 255.0, 255.0 / 224.0) + }; + let mat = [ + sy, + sy, + sy, + 0.0, + -2.0 * (1.0 - kb) * kb / kg * sc, + 2.0 * (1.0 - kb) * sc, + 2.0 * (1.0 - kr) * sc, + -2.0 * (1.0 - kr) * kr / kg * sc, + 0.0, + ]; + let off = [oy, -0.5, -0.5]; + for yuv in [ + [0.1f32, 0.3, 0.7], + [0.9, 0.5, 0.5], + [0.5, 0.2, 0.8], + [16.0 / 255.0, 0.5, 0.5], + ] { + let v = [yuv[0] + off[0], yuv[1] + off[1], yuv[2] + off[2]]; + let gl: [f32; 3] = + core::array::from_fn(|r| (0..3).map(|c| mat[c * 3 + r] * v[c]).sum()); + let ours = apply(&rows, yuv); + for (a, b) in gl.iter().zip(ours) { + assert!( + (a - b).abs() < 1e-5, + "{matrix}/{full}: gl {gl:?} rows {ours:?}" + ); + } + } + } + } +} diff --git a/crates/pf-client-core/src/video_software.rs b/crates/pf-client-core/src/video_software.rs new file mode 100644 index 00000000..57ff2908 --- /dev/null +++ b/crates/pf-client-core/src/video_software.rs @@ -0,0 +1,264 @@ +//! CPU/libavcodec software decode backend (swscale → RGBA). + +use crate::video::{averr, CpuFrame}; +use crate::video_color::ColorDesc; +use anyhow::{anyhow, Context as _, Result}; +use ffmpeg::format::Pixel; +use ffmpeg::software::scaling; +use ffmpeg::util::frame::Video as AvFrame; +use ffmpeg_next as ffmpeg; +use std::ptr; + +// --- software backend --------------------------------------------------------------- + +pub(crate) struct SoftwareDecoder { + decoder: ffmpeg::decoder::Video, + /// Rebuilt whenever the decoded format/size — or the colour signaling (a mid-stream + /// SDR↔HDR flip) — changes. + sws: Option<(scaling::Context, Pixel, u32, u32, ColorDesc)>, +} + +impl SoftwareDecoder { + pub(crate) fn new(codec_id: ffmpeg::codec::Id) -> Result { + let codec = ffmpeg::decoder::find(codec_id) + .ok_or_else(|| anyhow!("no {codec_id:?} decoder in libavcodec"))?; + let mut ctx = ffmpeg::codec::Context::new_with_codec(codec); + unsafe { + let raw = ctx.as_mut_ptr(); + (*raw).flags |= ffmpeg::ffi::AV_CODEC_FLAG_LOW_DELAY as i32; + // Slice threading adds no frame delay (frame threading adds thread_count-1). + (*raw).thread_type = ffmpeg::ffi::FF_THREAD_SLICE; + (*raw).thread_count = 0; // auto + } + let decoder = ctx.decoder().video().context("open video decoder")?; + Ok(SoftwareDecoder { decoder, sws: None }) + } + + pub(crate) fn decode(&mut self, au: &[u8]) -> Result> { + let packet = ffmpeg::Packet::copy(au); + self.decoder + .send_packet(&packet) + .map_err(|e| anyhow!("send_packet: {e}"))?; + let mut frame = AvFrame::empty(); + let mut out = None; + while self.decoder.receive_frame(&mut frame).is_ok() { + out = Some(self.convert_rgba(&frame)?); + } + Ok(out) + } + + fn convert_rgba(&mut self, frame: &AvFrame) -> Result { + let (fmt, w, h) = (frame.format(), frame.width(), frame.height()); + // SAFETY: `frame.as_ptr()` is the decoder-owned live AVFrame for this call. + let color = unsafe { ColorDesc::from_raw(frame.as_ptr()) }; + let rebuild = !matches!(&self.sws, + Some((_, f, sw, sh, c)) if *f == fmt && *sw == w && *sh == h && *c == color); + if rebuild { + let mut ctx = + scaling::Context::get(fmt, w, h, Pixel::RGBA, w, h, scaling::Flags::POINT) + .context("swscale context")?; + // swscale defaults to BT.601 coefficients — set them from the FRAME's signaling + // (unspecified → BT.709 limited, the host's SDR default; a Windows HDR desktop + // streams BT.2020 in-band). Without this, YUV→RGB decodes with the wrong matrix + // and colours shift. Destination = full-range RGB; the transfer function stays + // baked in (the presenter tags PQ textures so GTK applies the EOTF). + const SWS_CS_ITU709: i32 = 1; + const SWS_CS_ITU601: i32 = 5; + const SWS_CS_BT2020: i32 = 9; + let cs = match color.matrix { + 9 | 10 => SWS_CS_BT2020, + 5 | 6 => SWS_CS_ITU601, + _ => SWS_CS_ITU709, + }; + unsafe { + let coeffs = ffmpeg::ffi::sws_getCoefficients(cs); + ffmpeg::ffi::sws_setColorspaceDetails( + ctx.as_mut_ptr(), + coeffs, // inv_table: source (YUV) coefficients per the VUI + color.full_range as i32, // srcRange: 0 = limited/studio (MPEG) + coeffs, // table: destination coefficients (ignored for RGB output) + 1, // dstRange: 1 = full-range RGB + 0, + 1 << 16, + 1 << 16, // brightness, contrast, saturation (defaults) + ); + } + self.sws = Some((ctx, fmt, w, h, color)); + } + let (sws, ..) = self.sws.as_mut().unwrap(); + // Single-pass conversion: swscale writes straight into the Vec the texture will + // wrap. (The old path scaled into a scratch AVFrame and then copied `data(0)` out + // — a second full-frame pass per frame.) 64-byte row alignment keeps swscale on + // aligned SIMD stores; `GdkMemoryTexture` takes the resulting stride explicitly. + const ALIGN: i32 = 64; + use ffmpeg::ffi; + let dst_fmt = ffi::AVPixelFormat::AV_PIX_FMT_RGBA; + // SAFETY: pure size computation from format/dimensions; no pointers involved. + let size = unsafe { ffi::av_image_get_buffer_size(dst_fmt, w as i32, h as i32, ALIGN) }; + if size < 0 { + return Err(averr("av_image_get_buffer_size", size)); + } + let rgba = vec![0u8; size as usize]; + let mut dst_data: [*mut u8; 4] = [ptr::null_mut(); 4]; + let mut dst_linesize: [i32; 4] = [0; 4]; + // SAFETY: fill_arrays only derives plane pointers/strides into `rgba` (sized by + // av_image_get_buffer_size above, same format/align) — no allocation, no + // ownership transfer; `rgba` outlives the scale below. + let r = unsafe { + ffi::av_image_fill_arrays( + dst_data.as_mut_ptr(), + dst_linesize.as_mut_ptr(), + rgba.as_ptr(), + dst_fmt, + w as i32, + h as i32, + ALIGN, + ) + }; + if r < 0 { + return Err(averr("av_image_fill_arrays", r)); + } + // SAFETY: src pointers/strides belong to the decoder-owned `frame` (alive for the + // call); dst pointers were just filled over `rgba`, and sws_scale writes rows + // [0, h) only — exactly the buffer fill_arrays sized. + let r = unsafe { + ffi::sws_scale( + sws.as_mut_ptr(), + (*frame.as_ptr()).data.as_ptr() as *const *const u8, + (*frame.as_ptr()).linesize.as_ptr(), + 0, + h as i32, + dst_data.as_ptr(), + dst_linesize.as_ptr(), + ) + }; + if r < 0 { + return Err(averr("sws_scale", r)); + } + Ok(CpuFrame { + width: w, + height: h, + stride: dst_linesize[0] as usize, + rgba, + color, + // `is_key()` reads the same intra flag `frame_is_keyframe` derives from pict_type + // for the hardware paths; ffmpeg-next handles the FFmpeg-version binding split. + keyframe: frame.is_key(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The wire → `ColorDesc` plumbing: an HDR10 stream's VUI (BT.2020 primaries, PQ + /// transfer, BT.2020-NCL matrix, limited range) must arrive on the decoded frame — + /// this is what the Windows host emits in-band for an HDR desktop, and mis-rendering + /// it as BT.709 is the washed-out-colors bug. Fixture: one 64×64 Main10 IDR + /// (`tests/pq-frame.h265`, x265 with explicit VUI). + #[test] + fn software_decode_carries_pq_signaling() { + let au = include_bytes!("../tests/pq-frame.h265"); + let mut dec = SoftwareDecoder::new(ffmpeg::codec::Id::HEVC).expect("hevc decoder"); + let mut got = dec.decode(au).expect("decode"); + if got.is_none() { + // Low-delay decoders may still hold the frame until a flush — send EOF. + dec.decoder.send_eof().ok(); + let mut frame = AvFrame::empty(); + if dec.decoder.receive_frame(&mut frame).is_ok() { + got = Some(dec.convert_rgba(&frame).expect("convert")); + } + } + let f = got.expect("no frame decoded from the PQ fixture"); + assert_eq!( + f.color, + ColorDesc { + primaries: 9, + transfer: 16, + matrix: 9, + full_range: false + } + ); + assert!(f.color.is_pq()); + assert_eq!((f.width, f.height), (64, 64)); + } + + /// Golden colour fixtures: one 256×64 LOSSLESS x265 IDR of 8 fully-saturated colour bars per + /// signaling variant (generated offline with ffmpeg/libx265; the RGB→YUV conversion matched + /// to the VUI each fixture declares, so the original RGB is recoverable ±1 code). Decoding + /// through the real CPU path (`SoftwareDecoder` → per-frame `ColorDesc` → swscale with the + /// signaled matrix/range) must reproduce the bars — the end-to-end guard for the + /// signaling-driven CSC across BT.601/709 × limited/full. A hardcoded-709 regression fails + /// the 601 fixture by tens of code points; a range mix-up fails the full-range one. + #[test] + fn software_decode_reproduces_golden_bars() { + const BARS: [(u8, u8, u8); 8] = [ + (255, 255, 255), + (255, 255, 0), + (0, 255, 255), + (0, 255, 0), + (255, 0, 255), + (255, 0, 0), + (0, 0, 255), + (0, 0, 0), + ]; + let fixtures: [(&str, &[u8], ColorDesc); 3] = [ + ( + "601-limited", + include_bytes!("../tests/bars-601-limited.h265"), + ColorDesc { + primaries: 1, + transfer: 1, + matrix: 5, // BT.470BG — what a Linux host's RGB-input NVENC signals + full_range: false, + }, + ), + ( + "709-limited", + include_bytes!("../tests/bars-709-limited.h265"), + ColorDesc { + primaries: 1, + transfer: 1, + matrix: 1, + full_range: false, + }, + ), + ( + "709-full", + include_bytes!("../tests/bars-709-full.h265"), + ColorDesc { + primaries: 1, + transfer: 1, + matrix: 1, + full_range: true, // the PUNKTFUNK_444_FULLRANGE experiment's signaling + }, + ), + ]; + for (name, au, want_color) in fixtures { + let mut dec = SoftwareDecoder::new(ffmpeg::codec::Id::HEVC).expect("hevc decoder"); + let mut got = dec.decode(au).expect("decode"); + if got.is_none() { + dec.decoder.send_eof().ok(); + let mut frame = AvFrame::empty(); + if dec.decoder.receive_frame(&mut frame).is_ok() { + got = Some(dec.convert_rgba(&frame).expect("convert")); + } + } + let f = got.unwrap_or_else(|| panic!("{name}: no frame decoded")); + assert_eq!(f.color, want_color, "{name}: signaling"); + assert_eq!((f.width, f.height), (256, 64), "{name}: dims"); + for (i, (r, g, b)) in BARS.iter().enumerate() { + let (cx, cy) = (i * 32 + 16, 32usize); + let o = cy * f.stride + cx * 4; + let px = &f.rgba[o..o + 3]; + for (got, want) in px.iter().zip([r, g, b]) { + assert!( + got.abs_diff(*want) <= 3, + "{name} bar {i}: got {px:?}, want ({r},{g},{b})" + ); + } + } + } + } +} diff --git a/crates/pf-client-core/src/video_vaapi.rs b/crates/pf-client-core/src/video_vaapi.rs new file mode 100644 index 00000000..0c5b48b7 --- /dev/null +++ b/crates/pf-client-core/src/video_vaapi.rs @@ -0,0 +1,243 @@ +//! VAAPI (libavcodec hwaccel) decode backend → DRM-PRIME dmabuf for the presenter. Linux-only. + +use crate::video::{ + averr, drm_fourcc_for, frame_is_keyframe, DmabufFrame, DmabufPlane, DrmFrameGuard, + AVERROR_EAGAIN, +}; +use crate::video_color::ColorDesc; +use anyhow::{anyhow, bail, Result}; +use ffmpeg_next as ffmpeg; +use std::ptr; + +/// libavcodec offers the formats it can decode into; pick the VAAPI hw surface. Falling +/// back to the first (software) entry would silently decode on the CPU *and* break our +/// dmabuf mapping — return NONE instead so the error surfaces and the session demotes +/// to the software backend explicitly. +#[cfg(target_os = "linux")] +unsafe extern "C" fn pick_vaapi( + _ctx: *mut ffmpeg::ffi::AVCodecContext, + mut list: *const ffmpeg::ffi::AVPixelFormat, +) -> ffmpeg::ffi::AVPixelFormat { + unsafe { + while *list != ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE { + if *list == ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI { + return ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI; + } + list = list.add(1); + } + } + ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE +} + +#[cfg(target_os = "linux")] +pub(crate) struct VaapiDecoder { + ctx: *mut ffmpeg::ffi::AVCodecContext, + hw_device: *mut ffmpeg::ffi::AVBufferRef, + packet: *mut ffmpeg::ffi::AVPacket, + frame: *mut ffmpeg::ffi::AVFrame, +} + +// Single-owner pointers, only touched from the session pump thread. +#[cfg(target_os = "linux")] +unsafe impl Send for VaapiDecoder {} + +#[cfg(target_os = "linux")] +impl VaapiDecoder { + pub(crate) fn new(codec_id: ffmpeg::codec::Id) -> Result { + use ffmpeg::ffi; + unsafe { + let mut hw_device: *mut ffi::AVBufferRef = ptr::null_mut(); + let r = ffi::av_hwdevice_ctx_create( + &mut hw_device, + ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI, + ptr::null(), + ptr::null_mut(), + 0, + ); + if r < 0 { + bail!("no VAAPI device ({})", ffmpeg::Error::from(r)); + } + // The negotiated codec's decoder id (av_codec_id maps 1:1 from ffmpeg::codec::Id). + let codec = ffi::avcodec_find_decoder(codec_id.into()); + if codec.is_null() { + ffi::av_buffer_unref(&mut hw_device); + bail!("no {codec_id:?} decoder"); + } + let ctx = ffi::avcodec_alloc_context3(codec); + (*ctx).hw_device_ctx = ffi::av_buffer_ref(hw_device); + (*ctx).get_format = Some(pick_vaapi); + (*ctx).flags |= ffi::AV_CODEC_FLAG_LOW_DELAY as i32; + (*ctx).thread_count = 1; // hwaccel: threads only add latency + + // The presenter holds mapped surfaces PAST receive_frame (the paintable's + // current texture + the newest frame in flight each pin one until GDK's + // release func) — surfaces libavcodec doesn't know are missing from its + // fixed-size VAAPI pool. Without headroom the decoder can recycle a surface + // the renderer is still sampling (intermittent block corruption) or fail + // allocation under scheduling jitter. + (*ctx).extra_hw_frames = 4; + let r = ffi::avcodec_open2(ctx, codec, ptr::null_mut()); + if r < 0 { + let mut ctx = ctx; + ffi::avcodec_free_context(&mut ctx); + let mut hw_device = hw_device; + ffi::av_buffer_unref(&mut hw_device); + bail!("avcodec_open2: {}", ffmpeg::Error::from(r)); + } + Ok(VaapiDecoder { + ctx, + hw_device, + packet: ffi::av_packet_alloc(), + frame: ffi::av_frame_alloc(), + }) + } + } + + pub(crate) fn decode(&mut self, au: &[u8]) -> Result> { + use ffmpeg::ffi; + unsafe { + let r = ffi::av_new_packet(self.packet, au.len() as i32); + if r < 0 { + return Err(averr("av_new_packet", r)); + } + ptr::copy_nonoverlapping(au.as_ptr(), (*self.packet).data, au.len()); + let r = ffi::avcodec_send_packet(self.ctx, self.packet); + ffi::av_packet_unref(self.packet); + if r < 0 { + return Err(averr("send_packet", r)); + } + let mut out = None; + loop { + let r = ffi::avcodec_receive_frame(self.ctx, self.frame); + if r == AVERROR_EAGAIN { + break; + } + if r < 0 { + return Err(averr("receive_frame", r)); + } + out = Some(self.map_dmabuf()?); // newest wins; older guards drop here + ffi::av_frame_unref(self.frame); + } + Ok(out) + } + } + + /// Map the VAAPI surface to DRM PRIME (zero copy) and lift the descriptor into a + /// `DmabufFrame`. The mapped frame keeps the surface alive via its buffer refs. + /// + /// FFmpeg's VAAPI export uses `VA_EXPORT_SURFACE_SEPARATE_LAYERS`, so an NV12 surface + /// comes back as TWO layers (`R8` luma + `GR88` chroma), each one plane — NOT a single + /// `NV12` layer. The previous code took `layers[0]` only: GTK then saw an `R8` + /// single-plane texture with the chroma dropped, painting the screen green. The fix: + /// derive the COMBINED fourcc from the decoder's software pixel format (NV12 → + /// `DRM_FORMAT_NV12`) and flatten every plane across every layer in order (Y then UV). + unsafe fn map_dmabuf(&mut self) -> Result { + use ffmpeg::ffi; + unsafe { + if (*self.frame).format != ffi::AVPixelFormat::AV_PIX_FMT_VAAPI as i32 { + bail!("decoder returned a software frame (no VAAPI surface)"); + } + // The real pixel layout lives on the hardware frames context, not the + // DRM-PRIME layer formats (those are the per-plane R8/GR88 component formats). + let sw_format = { + let hwfc = (*self.frame).hw_frames_ctx; + if hwfc.is_null() { + bail!("VAAPI frame without a hardware frames context"); + } + (*((*hwfc).data as *const ffi::AVHWFramesContext)).sw_format + }; + let fourcc = drm_fourcc_for(sw_format) + .ok_or_else(|| anyhow!("unsupported VAAPI output format {sw_format:?}"))?; + + let drm = ffi::av_frame_alloc(); + (*drm).format = ffi::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32; + let r = ffi::av_hwframe_map(drm, self.frame, ffi::AV_HWFRAME_MAP_READ as i32); + if r < 0 { + let mut drm = drm; + ffi::av_frame_free(&mut drm); + return Err(averr("av_hwframe_map", r)); + } + let desc = (*drm).data[0] as *const ffi::AVDRMFrameDescriptor; + let guard = DrmFrameGuard(drm); + let d = &*desc; + if d.nb_layers < 1 || d.nb_objects < 1 { + bail!("DRM descriptor without layers/objects"); + } + + // Flatten planes across ALL layers, in declared order — the combined fourcc's + // plane order (Y, then UV for NV12) matches the layer order FFmpeg emits. + let mut planes = Vec::new(); + for layer in &d.layers[..d.nb_layers as usize] { + for p in &layer.planes[..layer.nb_planes as usize] { + let obj = &d.objects[p.object_index as usize]; + planes.push(DmabufPlane { + fd: obj.fd, + offset: p.offset as u32, + stride: p.pitch as u32, + }); + } + } + + // The whole surface shares one tiling modifier (one BO on radeonsi); GTK takes + // a single modifier for the texture. + let modifier = d.objects[0].format_modifier; + + log_descriptor_once(d, sw_format, fourcc, modifier); + + Ok(DmabufFrame { + width: (*self.frame).width as u32, + height: (*self.frame).height as u32, + fourcc, + modifier, + planes, + // SAFETY: `self.frame` is the live decoded AVFrame (unref'd only after + // this returns); plain CICP field reads. + color: ColorDesc::from_raw(self.frame), + keyframe: frame_is_keyframe(self.frame), + guard, + }) + } + } +} + +/// One-time dump of the DRM descriptor layout (objects, layers, planes, modifier) — so a +/// new client/driver combination's real layout is visible in the logs without a debugger. +#[cfg(target_os = "linux")] +fn log_descriptor_once( + d: &ffmpeg_next::ffi::AVDRMFrameDescriptor, + sw: ffmpeg_next::ffi::AVPixelFormat, + fourcc: u32, + modifier: u64, +) { + use std::sync::atomic::{AtomicBool, Ordering}; + static ONCE: AtomicBool = AtomicBool::new(true); + if !ONCE.swap(false, Ordering::Relaxed) { + return; + } + let layers: Vec<(u32, i32)> = d.layers[..d.nb_layers.max(0) as usize] + .iter() + .map(|l| (l.format, l.nb_planes)) + .collect(); + tracing::info!( + sw_format = ?sw, + chosen_fourcc = format_args!("{:#010x}", fourcc), + nb_objects = d.nb_objects, + nb_layers = d.nb_layers, + ?layers, + modifier = format_args!("{:#018x}", modifier), + "VAAPI dmabuf descriptor layout (first frame)" + ); +} + +#[cfg(target_os = "linux")] +impl Drop for VaapiDecoder { + fn drop(&mut self) { + use ffmpeg::ffi; + unsafe { + ffi::av_packet_free(&mut self.packet); + ffi::av_frame_free(&mut self.frame); + ffi::avcodec_free_context(&mut self.ctx); + ffi::av_buffer_unref(&mut self.hw_device); + } + } +} diff --git a/crates/pf-client-core/src/video_vulkan.rs b/crates/pf-client-core/src/video_vulkan.rs new file mode 100644 index 00000000..a89c8ec2 --- /dev/null +++ b/crates/pf-client-core/src/video_vulkan.rs @@ -0,0 +1,419 @@ +//! FFmpeg Vulkan Video decode over the presenter's own VkDevice (zero-copy VkImage). +#![allow(clippy::unnecessary_cast)] + +use crate::video::{ + averr, frame_is_keyframe, DrmFrameGuard, QueueLock, VkVideoFrame, VulkanDecodeDevice, + AVERROR_EAGAIN, +}; +use crate::video_color::ColorDesc; +use anyhow::{bail, Result}; +use ffmpeg_next as ffmpeg; +use std::ptr; + +// --- Vulkan Video backend ------------------------------------------------------------- + +/// FFmpeg's Vulkan Video decoder over the PRESENTER's device: the hwdevice context is +/// built from [`VulkanDecodeDevice`]'s handles (not `av_hwdevice_ctx_create`, which +/// would make FFmpeg create its own device the presenter can't sample from). Output +/// frames are `AVVkFrame`s whose VkImage the presenter feeds straight to its CSC pass. +pub(crate) struct VulkanDecoder { + ctx: *mut ffmpeg::ffi::AVCodecContext, + hw_device: *mut ffmpeg::ffi::AVBufferRef, + packet: *mut ffmpeg::ffi::AVPacket, + frame: *mut ffmpeg::ffi::AVFrame, + /// `vkWaitSemaphores` on the shared device — the decode-complete measurement + /// (resolved through the same get_proc_addr chain FFmpeg uses). + wait_semaphores: pf_ffvk::PFN_vkWaitSemaphores, + vk_device: pf_ffvk::VkDevice, + /// Storage `AVVulkanDeviceContext` points into (extension string arrays + the + /// feature chain) — FFmpeg reads the extension lists past init (frames-context + /// setup keys code paths off them), so this lives exactly as long as `hw_device`. + _ctx_storage: Box, +} + +// Single-owner pointers, only touched from the session pump thread. +unsafe impl Send for VulkanDecoder {} + +struct VkCtxStorage { + _inst: Vec, + inst_ptrs: Vec<*const std::os::raw::c_char>, + _dev: Vec, + dev_ptrs: Vec<*const std::os::raw::c_char>, + f11: pf_ffvk::VkPhysicalDeviceVulkan11Features, + f12: pf_ffvk::VkPhysicalDeviceVulkan12Features, + f13: pf_ffvk::VkPhysicalDeviceVulkan13Features, + /// Keeps the shared queue lock alive for `AVHWDeviceContext.user_opaque` — the + /// `lock_queue`/`unlock_queue` trampolines below dereference it for as long as the + /// hw device context can fire them. + _queue_lock: std::sync::Arc, +} + +/// FFmpeg `AVVulkanDeviceContext.lock_queue` trampoline: take the device's shared +/// [`QueueLock`] (stashed in `AVHWDeviceContext.user_opaque`; owned by +/// [`VkCtxStorage`], which outlives the context). Replaces FFmpeg's internal default, +/// which only serializes FFmpeg against itself — the presenter submits to the same +/// graphics queue from another thread and holds this same lock around its calls. +unsafe extern "C" fn ffvk_lock_queue( + ctx: *mut pf_ffvk::AVHWDeviceContext, + _queue_family: u32, + _index: u32, +) { + let dev = ctx as *mut ffmpeg::ffi::AVHWDeviceContext; + let lock = (*dev).user_opaque as *const QueueLock; + (*lock).lock(); +} + +/// The matching `unlock_queue` trampoline — see [`ffvk_lock_queue`]. +unsafe extern "C" fn ffvk_unlock_queue( + ctx: *mut pf_ffvk::AVHWDeviceContext, + _queue_family: u32, + _index: u32, +) { + let dev = ctx as *mut ffmpeg::ffi::AVHWDeviceContext; + let lock = (*dev).user_opaque as *const QueueLock; + (*lock).unlock(); +} + +impl VulkanDecoder { + pub(crate) fn new( + codec_id: ffmpeg::codec::Id, + vk: &VulkanDecodeDevice, + ) -> Result { + use ffmpeg::ffi; + unsafe { + let mut hw_device = + ffi::av_hwdevice_ctx_alloc(ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VULKAN); + if hw_device.is_null() { + bail!("av_hwdevice_ctx_alloc(VULKAN) failed (FFmpeg built without Vulkan?)"); + } + let devctx = (*hw_device).data as *mut ffi::AVHWDeviceContext; + let hwctx = (*devctx).hwctx as *mut pf_ffvk::AVVulkanDeviceContext; + + // Pinned storage for everything the context points into. + let mut store = Box::new(VkCtxStorage { + _inst: vk.instance_extensions.clone(), + inst_ptrs: Vec::new(), + _dev: vk.device_extensions.clone(), + dev_ptrs: Vec::new(), + f11: std::mem::zeroed(), + f12: std::mem::zeroed(), + f13: std::mem::zeroed(), + _queue_lock: vk.queue_lock.clone(), + }); + store.inst_ptrs = store._inst.iter().map(|c| c.as_ptr()).collect(); + store.dev_ptrs = store._dev.iter().map(|c| c.as_ptr()).collect(); + // The features enabled at device creation, as the 1.1/1.2/1.3 chain FFmpeg + // walks to learn what it may use (sType values are vulkan.h constants). + store.f11.sType = + pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES; + store.f11.samplerYcbcrConversion = vk.f_sampler_ycbcr as u32; + store.f12.sType = + pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; + store.f12.timelineSemaphore = vk.f_timeline_semaphore as u32; + store.f13.sType = + pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES; + store.f13.synchronization2 = vk.f_synchronization2 as u32; + store.f11.pNext = &mut store.f12 as *mut _ as *mut std::ffi::c_void; + store.f12.pNext = &mut store.f13 as *mut _ as *mut std::ffi::c_void; + + (*hwctx).get_proc_addr = std::mem::transmute::( + vk.get_instance_proc_addr, + ); + (*hwctx).inst = vk.instance as pf_ffvk::VkInstance; + (*hwctx).phys_dev = vk.physical_device as pf_ffvk::VkPhysicalDevice; + (*hwctx).act_dev = vk.device as pf_ffvk::VkDevice; + (*hwctx).device_features.sType = + pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + (*hwctx).device_features.pNext = &mut store.f11 as *mut _ as *mut std::ffi::c_void; + (*hwctx).enabled_inst_extensions = store.inst_ptrs.as_ptr(); + (*hwctx).nb_enabled_inst_extensions = store.inst_ptrs.len() as i32; + (*hwctx).enabled_dev_extensions = store.dev_ptrs.as_ptr(); + (*hwctx).nb_enabled_dev_extensions = store.dev_ptrs.len() as i32; + + // Queue map: the deprecated per-role indices (tx/comp are "Required") plus + // the qf[] list, which per the header must also carry every family named + // above. One merged entry when decode shares the graphics family. + let g = vk.graphics_qf as i32; + let d = vk.decode_qf as i32; + (*hwctx).queue_family_index = g; + (*hwctx).nb_graphics_queues = 1; + (*hwctx).queue_family_tx_index = g; + (*hwctx).nb_tx_queues = 1; + (*hwctx).queue_family_comp_index = g; + (*hwctx).nb_comp_queues = 1; + (*hwctx).queue_family_encode_index = -1; + (*hwctx).nb_encode_queues = 0; + (*hwctx).queue_family_decode_index = d; + (*hwctx).nb_decode_queues = 1; + const VIDEO_DECODE_BIT: u32 = 0x20; // VK_QUEUE_VIDEO_DECODE_BIT_KHR + // `flags`/`video_caps` are bindgen enum types: i32 under MSVC, u32 under + // Linux clang — the `as _` casts absorb the difference. + if g == d { + (*hwctx).qf[0] = pf_ffvk::AVVulkanDeviceQueueFamily { + idx: g, + num: 1, + flags: (vk.graphics_queue_flags | VIDEO_DECODE_BIT) as _, + video_caps: vk.decode_video_caps as _, + }; + (*hwctx).nb_qf = 1; + } else { + (*hwctx).qf[0] = pf_ffvk::AVVulkanDeviceQueueFamily { + idx: g, + num: 1, + flags: vk.graphics_queue_flags as _, + video_caps: 0, + }; + (*hwctx).qf[1] = pf_ffvk::AVVulkanDeviceQueueFamily { + idx: d, + num: 1, + flags: VIDEO_DECODE_BIT as _, + video_caps: vk.decode_video_caps as _, + }; + (*hwctx).nb_qf = 2; + } + + // Shared-queue external sync (see [`QueueLock`]): FFmpeg must take the + // same lock the presenter holds around its own submits/presents — set + // BEFORE init so FFmpeg never installs its internal defaults (which only + // serialize FFmpeg against itself; the cross-thread race with the + // presenter's queue was an intermittent VK_ERROR_DEVICE_LOST). + (*devctx).user_opaque = + std::sync::Arc::as_ptr(&store._queue_lock) as *mut std::ffi::c_void; + (*hwctx).lock_queue = Some(ffvk_lock_queue); + (*hwctx).unlock_queue = Some(ffvk_unlock_queue); + + let r = ffi::av_hwdevice_ctx_init(hw_device); + if r < 0 { + ffi::av_buffer_unref(&mut hw_device); + return Err(averr("av_hwdevice_ctx_init(VULKAN)", r)); + } + + // vkWaitSemaphores for the pump's decode-complete stat: loader → + // vkGetDeviceProcAddr → device fn (core 1.2, guaranteed by our gate). + let gipa = (*hwctx) + .get_proc_addr + .expect("get_proc_addr was just set above"); + let gdpa: pf_ffvk::PFN_vkGetDeviceProcAddr = + std::mem::transmute(gipa((*hwctx).inst, c"vkGetDeviceProcAddr".as_ptr())); + let wait_semaphores: pf_ffvk::PFN_vkWaitSemaphores = std::mem::transmute(gdpa + .expect("vkGetDeviceProcAddr resolvable")( + (*hwctx).act_dev, + c"vkWaitSemaphores".as_ptr(), + )); + if wait_semaphores.is_none() { + ffi::av_buffer_unref(&mut hw_device); + bail!("vkWaitSemaphores unresolvable on this device"); + } + let vk_device = (*hwctx).act_dev; + + let codec = ffi::avcodec_find_decoder(codec_id.into()); + if codec.is_null() { + ffi::av_buffer_unref(&mut hw_device); + bail!("no {codec_id:?} decoder"); + } + let ctx = ffi::avcodec_alloc_context3(codec); + (*ctx).hw_device_ctx = ffi::av_buffer_ref(hw_device); + (*ctx).get_format = Some(pick_vulkan); + (*ctx).flags |= ffi::AV_CODEC_FLAG_LOW_DELAY as i32; + (*ctx).thread_count = 1; // hwaccel: threads only add latency + // Same pool headroom rationale as VAAPI: the presenter pins the on-screen + // frame + the newest in flight past receive_frame. + (*ctx).extra_hw_frames = 4; + let r = ffi::avcodec_open2(ctx, codec, ptr::null_mut()); + if r < 0 { + let mut ctx = ctx; + ffi::avcodec_free_context(&mut ctx); + ffi::av_buffer_unref(&mut hw_device); + return Err(averr("avcodec_open2 (vulkan)", r)); + } + Ok(VulkanDecoder { + ctx, + hw_device, + packet: ffi::av_packet_alloc(), + frame: ffi::av_frame_alloc(), + wait_semaphores, + vk_device, + _ctx_storage: store, + }) + } + } + + pub(crate) fn decode(&mut self, au: &[u8]) -> Result> { + use ffmpeg::ffi; + unsafe { + let r = ffi::av_new_packet(self.packet, au.len() as i32); + if r < 0 { + return Err(averr("av_new_packet", r)); + } + ptr::copy_nonoverlapping(au.as_ptr(), (*self.packet).data, au.len()); + let r = ffi::avcodec_send_packet(self.ctx, self.packet); + ffi::av_packet_unref(self.packet); + if r < 0 { + return Err(averr("send_packet", r)); + } + let mut out = None; + loop { + let r = ffi::avcodec_receive_frame(self.ctx, self.frame); + if r == AVERROR_EAGAIN { + break; + } + if r < 0 { + return Err(averr("receive_frame", r)); + } + out = Some(self.extract()?); // newest wins; older guards drop here + ffi::av_frame_unref(self.frame); + } + Ok(out) + } + } + + /// Block until the timeline semaphore reaches `value` (GPU decode complete) or the + /// timeout passes. Pure measurement — the presenter's own GPU wait is what gates + /// sampling, so a timeout here only degrades the stat, never the picture. + pub(crate) fn wait_timeline(&self, sem: u64, value: u64, timeout_ns: u64) -> bool { + let sems = [sem as pf_ffvk::VkSemaphore]; + let values = [value]; + let info = pf_ffvk::VkSemaphoreWaitInfo { + sType: pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, + pNext: std::ptr::null(), + flags: 0, + semaphoreCount: 1, + pSemaphores: sems.as_ptr(), + pValues: values.as_ptr(), + }; + // SAFETY: resolved from this device at init; handles outlive the decoder. + let r = unsafe { + self.wait_semaphores.expect("checked at init")(self.vk_device, &info, timeout_ns) + }; + r == 0 // VK_SUCCESS (VK_TIMEOUT = 2) + } + + /// Lift the decoded `AVVkFrame` into a [`VkVideoFrame`]: clone the AVFrame (the + /// guard — keeps the image + frames context alive through present) and ship the + /// POINTERS; the presenter reads the live sync state under the frames-context lock + /// at its own submit time. + unsafe fn extract(&mut self) -> Result { + use ffmpeg::ffi; + unsafe { + if (*self.frame).format != ffi::AVPixelFormat::AV_PIX_FMT_VULKAN as i32 { + bail!("decoder returned a non-Vulkan frame"); + } + let hwfc_ref = (*self.frame).hw_frames_ctx; + if hwfc_ref.is_null() { + bail!("Vulkan frame without a hardware frames context"); + } + let fc = (*hwfc_ref).data as *mut ffi::AVHWFramesContext; + let sw = (*fc).sw_format; + if sw != ffi::AVPixelFormat::AV_PIX_FMT_NV12 + && sw != ffi::AVPixelFormat::AV_PIX_FMT_P010LE + { + bail!("Vulkan decode output {sw:?} unsupported (NV12/P010 only)"); + } + let vkfc = (*fc).hwctx as *const pf_ffvk::AVVulkanFramesContext; + let vk_format = (*vkfc).format[0] as i32; + let lock_frame = (*vkfc).lock_frame.map_or(0, |f| f as usize); + let unlock_frame = (*vkfc).unlock_frame.map_or(0, |f| f as usize); + if lock_frame == 0 || unlock_frame == 0 { + bail!("Vulkan frames context without lock functions"); + } + + let clone = ffi::av_frame_clone(self.frame); + if clone.is_null() { + bail!("av_frame_clone failed"); + } + let vkf = (*clone).data[0] as *mut pf_ffvk::AVVkFrame; + // v1 handles the (default) single multiplanar image; a disjoint/multi-image + // pool would need per-plane images — bail so the session demotes cleanly. + if !(*vkf).img[1].is_null() { + let mut clone = clone; + ffi::av_frame_free(&mut clone); + bail!("multi-image Vulkan frames unsupported (disjoint pool)"); + } + // Safe without the frames lock: the handle is creation-constant and + // sem_value was last written by the decode submission on THIS thread. + let timeline_sem = (*vkf).sem[0] as u64; + let decode_done_value = (*vkf).sem_value[0]; + Ok(VkVideoFrame { + vkframe: vkf as usize, + frames_ctx: fc as usize, + lock_frame, + unlock_frame, + vk_format, + timeline_sem, + decode_done_value, + width: (*self.frame).width as u32, + height: (*self.frame).height as u32, + color: ColorDesc::from_raw(self.frame), + keyframe: frame_is_keyframe(self.frame), + guard: DrmFrameGuard(clone), + }) + } + } +} + +impl Drop for VulkanDecoder { + fn drop(&mut self) { + use ffmpeg::ffi; + unsafe { + ffi::av_packet_free(&mut self.packet); + ffi::av_frame_free(&mut self.frame); + ffi::avcodec_free_context(&mut self.ctx); + ffi::av_buffer_unref(&mut self.hw_device); + } + } +} + +/// libavcodec offers the formats it can decode into; pick the Vulkan hw surface and +/// hand the decoder OUR frames context — the default one lacks +/// `VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT`, without which the presenter can't create the +/// per-plane views its CSC pass samples. Returning NONE (over the software entry) keeps +/// failures loud: the session demotes explicitly instead of silently CPU-decoding. +unsafe extern "C" fn pick_vulkan( + ctx: *mut ffmpeg::ffi::AVCodecContext, + mut list: *const ffmpeg::ffi::AVPixelFormat, +) -> ffmpeg::ffi::AVPixelFormat { + use ffmpeg::ffi; + unsafe { + let mut offered = false; + while *list != ffi::AVPixelFormat::AV_PIX_FMT_NONE { + if *list == ffi::AVPixelFormat::AV_PIX_FMT_VULKAN { + offered = true; + break; + } + list = list.add(1); + } + if !offered { + return ffi::AVPixelFormat::AV_PIX_FMT_NONE; + } + let mut fr: *mut ffi::AVBufferRef = ptr::null_mut(); + let r = ffi::avcodec_get_hw_frames_parameters( + ctx, + (*ctx).hw_device_ctx, + ffi::AVPixelFormat::AV_PIX_FMT_VULKAN, + &mut fr, + ); + if r < 0 || fr.is_null() { + tracing::warn!(code = r, "avcodec_get_hw_frames_parameters(VULKAN) failed"); + return ffi::AVPixelFormat::AV_PIX_FMT_NONE; + } + let fc = (*fr).data as *mut ffi::AVHWFramesContext; + let vkfc = (*fc).hwctx as *mut pf_ffvk::AVVulkanFramesContext; + // MUTABLE_FORMAT: per-plane views (spec requirement); ALIAS is FFmpeg's default. + // (`as _`: the FlagBits constants are i32 under MSVC, the img_flags field u32.) + (*vkfc).img_flags = (pf_ffvk::VkImageCreateFlagBits_VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT + | pf_ffvk::VkImageCreateFlagBits_VK_IMAGE_CREATE_ALIAS_BIT) + as _; + let r = ffi::av_hwframe_ctx_init(fr); + if r < 0 { + tracing::warn!(code = r, "av_hwframe_ctx_init(VULKAN) failed"); + let mut fr = fr; + ffi::av_buffer_unref(&mut fr); + return ffi::AVPixelFormat::AV_PIX_FMT_NONE; + } + if !(*ctx).hw_frames_ctx.is_null() { + ffi::av_buffer_unref(&mut (*ctx).hw_frames_ctx); + } + (*ctx).hw_frames_ctx = fr; // the codec owns our ref now + ffi::AVPixelFormat::AV_PIX_FMT_VULKAN + } +}