From c78ddc40cbc5fbfaed30df2582ca493ad4ae8822 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Tue, 7 Jul 2026 21:46:34 +0200 Subject: [PATCH] feat(video): Vulkan Video decode on the presenter's device (NVIDIA hw decode) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FFmpeg's Vulkan Video decoder now runs on the PRESENTER's own VkDevice — the decoded VkImage feeds the existing CICP CSC pass directly: zero copy, no interop, and NVIDIA gets hardware decode for the first time (its VAAPI is unusable by design). One decode architecture for every vendor going forward; VAAPI-dmabuf and software remain the fallbacks (auto: vulkan → vaapi → software; PUNKTFUNK_DECODER=vulkan pins it). Presenter: instance 1.3; probes VK_KHR_video_queue/decode_queue + codec extensions, a VIDEO_DECODE queue family (+ its codec caps via QueueFamilyVideoPropertiesKHR), and the samplerYcbcrConversion/ timelineSemaphore/synchronization2 features — all enabled at device creation when present, exported as a VulkanDecodeDevice handle bundle. Decoder: AVVulkanDeviceContext built over those handles via pf-ffvk (features chain, extension lists, deprecated queue indices + the qf[] map); get_format supplies OUR frames context with MUTABLE_FORMAT so the presenter's per-plane views are legal; output is DecodedImage::VkFrame carrying AVVkFrame/frames-ctx pointers plus the lock fns. Present: R8+R8G8 plane views over the multiplanar image, the live sync state read under the AVVulkanFramesContext lock, a timeline-semaphore wait(sem_value)/signal(sem_value+1) folded into the submit, layout/ queue-family/sem_value written back per FFmpeg's contract, and the frame guard parked in the retire queue until the fence. CSC pass + video framebuffer are now unconditional (NVIDIA has no dmabuf-import path). Verified on the RTX 5070 Ti: device creates with decode_qf=3, caps=DECODE_H264|H265|AV1|VP9; swapchain unaffected. Live stream validation next. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 2 + clients/linux-session/src/browse.rs | 7 +- clients/linux-session/src/main.rs | 30 +- crates/pf-client-core/Cargo.toml | 2 + crates/pf-client-core/src/session.rs | 7 +- crates/pf-client-core/src/video.rs | 495 ++++++++++++++++++-- crates/pf-ffvk/build.rs | 7 + crates/pf-presenter/Cargo.toml | 2 + crates/pf-presenter/src/run.rs | 81 +++- crates/pf-presenter/src/vk.rs | 645 ++++++++++++++++++++++----- 10 files changed, 1087 insertions(+), 191 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1344a6bb..455f59db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2774,6 +2774,7 @@ dependencies = [ "ffmpeg-next", "mdns-sd", "opus", + "pf-ffvk", "pipewire", "punktfunk-core", "rustls", @@ -2821,6 +2822,7 @@ dependencies = [ "ash", "async-channel", "pf-client-core", + "pf-ffvk", "punktfunk-core", "sdl3", "tracing", diff --git a/clients/linux-session/src/browse.rs b/clients/linux-session/src/browse.rs index ac3efd27..af3c3a85 100644 --- a/clients/linux-session/src/browse.rs +++ b/clients/linux-session/src/browse.rs @@ -66,7 +66,7 @@ pub fn run(target: &str) -> u8 { overlay: Some(Box::new(overlay)), }; - let result = pf_presenter::run_browse(opts, |action, gamepad, native, force_software| { + let result = pf_presenter::run_browse(opts, |action, gamepad, native, force_software, vulkan| { match action { OverlayAction::Launch { id, title } => { // The carousel only renders for a paired host, so the pin exists; the @@ -76,7 +76,7 @@ pub fn run(target: &str) -> u8 { return ActionOutcome::Handled; }; tracing::info!(%id, %title, "launching from the library"); - ActionOutcome::Start(session_params( + ActionOutcome::Start(Box::new(session_params( &settings, addr.clone(), port, @@ -86,7 +86,8 @@ pub fn run(target: &str) -> u8 { gamepad, native, force_software, - )) + vulkan, + ))) } OverlayAction::Retry => { spawn_fetch(shared.clone(), addr.clone(), mgmt, identity.clone(), pin); diff --git a/clients/linux-session/src/main.rs b/clients/linux-session/src/main.rs index 44327545..d700503e 100644 --- a/clients/linux-session/src/main.rs +++ b/clients/linux-session/src/main.rs @@ -89,6 +89,7 @@ mod session_main { gamepad: &GamepadService, native: Mode, force_software: Arc, + vulkan: Option, ) -> SessionParams { let mode = Mode { width: if settings.width == 0 { @@ -126,6 +127,7 @@ mod session_main { // PUNKTFUNK_DECODER still overrides inside the decoder for bisects. decoder: settings.decoder.clone(), launch, + vulkan, pin: Some(pin), identity, connect_timeout: connect_timeout(), @@ -259,19 +261,21 @@ mod session_main { overlay: None, }; - let outcome = pf_presenter::run_session(opts, move |gamepad, native, force_software| { - session_params( - &settings, - addr, - port, - pin, - identity, - launch, - gamepad, - native, - force_software, - ) - }); + let outcome = + pf_presenter::run_session(opts, move |gamepad, native, force_software, vulkan| { + session_params( + &settings, + addr, + port, + pin, + identity, + launch, + gamepad, + native, + force_software, + vulkan, + ) + }); match outcome { Ok(pf_presenter::Outcome::Ended(None)) => 0, diff --git a/crates/pf-client-core/Cargo.toml b/crates/pf-client-core/Cargo.toml index d9044e2a..312f3395 100644 --- a/crates/pf-client-core/Cargo.toml +++ b/crates/pf-client-core/Cargo.toml @@ -13,6 +13,8 @@ repository.workspace = true # emptiness. `wol` is pure std and stays cross-platform, matching the old main.rs. [target.'cfg(target_os = "linux")'.dependencies] punktfunk-core = { path = "../punktfunk-core", features = ["quic"] } +# FFmpeg's Vulkan hwcontext surface (Vulkan Video decode on the presenter's device). +pf-ffvk = { path = "../pf-ffvk" } async-channel = "2" # Video decode (same FFmpeg pin as the host) and audio. diff --git a/crates/pf-client-core/src/session.rs b/crates/pf-client-core/src/session.rs index aa2436c0..5807ff69 100644 --- a/crates/pf-client-core/src/session.rs +++ b/crates/pf-client-core/src/session.rs @@ -35,6 +35,9 @@ pub struct SessionParams { /// Library id for the host to launch this session (`"steam:570"`, from the library /// page); `None` = plain desktop session. pub launch: Option, + /// The presenter's shared Vulkan device, when its stack can run FFmpeg's Vulkan + /// Video decoder (decode lands as VkImages the presenter samples directly). + pub vulkan: Option, /// Pinned host fingerprint; `None` = trust on first use (caller persists the observed one). pub pin: Option<[u8; 32]>, pub identity: (String, String), @@ -241,7 +244,7 @@ fn pump( welcome_codec = connector.codec, "negotiated video codec" ); - let mut decoder = match Decoder::new(codec_id, ¶ms.decoder) { + let mut decoder = match Decoder::new(codec_id, ¶ms.decoder, params.vulkan.as_ref()) { Ok(d) => d, Err(e) => { let _ = ev_tx.send_blocking(SessionEvent::Ended(Some(format!("video decoder: {e}")))); @@ -314,11 +317,13 @@ fn pump( dec_path = match &image { DecodedImage::Cpu(_) => "software", DecodedImage::Dmabuf(_) => "vaapi", + DecodedImage::VkFrame(_) => "vulkan", }; if total_frames == 1 { let (w, h, path) = match &image { DecodedImage::Cpu(c) => (c.width, c.height, "software"), DecodedImage::Dmabuf(d) => (d.width, d.height, "vaapi-dmabuf"), + DecodedImage::VkFrame(v) => (v.width, v.height, "vulkan-video"), }; tracing::info!(width = w, height = h, path, "first frame decoded"); } diff --git a/crates/pf-client-core/src/video.rs b/crates/pf-client-core/src/video.rs index e3fa2a52..ce69b2e6 100644 --- a/crates/pf-client-core/src/video.rs +++ b/crates/pf-client-core/src/video.rs @@ -39,6 +39,38 @@ pub struct DecodedFrame { pub enum DecodedImage { Cpu(CpuFrame), Dmabuf(DmabufFrame), + /// FFmpeg Vulkan Video output: a VkImage already on the PRESENTER's device. + VkFrame(VkVideoFrame), +} + +/// One Vulkan-decoded frame. The image lives on the presenter's own VkDevice (the +/// decoder was built over its handles), so presenting is: plane views → CSC pass — no +/// import, no copy. The live synchronization state (layout / timeline value / owning +/// queue family) is deliberately NOT snapshotted here: FFmpeg updates it per submission, +/// so the presenter reads it through `vkframe` under the frames-context lock at ITS +/// submit time (the `AVVulkanFramesContext.lock_frame` contract). +pub struct VkVideoFrame { + /// `AVVkFrame*` — img[0] is the (multiplanar) image; sem/sem_value/layout/ + /// queue_family are the live sync state. Valid while `guard` lives. + pub vkframe: usize, + /// `AVHWFramesContext*` (FFmpeg's) — the first argument to the lock functions. + /// Valid while `guard` lives. + pub frames_ctx: usize, + /// `AVVulkanFramesContext.lock_frame` / `.unlock_frame` (filled in by FFmpeg's + /// init): the presenter MUST hold the lock while reading the live sync state and + /// writing back the incremented semaphore value around its submission. + pub lock_frame: usize, + pub unlock_frame: usize, + /// The frame pool's VkFormat (`AVVulkanFramesContext.format[0]`, raw i32) — the + /// multiplanar format the presenter builds its per-plane views against. + pub vk_format: i32, + pub width: u32, + pub height: u32, + pub color: ColorDesc, + /// Keeps the cloned AVFrame (and through it the VkImage + frames context) alive + /// until the presenter's fence proves the GPU reads done — same mechanism as the + /// VAAPI path's DRM guard. + pub guard: DrmFrameGuard, } /// The stream's colour signaling, read PER-FRAME from the decoder (HEVC VUI → the @@ -127,6 +159,7 @@ impl Drop for DrmFrameGuard { } enum Backend { + Vulkan(VulkanDecoder), Vaapi(VaapiDecoder), Software(SoftwareDecoder), } @@ -136,8 +169,9 @@ pub struct Decoder { /// The negotiated codec (from the host's Welcome), so a mid-session VAAPI→software demotion /// rebuilds the software decoder for the SAME codec. codec_id: ffmpeg::codec::Id, - /// Consecutive VAAPI decode errors — a single transient failure (e.g. a reference-missing - /// frame after packet loss) shouldn't cost the whole session its hardware decoder. + /// Consecutive hardware decode errors (Vulkan or VAAPI) — a single transient failure + /// (e.g. a reference-missing frame after packet loss) shouldn't cost the whole + /// session its hardware decoder. vaapi_fails: u32, /// Set when the decoder needs a fresh IDR to resynchronize (after an error or a demotion). /// The pump drains it and asks the host — under the infinite GOP there is no periodic @@ -196,33 +230,64 @@ fn quiet_ffmpeg_log() { impl Decoder { /// `codec_id` is the codec the host resolved in the Welcome (never assume HEVC). - /// `pref` is the Settings "Video decoder" value (`auto`/`vaapi`/`software`). + /// `pref` is the Settings "Video decoder" value (`auto`/`vulkan`/`vaapi`/`software`). + /// `vk` is the presenter's shared Vulkan device when its stack can run FFmpeg's + /// Vulkan Video decoder — decode lands as VkImages the presenter samples directly. /// Precedence: the `PUNKTFUNK_DECODER` env override wins (support/debug escape /// hatch, and the documented knob), then the setting; both default to auto - /// (VAAPI → software). - pub fn new(codec_id: ffmpeg::codec::Id, pref: &str) -> Result { + /// (Vulkan → VAAPI → software). + pub fn new( + codec_id: ffmpeg::codec::Id, + pref: &str, + vk: Option<&VulkanDecodeDevice>, + ) -> Result { ffmpeg::init().context("ffmpeg init")?; quiet_ffmpeg_log(); let choice = std::env::var("PUNKTFUNK_DECODER") .ok() .filter(|v| !v.is_empty()) .unwrap_or_else(|| pref.to_string()); - // Deck note: `auto` means VAAPI here too. GTK's tiled-NV12 dmabuf import is broken on - // the Deck (Mesa ≥ 25.1 exports VCN surfaces TILED; artifacts/gray/washed-out), but the - // presenter routes Deck frames through the in-process GL converter (`video_gl`) instead - // of GdkDmabufTexture — and if THAT can't initialize, it demotes this decoder to - // software mid-session via [`Decoder::force_software`]. The broken direct path is never - // the fallback. - if choice != "software" { + let done = |backend| { + Ok(Decoder { + backend, + codec_id, + vaapi_fails: 0, + want_keyframe: false, + }) + }; + if matches!(choice.as_str(), "auto" | "" | "vulkan") { + match vk { + Some(vk) => match VulkanDecoder::new(codec_id, vk) { + Ok(v) => { + tracing::info!( + ?codec_id, + "Vulkan Video hardware decode active (presenter-shared device)" + ); + return done(Backend::Vulkan(v)); + } + Err(e) => { + if choice == "vulkan" { + return Err(e.context("PUNKTFUNK_DECODER=vulkan but it failed")); + } + tracing::info!(reason = %format!("{e:#}"), + "Vulkan Video unavailable — trying VAAPI"); + } + }, + None if choice == "vulkan" => { + bail!("PUNKTFUNK_DECODER=vulkan but the presenter's device can't (missing \ + video extensions/queue) — see the presenter log") + } + None => {} + } + } + // Deck note: `auto` reaches VAAPI when Vulkan Video isn't available. A presenter + // that can't display the dmabufs demotes this decoder to software mid-session + // via [`Decoder::force_software`]. + if choice != "software" && choice != "vulkan" { match VaapiDecoder::new(codec_id) { Ok(v) => { tracing::info!(?codec_id, "VAAPI hardware decode active (zero-copy dmabuf)"); - return Ok(Decoder { - backend: Backend::Vaapi(v), - codec_id, - vaapi_fails: 0, - want_keyframe: false, - }); + return done(Backend::Vaapi(v)); } Err(e) => { if choice == "vaapi" { @@ -232,12 +297,7 @@ impl Decoder { } } } - Ok(Decoder { - backend: Backend::Software(SoftwareDecoder::new(codec_id)?), - codec_id, - vaapi_fails: 0, - want_keyframe: false, - }) + done(Backend::Software(SoftwareDecoder::new(codec_id)?)) } /// Drain the "please ask the host for an IDR" flag — the pump calls this each iteration @@ -269,28 +329,34 @@ impl Decoder { /// pump asks the host for a fresh IDR — under the infinite GOP nothing else resyncs a /// rebuilt/erroring decoder, so skipping this leaves the picture gray/frozen for good. pub fn decode(&mut self, au: &[u8]) -> Result> { - match &mut self.backend { - Backend::Vaapi(v) => match v.decode(au) { - Ok(f) => { + let result = match &mut self.backend { + Backend::Vulkan(v) => v.decode(au).map(|f| f.map(DecodedImage::VkFrame)), + Backend::Vaapi(v) => v.decode(au).map(|f| f.map(DecodedImage::Dmabuf)), + Backend::Software(s) => return Ok(s.decode(au)?.map(DecodedImage::Cpu)), + }; + match result { + Ok(f) => { + self.vaapi_fails = 0; + Ok(f) + } + Err(e) => { + let which = match self.backend { + Backend::Vulkan(_) => "Vulkan Video", + _ => "VAAPI", + }; + self.vaapi_fails += 1; + self.want_keyframe = true; + if self.vaapi_fails >= VAAPI_DEMOTE_AFTER { + tracing::warn!(error = %e, fails = self.vaapi_fails, + "{which} decode failing repeatedly — demoting to software"); + self.backend = Backend::Software(SoftwareDecoder::new(self.codec_id)?); self.vaapi_fails = 0; - Ok(f.map(DecodedImage::Dmabuf)) + } else { + tracing::warn!(error = %e, + "{which} decode error — requesting keyframe, keeping hardware decode"); } - Err(e) => { - self.vaapi_fails += 1; - self.want_keyframe = true; - if self.vaapi_fails >= VAAPI_DEMOTE_AFTER { - tracing::warn!(error = %e, fails = self.vaapi_fails, - "VAAPI decode failing repeatedly — demoting to software"); - self.backend = Backend::Software(SoftwareDecoder::new(self.codec_id)?); - self.vaapi_fails = 0; - } else { - tracing::warn!(error = %e, - "VAAPI decode error — requesting keyframe, keeping hardware decode"); - } - Ok(None) - } - }, - Backend::Software(s) => Ok(s.decode(au)?.map(DecodedImage::Cpu)), + Ok(None) + } } } } @@ -628,6 +694,39 @@ impl VaapiDecoder { } } +/// The presenter's Vulkan device handles, exported so FFmpeg's Vulkan Video decoder +/// runs on the SAME device the presenter samples from — the whole point: the decoded +/// VkImage is composited directly, no interop, no copy (plan: Vulkan Video phase). +/// +/// Plain integers/strings on purpose: pf-client-core has no ash dependency; pf-ffvk +/// casts these into vulkan.h handle types when filling `AVVulkanDeviceContext`. All +/// handles stay valid for the presenter's lifetime, which outlives every session pump +/// (the run loop tears the pump down before the presenter). +#[derive(Clone)] +pub struct VulkanDecodeDevice { + /// `PFN_vkGetInstanceProcAddr` from the loader — FFmpeg resolves everything else. + pub get_instance_proc_addr: usize, + pub instance: usize, + pub physical_device: usize, + pub device: usize, + /// The presenter's graphics+present family (FFmpeg's "required" tx/comp family too). + pub graphics_qf: u32, + /// Raw `VkQueueFlags` of that family (the qf[] entry wants the real capabilities). + pub graphics_queue_flags: u32, + /// The video-decode family (may equal `graphics_qf` on some hardware). + pub decode_qf: u32, + /// Raw `VkVideoCodecOperationFlagsKHR` the decode family advertises. + pub decode_video_caps: u32, + /// Everything enabled at instance/device creation — FFmpeg keys code paths off the + /// extension STRINGS, so the lists must match reality exactly. + pub instance_extensions: Vec, + pub device_extensions: Vec, + /// Features enabled at device creation (reported via `device_features`). + pub f_sampler_ycbcr: bool, + pub f_timeline_semaphore: bool, + pub f_synchronization2: bool, +} + /// `fourcc(a,b,c,d)` — the DRM FourCC packing (little-endian, `a | b<<8 | c<<16 | d<<24`). const fn fourcc(a: u8, b: u8, c: u8, d: u8) -> u32 { (a as u32) | ((b as u32) << 8) | ((c as u32) << 16) | ((d as u32) << 24) @@ -684,6 +783,314 @@ impl Drop for VaapiDecoder { } } +// --- Vulkan Video backend ------------------------------------------------------------- + +/// FFmpeg's Vulkan Video decoder over the PRESENTER's device: the hwdevice context is +/// built from [`VulkanDecodeDevice`]'s handles (not `av_hwdevice_ctx_create`, which +/// would make FFmpeg create its own device the presenter can't sample from). Output +/// frames are `AVVkFrame`s whose VkImage the presenter feeds straight to its CSC pass. +struct VulkanDecoder { + ctx: *mut ffmpeg::ffi::AVCodecContext, + hw_device: *mut ffmpeg::ffi::AVBufferRef, + packet: *mut ffmpeg::ffi::AVPacket, + frame: *mut ffmpeg::ffi::AVFrame, + /// Storage `AVVulkanDeviceContext` points into (extension string arrays + the + /// feature chain) — FFmpeg reads the extension lists past init (frames-context + /// setup keys code paths off them), so this lives exactly as long as `hw_device`. + _ctx_storage: Box, +} + +// Single-owner pointers, only touched from the session pump thread. +unsafe impl Send for VulkanDecoder {} + +struct VkCtxStorage { + _inst: Vec, + inst_ptrs: Vec<*const std::os::raw::c_char>, + _dev: Vec, + dev_ptrs: Vec<*const std::os::raw::c_char>, + f11: pf_ffvk::VkPhysicalDeviceVulkan11Features, + f12: pf_ffvk::VkPhysicalDeviceVulkan12Features, + f13: pf_ffvk::VkPhysicalDeviceVulkan13Features, +} + +impl VulkanDecoder { + fn new(codec_id: ffmpeg::codec::Id, vk: &VulkanDecodeDevice) -> Result { + use ffmpeg::ffi; + unsafe { + let mut hw_device = + ffi::av_hwdevice_ctx_alloc(ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VULKAN); + if hw_device.is_null() { + bail!("av_hwdevice_ctx_alloc(VULKAN) failed (FFmpeg built without Vulkan?)"); + } + let devctx = (*hw_device).data as *mut ffi::AVHWDeviceContext; + let hwctx = (*devctx).hwctx as *mut pf_ffvk::AVVulkanDeviceContext; + + // Pinned storage for everything the context points into. + let mut store = Box::new(VkCtxStorage { + _inst: vk.instance_extensions.clone(), + inst_ptrs: Vec::new(), + _dev: vk.device_extensions.clone(), + dev_ptrs: Vec::new(), + f11: std::mem::zeroed(), + f12: std::mem::zeroed(), + f13: std::mem::zeroed(), + }); + store.inst_ptrs = store._inst.iter().map(|c| c.as_ptr()).collect(); + store.dev_ptrs = store._dev.iter().map(|c| c.as_ptr()).collect(); + // The features enabled at device creation, as the 1.1/1.2/1.3 chain FFmpeg + // walks to learn what it may use (sType values are vulkan.h constants). + store.f11.sType = + pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES; + store.f11.samplerYcbcrConversion = vk.f_sampler_ycbcr as u32; + store.f12.sType = + pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; + store.f12.timelineSemaphore = vk.f_timeline_semaphore as u32; + store.f13.sType = + pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES; + store.f13.synchronization2 = vk.f_synchronization2 as u32; + store.f11.pNext = &mut store.f12 as *mut _ as *mut std::ffi::c_void; + store.f12.pNext = &mut store.f13 as *mut _ as *mut std::ffi::c_void; + + (*hwctx).get_proc_addr = + std::mem::transmute::( + vk.get_instance_proc_addr, + ); + (*hwctx).inst = vk.instance as pf_ffvk::VkInstance; + (*hwctx).phys_dev = vk.physical_device as pf_ffvk::VkPhysicalDevice; + (*hwctx).act_dev = vk.device as pf_ffvk::VkDevice; + (*hwctx).device_features.sType = + pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + (*hwctx).device_features.pNext = + &mut store.f11 as *mut _ as *mut std::ffi::c_void; + (*hwctx).enabled_inst_extensions = store.inst_ptrs.as_ptr(); + (*hwctx).nb_enabled_inst_extensions = store.inst_ptrs.len() as i32; + (*hwctx).enabled_dev_extensions = store.dev_ptrs.as_ptr(); + (*hwctx).nb_enabled_dev_extensions = store.dev_ptrs.len() as i32; + + // Queue map: the deprecated per-role indices (tx/comp are "Required") plus + // the qf[] list, which per the header must also carry every family named + // above. One merged entry when decode shares the graphics family. + let g = vk.graphics_qf as i32; + let d = vk.decode_qf as i32; + (*hwctx).queue_family_index = g; + (*hwctx).nb_graphics_queues = 1; + (*hwctx).queue_family_tx_index = g; + (*hwctx).nb_tx_queues = 1; + (*hwctx).queue_family_comp_index = g; + (*hwctx).nb_comp_queues = 1; + (*hwctx).queue_family_encode_index = -1; + (*hwctx).nb_encode_queues = 0; + (*hwctx).queue_family_decode_index = d; + (*hwctx).nb_decode_queues = 1; + const VIDEO_DECODE_BIT: u32 = 0x20; // VK_QUEUE_VIDEO_DECODE_BIT_KHR + if g == d { + (*hwctx).qf[0] = pf_ffvk::AVVulkanDeviceQueueFamily { + idx: g, + num: 1, + flags: vk.graphics_queue_flags | VIDEO_DECODE_BIT, + video_caps: vk.decode_video_caps, + }; + (*hwctx).nb_qf = 1; + } else { + (*hwctx).qf[0] = pf_ffvk::AVVulkanDeviceQueueFamily { + idx: g, + num: 1, + flags: vk.graphics_queue_flags, + video_caps: 0, + }; + (*hwctx).qf[1] = pf_ffvk::AVVulkanDeviceQueueFamily { + idx: d, + num: 1, + flags: VIDEO_DECODE_BIT, + video_caps: vk.decode_video_caps, + }; + (*hwctx).nb_qf = 2; + } + + let r = ffi::av_hwdevice_ctx_init(hw_device); + if r < 0 { + ffi::av_buffer_unref(&mut hw_device); + return Err(averr("av_hwdevice_ctx_init(VULKAN)", r)); + } + + let codec = ffi::avcodec_find_decoder(codec_id.into()); + if codec.is_null() { + ffi::av_buffer_unref(&mut hw_device); + bail!("no {codec_id:?} decoder"); + } + let ctx = ffi::avcodec_alloc_context3(codec); + (*ctx).hw_device_ctx = ffi::av_buffer_ref(hw_device); + (*ctx).get_format = Some(pick_vulkan); + (*ctx).flags |= ffi::AV_CODEC_FLAG_LOW_DELAY as i32; + (*ctx).thread_count = 1; // hwaccel: threads only add latency + // Same pool headroom rationale as VAAPI: the presenter pins the on-screen + // frame + the newest in flight past receive_frame. + (*ctx).extra_hw_frames = 4; + let r = ffi::avcodec_open2(ctx, codec, ptr::null_mut()); + if r < 0 { + let mut ctx = ctx; + ffi::avcodec_free_context(&mut ctx); + ffi::av_buffer_unref(&mut hw_device); + return Err(averr("avcodec_open2 (vulkan)", r)); + } + Ok(VulkanDecoder { + ctx, + hw_device, + packet: ffi::av_packet_alloc(), + frame: ffi::av_frame_alloc(), + _ctx_storage: store, + }) + } + } + + fn decode(&mut self, au: &[u8]) -> Result> { + use ffmpeg::ffi; + unsafe { + let r = ffi::av_new_packet(self.packet, au.len() as i32); + if r < 0 { + return Err(averr("av_new_packet", r)); + } + ptr::copy_nonoverlapping(au.as_ptr(), (*self.packet).data, au.len()); + let r = ffi::avcodec_send_packet(self.ctx, self.packet); + ffi::av_packet_unref(self.packet); + if r < 0 { + return Err(averr("send_packet", r)); + } + let mut out = None; + loop { + let r = ffi::avcodec_receive_frame(self.ctx, self.frame); + if r == AVERROR_EAGAIN { + break; + } + if r < 0 { + return Err(averr("receive_frame", r)); + } + out = Some(self.extract()?); // newest wins; older guards drop here + ffi::av_frame_unref(self.frame); + } + Ok(out) + } + } + + /// Lift the decoded `AVVkFrame` into a [`VkVideoFrame`]: clone the AVFrame (the + /// guard — keeps the image + frames context alive through present) and ship the + /// POINTERS; the presenter reads the live sync state under the frames-context lock + /// at its own submit time. + unsafe fn extract(&mut self) -> Result { + use ffmpeg::ffi; + unsafe { + if (*self.frame).format != ffi::AVPixelFormat::AV_PIX_FMT_VULKAN as i32 { + bail!("decoder returned a non-Vulkan frame"); + } + let hwfc_ref = (*self.frame).hw_frames_ctx; + if hwfc_ref.is_null() { + bail!("Vulkan frame without a hardware frames context"); + } + let fc = (*hwfc_ref).data as *mut ffi::AVHWFramesContext; + let sw = (*fc).sw_format; + if sw != ffi::AVPixelFormat::AV_PIX_FMT_NV12 { + bail!("Vulkan decode output {sw:?} unsupported (NV12 only for now)"); + } + let vkfc = (*fc).hwctx as *const pf_ffvk::AVVulkanFramesContext; + let vk_format = (*vkfc).format[0] as i32; + let lock_frame = (*vkfc).lock_frame.map_or(0, |f| f as usize); + let unlock_frame = (*vkfc).unlock_frame.map_or(0, |f| f as usize); + if lock_frame == 0 || unlock_frame == 0 { + bail!("Vulkan frames context without lock functions"); + } + + let clone = ffi::av_frame_clone(self.frame); + if clone.is_null() { + bail!("av_frame_clone failed"); + } + let vkf = (*clone).data[0] as *mut pf_ffvk::AVVkFrame; + // v1 handles the (default) single multiplanar image; a disjoint/multi-image + // pool would need per-plane images — bail so the session demotes cleanly. + if !(*vkf).img[1].is_null() { + let mut clone = clone; + ffi::av_frame_free(&mut clone); + bail!("multi-image Vulkan frames unsupported (disjoint pool)"); + } + Ok(VkVideoFrame { + vkframe: vkf as usize, + frames_ctx: fc as usize, + lock_frame, + unlock_frame, + vk_format, + width: (*self.frame).width as u32, + height: (*self.frame).height as u32, + color: ColorDesc::from_raw(self.frame), + guard: DrmFrameGuard(clone), + }) + } + } +} + +impl Drop for VulkanDecoder { + fn drop(&mut self) { + use ffmpeg::ffi; + unsafe { + ffi::av_packet_free(&mut self.packet); + ffi::av_frame_free(&mut self.frame); + ffi::avcodec_free_context(&mut self.ctx); + ffi::av_buffer_unref(&mut self.hw_device); + } + } +} + +/// libavcodec offers the formats it can decode into; pick the Vulkan hw surface and +/// hand the decoder OUR frames context — the default one lacks +/// `VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT`, without which the presenter can't create the +/// per-plane views its CSC pass samples. Returning NONE (over the software entry) keeps +/// failures loud: the session demotes explicitly instead of silently CPU-decoding. +unsafe extern "C" fn pick_vulkan( + ctx: *mut ffmpeg::ffi::AVCodecContext, + mut list: *const ffmpeg::ffi::AVPixelFormat, +) -> ffmpeg::ffi::AVPixelFormat { + use ffmpeg::ffi; + unsafe { + let mut offered = false; + while *list != ffi::AVPixelFormat::AV_PIX_FMT_NONE { + if *list == ffi::AVPixelFormat::AV_PIX_FMT_VULKAN { + offered = true; + break; + } + list = list.add(1); + } + if !offered { + return ffi::AVPixelFormat::AV_PIX_FMT_NONE; + } + let mut fr: *mut ffi::AVBufferRef = ptr::null_mut(); + let r = ffi::avcodec_get_hw_frames_parameters( + ctx, + (*ctx).hw_device_ctx, + ffi::AVPixelFormat::AV_PIX_FMT_VULKAN, + &mut fr, + ); + if r < 0 || fr.is_null() { + tracing::warn!("avcodec_get_hw_frames_parameters(VULKAN) failed ({r})"); + return ffi::AVPixelFormat::AV_PIX_FMT_NONE; + } + let fc = (*fr).data as *mut ffi::AVHWFramesContext; + let vkfc = (*fc).hwctx as *mut pf_ffvk::AVVulkanFramesContext; + // MUTABLE_FORMAT: per-plane views (spec requirement); ALIAS is FFmpeg's default. + (*vkfc).img_flags = pf_ffvk::VkImageCreateFlagBits_VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT + | pf_ffvk::VkImageCreateFlagBits_VK_IMAGE_CREATE_ALIAS_BIT; + let r = ffi::av_hwframe_ctx_init(fr); + if r < 0 { + tracing::warn!("av_hwframe_ctx_init(VULKAN) failed ({r})"); + let mut fr = fr; + ffi::av_buffer_unref(&mut fr); + return ffi::AVPixelFormat::AV_PIX_FMT_NONE; + } + if !(*ctx).hw_frames_ctx.is_null() { + ffi::av_buffer_unref(&mut (*ctx).hw_frames_ctx); + } + (*ctx).hw_frames_ctx = fr; // the codec owns our ref now + ffi::AVPixelFormat::AV_PIX_FMT_VULKAN + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/pf-ffvk/build.rs b/crates/pf-ffvk/build.rs index 73d2ec61..f5212fce 100644 --- a/crates/pf-ffvk/build.rs +++ b/crates/pf-ffvk/build.rs @@ -50,6 +50,13 @@ fn main() { .allowlist_type("AVVkFrame.*") .allowlist_function("av_vk_frame_alloc") .allowlist_function("av_vkfmt_from_pixfmt") + // The feature structs chained into AVVulkanDeviceContext.device_features (plain + // vulkan.h types; generating them here keeps the chain in one type system). + .allowlist_type("VkPhysicalDeviceVulkan11Features") + .allowlist_type("VkPhysicalDeviceVulkan12Features") + .allowlist_type("VkPhysicalDeviceVulkan13Features") + // AVVulkanFramesContext.img_flags values (plane views need MUTABLE_FORMAT). + .allowlist_type("VkImageCreateFlagBits") // …plus nothing else of FFmpeg: the core types these structs reference only // ever appear behind pointers here, so keep them opaque instead of duplicating // ffmpeg-sys-next's definitions (callers cast pointers between the crates). diff --git a/crates/pf-presenter/Cargo.toml b/crates/pf-presenter/Cargo.toml index a6fcebf6..ee8b6930 100644 --- a/crates/pf-presenter/Cargo.toml +++ b/crates/pf-presenter/Cargo.toml @@ -11,6 +11,8 @@ repository.workspace = true # Same Linux gating as the rest of the client stack. [target.'cfg(target_os = "linux")'.dependencies] pf-client-core = { path = "../pf-client-core" } +# AVVkFrame access (Vulkan Video frames: live sync state under the frames lock). +pf-ffvk = { path = "../pf-ffvk" } punktfunk-core = { path = "../punktfunk-core", features = ["quic"] } # `loaded` dlopens libvulkan at runtime (no link-time dependency — GPU-less boxes still diff --git a/crates/pf-presenter/src/run.rs b/crates/pf-presenter/src/run.rs index 128f90d7..26a35d25 100644 --- a/crates/pf-presenter/src/run.rs +++ b/crates/pf-presenter/src/run.rs @@ -16,6 +16,7 @@ use crate::overlay::{FrameCtx, Overlay, OverlayAction, OverlayFrame, SessionPhas use crate::vk::{FrameInput, Presenter}; use anyhow::{Context as _, Result}; use pf_client_core::gamepad::GamepadService; +use pf_client_core::video::VulkanDecodeDevice; use pf_client_core::session::{self, SessionEvent, SessionHandle, SessionParams, Stats}; use pf_client_core::video::{DecodedFrame, DecodedImage}; use punktfunk_core::client::NativeClient; @@ -58,8 +59,8 @@ pub enum ActionOutcome { /// Consumed binary-side (a Retry respawned the fetch, …). Handled, /// Start this session (a Launch action; `force_software` from the callback args is - /// wired into these params). - Start(SessionParams), + /// wired into these params). Boxed: SessionParams is large next to the unit variants. + Start(Box), /// Quit the launcher. Quit, } @@ -67,13 +68,13 @@ pub enum ActionOutcome { /// One `--connect` stream session; returns when it ends (the shell↔session contract). pub fn run_session(opts: SessionOpts, build_params: F) -> Result where - F: FnOnce(&GamepadService, Mode, Arc) -> SessionParams, + F: FnOnce(&GamepadService, Mode, Arc, Option) -> SessionParams, { let mut build = Some(build_params); run_inner( opts, - ModeCtl::Single(Box::new(move |gp, native, fs| { - (build.take().expect("single build runs once"))(gp, native, fs) + ModeCtl::Single(Box::new(move |gp, native, fs, vk| { + (build.take().expect("single build runs once"))(gp, native, fs, vk) })), ) .map(|o| o.expect("single mode always yields an outcome")) @@ -85,7 +86,13 @@ where /// per-session `force_software` flag. pub fn run_browse(opts: SessionOpts, on_action: F) -> Result<()> where - F: FnMut(OverlayAction, &GamepadService, Mode, Arc) -> ActionOutcome, + F: FnMut( + OverlayAction, + &GamepadService, + Mode, + Arc, + Option, + ) -> ActionOutcome, { anyhow::ensure!( opts.overlay.is_some(), @@ -95,10 +102,21 @@ where } /// Params builder for the one single-mode session (called exactly once, post-setup). -type BuildParams<'a> = Box) -> SessionParams + 'a>; +type BuildParams<'a> = Box< + dyn FnMut(&GamepadService, Mode, Arc, Option) -> SessionParams + + 'a, +>; /// The browse-mode action callback (Launch → params, Retry/Quit → outcome). -type OnAction<'a> = - Box) -> ActionOutcome + 'a>; +type OnAction<'a> = Box< + dyn FnMut( + OverlayAction, + &GamepadService, + Mode, + Arc, + Option, + ) -> ActionOutcome + + 'a, +>; /// The two run modes, type-erased so one loop serves both. enum ModeCtl<'a> { @@ -227,7 +245,12 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result let mut stream: Option = match &mut mode { ModeCtl::Single(build) => { let force_software = Arc::new(AtomicBool::new(false)); - let params = build(&gamepad, native, force_software.clone()); + let params = build( + &gamepad, + native, + force_software.clone(), + presenter.vulkan_decode(), + ); Some(StreamState::new(params, force_software)) } ModeCtl::Browse(_) => None, @@ -416,10 +439,16 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result } if let Some(action) = overlay.as_mut().and_then(|o| o.take_action()) { let force_software = Arc::new(AtomicBool::new(false)); - match on_action(action, &gamepad, native, force_software.clone()) { + match on_action( + action, + &gamepad, + native, + force_software.clone(), + presenter.vulkan_decode(), + ) { ActionOutcome::Handled => {} ActionOutcome::Start(params) => { - stream = Some(StreamState::new(params, force_software)); + stream = Some(StreamState::new(*params, force_software)); if let Some(o) = overlay.as_mut() { o.session_phase(SessionPhase::Connecting); } @@ -605,6 +634,34 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result } false } + // Vulkan-Video: decoded on the presenter's own device — present is + // views + CSC, no import step to gate on. Same failure-streak + // demotion contract as the dmabuf path. + DecodedImage::VkFrame(v) if !st.dmabuf_demoted => { + st.hdr = v.color.is_pq(); + match presenter.present( + &window, + FrameInput::VkFrame(v), + overlay_frame.as_ref(), + ) { + Ok(p) => { + st.hw_fails = 0; + p + } + Err(e) => { + st.hw_fails += 1; + tracing::warn!(error = %format!("{e:#}"), fails = st.hw_fails, + "vulkan-video present failed"); + if st.hw_fails >= 3 { + st.dmabuf_demoted = true; + tracing::warn!("demoting the decoder to software"); + st.force_software.store(true, Ordering::Relaxed); + } + false + } + } + } + DecodedImage::VkFrame(_) => false, // demoted — drain until rebuild }; if did_present { presented_video = true; diff --git a/crates/pf-presenter/src/vk.rs b/crates/pf-presenter/src/vk.rs index 2e1f659d..120fda5b 100644 --- a/crates/pf-presenter/src/vk.rs +++ b/crates/pf-presenter/src/vk.rs @@ -19,7 +19,8 @@ use crate::dmabuf::{self, HwFrame}; use crate::overlay::{OverlayFrame, SharedDevice}; use anyhow::{anyhow, bail, Context as _, Result}; use ash::vk; -use pf_client_core::video::{CpuFrame, DmabufFrame}; +use ash::vk::Handle as _; +use pf_client_core::video::{CpuFrame, DmabufFrame, VkVideoFrame}; use std::ffi::CString; /// One presenter iteration's video input. @@ -28,12 +29,40 @@ pub enum FrameInput<'a> { Redraw, Cpu(&'a CpuFrame), Dmabuf(DmabufFrame), + /// FFmpeg Vulkan Video output — a VkImage already on THIS device (zero copy). + VkFrame(VkVideoFrame), } /// The dmabuf/CSC machinery, present only when the device carries the import extensions. struct HwCtx { ext_mem_fd: ash::khr::external_memory_fd::Device, - csc: CscPass, +} + +/// A submitted hardware frame parked until the in-flight fence proves the GPU reads +/// done: imported dmabuf planes, or a Vulkan-Video frame (FFmpeg's image — we own only +/// the plane views; dropping the frame's guard releases the AVFrame back to the pool). +enum Retired { + Dmabuf(HwFrame), + Vk { + frame: VkVideoFrame, + views: [vk::ImageView; 2], + }, +} + +impl Retired { + fn destroy(self, device: &ash::Device) { + match self { + Retired::Dmabuf(f) => f.destroy(device), + Retired::Vk { frame, views } => { + unsafe { + for v in views { + device.destroy_image_view(v, None); + } + } + drop(frame); // guard drops here — AVFrame (and the VkImage) released + } + } + } } /// The overlay composite: one premultiplied-alpha quad blended over the swapchain image @@ -251,13 +280,18 @@ pub struct Presenter { swap_d: ash::khr::swapchain::Device, queue: vk::Queue, qfi: u32, - /// Dmabuf import + CSC — `None` when the device lacks the import extensions. + /// Dmabuf import — `None` when the device lacks the import extensions (the CSC + /// pass itself is unconditional: Vulkan-Video frames need it everywhere). hw: Option, + csc: CscPass, + /// FFmpeg Vulkan Video decode handles — `None` when the stack can't do it. + video_export: Option, /// The console-UI composite quad (§6.1's presenter half). overlay_pipe: OverlayPipe, - /// The submitted hw frame (plane images + decoder-surface guard): its GPU reads end - /// with the in-flight fence, so it's destroyed right after the next fence wait. - retired_hw: Option, + /// The submitted hardware frame (dmabuf plane images + guard, or a Vulkan-Video + /// frame + our plane views): its GPU reads end with the in-flight fence, so it's + /// destroyed right after the next fence wait. + retired_hw: Option, format: vk::SurfaceFormatKHR, present_mode: vk::PresentModeKHR, swapchain: vk::SwapchainKHR, @@ -285,9 +319,12 @@ impl Presenter { let entry = unsafe { ash::Entry::load() }.context("libvulkan not loadable")?; let app_name = CString::new("punktfunk-session").unwrap(); + // 1.3: FFmpeg's Vulkan hwcontext requires an instance of at least 1.3 (any + // current loader accepts it regardless of device support; device-level gating + // happens below). let app_info = vk::ApplicationInfo::default() .application_name(&app_name) - .api_version(vk::API_VERSION_1_2); + .api_version(vk::API_VERSION_1_3); let ext_cstrings: Vec = instance_extensions .iter() .map(|e| CString::new(e.as_str()).unwrap()) @@ -318,11 +355,8 @@ impl Presenter { tracing::info!(device = %name, queue_family = qfi, "vulkan device"); } - let queue_info = [vk::DeviceQueueCreateInfo::default() - .queue_family_index(qfi) - .queue_priorities(&[1.0])]; // The dmabuf import set is optional: enabled when the device offers all four, - // else the presenter is software-only (`supports_dmabuf() == false`). + // else that path is off (`supports_dmabuf() == false`). let available = unsafe { instance.enumerate_device_extension_properties(pdev) }?; let has = |name: &std::ffi::CStr| { available @@ -335,8 +369,123 @@ impl Presenter { dev_exts.extend(dmabuf::DEVICE_EXTENSIONS.iter().map(|n| n.as_ptr())); } else { tracing::info!( - "device lacks the dmabuf import extensions — hardware frames unavailable \ - (software decode only)" + "device lacks the dmabuf import extensions — VAAPI hardware frames \ + unavailable" + ); + } + + // --- Vulkan Video decode (the FFmpeg-on-our-device path) --------------------- + // Probed, never required: a capable stack gets the video extensions, a second + // (decode) queue, and the features FFmpeg's decoder needs; anything less means + // `vulkan_decode() == None` and the decoder chain falls back (VAAPI/software). + let dev_props = unsafe { instance.get_physical_device_properties(pdev) }; + let dev_is_13 = vk::api_version_major(dev_props.api_version) > 1 + || vk::api_version_minor(dev_props.api_version) >= 3; + let mut have_f11 = vk::PhysicalDeviceVulkan11Features::default(); + let mut have_f12 = vk::PhysicalDeviceVulkan12Features::default(); + let mut have_f13 = vk::PhysicalDeviceVulkan13Features::default(); + let mut have_f2 = vk::PhysicalDeviceFeatures2::default() + .push_next(&mut have_f11) + .push_next(&mut have_f12) + .push_next(&mut have_f13); + unsafe { instance.get_physical_device_features2(pdev, &mut have_f2) }; + let features_ok = have_f11.sampler_ycbcr_conversion == vk::TRUE + && have_f12.timeline_semaphore == vk::TRUE + && have_f13.synchronization2 == vk::TRUE; + + // The decode queue family + which codec operations it can run. + let decode_family: Option<(u32, vk::VideoCodecOperationFlagsKHR)> = { + let n = unsafe { instance.get_physical_device_queue_family_properties2_len(pdev) }; + let mut video: Vec = + vec![vk::QueueFamilyVideoPropertiesKHR::default(); n]; + let mut props: Vec = video + .iter_mut() + .map(|v| vk::QueueFamilyProperties2::default().push_next(v)) + .collect(); + unsafe { instance.get_physical_device_queue_family_properties2(pdev, &mut props) }; + // `props` mutably borrows `video` (push_next); copy the flags out, then + // read the driver-filled video properties directly. + let flags: Vec = props + .iter() + .map(|p| p.queue_family_properties.queue_flags) + .collect(); + drop(props); + flags + .iter() + .zip(&video) + .enumerate() + .find(|(_, (f, _))| f.contains(vk::QueueFlags::VIDEO_DECODE_KHR)) + .map(|(i, (_, v))| (i as u32, v.video_codec_operations)) + }; + + const VIDEO_BASE: [&std::ffi::CStr; 2] = [ + ash::khr::video_queue::NAME, + ash::khr::video_decode_queue::NAME, + ]; + const VIDEO_CODECS: [&std::ffi::CStr; 3] = [ + ash::khr::video_decode_h264::NAME, + ash::khr::video_decode_h265::NAME, + c"VK_KHR_video_decode_av1", + ]; + let codec_exts: Vec<&std::ffi::CStr> = VIDEO_CODECS + .into_iter() + .filter(|n| has(n)) + .collect(); + let video_ok = dev_is_13 + && features_ok + && decode_family.is_some() + && VIDEO_BASE.iter().all(|n| has(n)) + && !codec_exts.is_empty(); + + let (decode_qf, decode_caps) = decode_family.unwrap_or((qfi, Default::default())); + let mut video_ext_names: Vec<&std::ffi::CStr> = Vec::new(); + if video_ok { + video_ext_names.extend(VIDEO_BASE); + video_ext_names.extend(&codec_exts); + // Optional decoder niceties FFmpeg uses when present. + for opt in [c"VK_KHR_video_maintenance1", c"VK_KHR_video_maintenance2"] { + if has(opt) { + video_ext_names.push(opt); + } + } + dev_exts.extend(video_ext_names.iter().map(|n| n.as_ptr())); + tracing::info!( + decode_qf, + caps = ?decode_caps, + exts = ?video_ext_names, + "Vulkan Video decode available on this device" + ); + } else { + tracing::info!( + dev_is_13, + features_ok, + decode_family = decode_family.is_some(), + "Vulkan Video decode unavailable — decoder falls back (VAAPI/software)" + ); + } + + // Enable only the features the video path needs, and only where supported + // (harmless when the path is off; reported to FFmpeg via device_features). + let mut en_f11 = vk::PhysicalDeviceVulkan11Features::default() + .sampler_ycbcr_conversion(have_f11.sampler_ycbcr_conversion == vk::TRUE); + let mut en_f12 = vk::PhysicalDeviceVulkan12Features::default() + .timeline_semaphore(have_f12.timeline_semaphore == vk::TRUE); + let mut en_f13 = vk::PhysicalDeviceVulkan13Features::default() + .synchronization2(have_f13.synchronization2 == vk::TRUE); + let mut en_f2 = vk::PhysicalDeviceFeatures2::default() + .push_next(&mut en_f11) + .push_next(&mut en_f12) + .push_next(&mut en_f13); + + let priorities = [1.0f32]; + let mut queue_info = vec![vk::DeviceQueueCreateInfo::default() + .queue_family_index(qfi) + .queue_priorities(&priorities)]; + if video_ok && decode_qf != qfi { + queue_info.push( + vk::DeviceQueueCreateInfo::default() + .queue_family_index(decode_qf) + .queue_priorities(&priorities), ); } let device = unsafe { @@ -344,7 +493,8 @@ impl Presenter { pdev, &vk::DeviceCreateInfo::default() .queue_create_infos(&queue_info) - .enabled_extension_names(&dev_exts), + .enabled_extension_names(&dev_exts) + .push_next(&mut en_f2), None, ) } @@ -354,7 +504,41 @@ impl Presenter { let hw = if hw_capable { Some(HwCtx { ext_mem_fd: ash::khr::external_memory_fd::Device::new(&instance, &device), - csc: CscPass::new(&device)?, + }) + } else { + None + }; + let csc = CscPass::new(&device)?; + + // The exported handle bundle for FFmpeg's Vulkan Video decoder (None = the + // decoder chain skips straight to VAAPI/software). Extension lists must mirror + // creation exactly — FFmpeg keys its code paths off the strings. + let video_export = if video_ok { + let qf_props = unsafe { instance.get_physical_device_queue_family_properties(pdev) }; + let mut device_extensions: Vec = + vec![CString::from(ash::khr::swapchain::NAME)]; + if hw_capable { + device_extensions + .extend(dmabuf::DEVICE_EXTENSIONS.iter().map(|n| CString::from(*n))); + } + device_extensions.extend(video_ext_names.iter().map(|n| CString::from(*n))); + Some(pf_client_core::video::VulkanDecodeDevice { + get_instance_proc_addr: entry.static_fn().get_instance_proc_addr as usize, + instance: instance.handle().as_raw() as usize, + physical_device: pdev.as_raw() as usize, + device: device.handle().as_raw() as usize, + graphics_qf: qfi, + graphics_queue_flags: qf_props[qfi as usize].queue_flags.as_raw(), + decode_qf, + decode_video_caps: decode_caps.as_raw(), + instance_extensions: instance_extensions + .iter() + .map(|e| CString::new(e.as_str()).unwrap()) + .collect(), + device_extensions, + f_sampler_ycbcr: true, + f_timeline_semaphore: true, + f_synchronization2: true, }) } else { None @@ -402,6 +586,8 @@ impl Presenter { queue, qfi, hw, + csc, + video_export, overlay_pipe, retired_hw: None, format, @@ -502,6 +688,13 @@ impl Presenter { self.hw.is_some() } + /// The FFmpeg Vulkan Video decode handle bundle — `None` when this stack can't + /// (device < 1.3, missing video extensions/queue/features). The decoder chain + /// falls back to VAAPI/software then. + pub fn vulkan_decode(&self) -> Option { + self.video_export.clone() + } + /// Quiesce the queue — the run loop calls this before dropping the overlay so /// nothing in flight still references its images. pub fn wait_idle(&self) { @@ -535,9 +728,11 @@ impl Presenter { if self.extent.width == 0 || self.extent.height == 0 { return Ok(true); // minimized — nothing to do } - // A dmabuf frame imports before anything touches the queue: an import the driver - // rejects must fail out here, before this present consumed the acquire semaphore. + // Hardware frames prepare before anything touches the queue: an import/view the + // driver rejects must fail out here, before this present consumed the acquire + // semaphore. let mut hw_frame: Option = None; + let mut vk_frame: Option<(VkVideoFrame, [vk::ImageView; 2])> = None; let cpu_frame = match input { FrameInput::Redraw => None, FrameInput::Cpu(f) => Some(f), @@ -549,6 +744,11 @@ impl Presenter { hw_frame = Some(dmabuf::import(&self.device, &hw.ext_mem_fd, d)?); None } + FrameInput::VkFrame(v) => { + let views = self.vkframe_plane_views(&v)?; + vk_frame = Some((v, views)); + None + } }; // One frame in flight: the fence covers the command buffer, the staging buffer @@ -577,8 +777,18 @@ impl Presenter { tracing::info!(width = f.width, height = f.height, "video image (re)built"); } // Safe while nothing in flight references the set — the fence wait above. - let hw = self.hw.as_ref().unwrap(); - hw.csc.bind_planes(&self.device, f.luma_view, f.chroma_view); + self.csc.bind_planes(&self.device, f.luma_view, f.chroma_view); + } + if let Some((f, views)) = &vk_frame { + if self + .video + .as_ref() + .is_none_or(|v| v.width != f.width || v.height != f.height) + { + self.rebuild_video_image(f.width, f.height)?; + tracing::info!(width = f.width, height = f.height, "video image (re)built"); + } + self.csc.bind_planes(&self.device, views[0], views[1]); } if let Some(o) = overlay { // Point the composite at this overlay image (same fence-wait safety). @@ -621,10 +831,10 @@ impl Presenter { .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), )?; - // Hardware frame: acquire the foreign planes, then the CSC pass renders + // Dmabuf frame: acquire the foreign planes, then the CSC pass renders // NV12→RGBA into the video image (render pass ends it in TRANSFER_SRC for // the blit below). - if let (Some(f), Some(hw), Some(v)) = (&hw_frame, &self.hw, &self.video) { + if let (Some(f), Some(v)) = (&hw_frame, &self.video) { for view_image in [f.luma_image(), f.chroma_image()] { foreign_acquire_barrier(&self.device, self.cmd_buf, view_image, self.qfi); } @@ -632,61 +842,30 @@ impl Presenter { width: v.width, height: v.height, }; - self.device.cmd_begin_render_pass( + self.record_csc(v.framebuffer, extent, f.color); + } + + // Vulkan-Video frame: the decoded image is already on THIS device. Read the + // live sync state under the frames lock (held through submission — the + // AVVulkanFramesContext contract), acquire from the decode queue family, + // then the same CSC pass. + let mut vk_sync: Option = None; + if let (Some((f, _)), Some(v)) = (&vk_frame, &self.video) { + let sync = lock_vkframe(f); + vkframe_acquire_barrier( + &self.device, self.cmd_buf, - &vk::RenderPassBeginInfo::default() - .render_pass(hw.csc.render_pass) - .framebuffer(v.framebuffer) - .render_area(vk::Rect2D { - offset: vk::Offset2D { x: 0, y: 0 }, - extent, - }), - vk::SubpassContents::INLINE, + vk::Image::from_raw(sync.image), + vk::ImageLayout::from_raw(sync.layout), + sync.queue_family, + self.qfi, ); - self.device.cmd_bind_pipeline( - self.cmd_buf, - vk::PipelineBindPoint::GRAPHICS, - hw.csc.pipeline, - ); - self.device.cmd_set_viewport( - self.cmd_buf, - 0, - &[vk::Viewport { - x: 0.0, - y: 0.0, - width: extent.width as f32, - height: extent.height as f32, - min_depth: 0.0, - max_depth: 1.0, - }], - ); - self.device.cmd_set_scissor( - self.cmd_buf, - 0, - &[vk::Rect2D { - offset: vk::Offset2D { x: 0, y: 0 }, - extent, - }], - ); - self.device.cmd_bind_descriptor_sets( - self.cmd_buf, - vk::PipelineBindPoint::GRAPHICS, - hw.csc.pipeline_layout, - 0, - &[hw.csc.desc_set], - &[], - ); - let rows = csc_rows(f.color); - let bytes = std::slice::from_raw_parts(rows.as_ptr().cast::(), 48); - self.device.cmd_push_constants( - self.cmd_buf, - hw.csc.pipeline_layout, - vk::ShaderStageFlags::FRAGMENT, - 0, - bytes, - ); - self.device.cmd_draw(self.cmd_buf, 3, 1, 0, 0); - self.device.cmd_end_render_pass(self.cmd_buf); + let extent = vk::Extent2D { + width: v.width, + height: v.height, + }; + self.record_csc(v.framebuffer, extent, f.color); + vk_sync = Some(sync); } // New frame: staging → video image (stride carried by buffer_row_length). @@ -840,23 +1019,53 @@ impl Presenter { self.device.end_command_buffer(self.cmd_buf)?; let render_sem = self.render_sems[index as usize]; - let wait_sems = [self.acquire_sem]; - let wait_stages = [vk::PipelineStageFlags::TRANSFER]; let cmd_bufs = [self.cmd_buf]; - let signal_sems = [render_sem]; - self.device.queue_submit( - self.queue, - &[vk::SubmitInfo::default() - .wait_semaphores(&wait_sems) - .wait_dst_stage_mask(&wait_stages) - .command_buffers(&cmd_bufs) - .signal_semaphores(&signal_sems)], - self.fence, - )?; + let mut wait_sems = vec![self.acquire_sem]; + let mut wait_stages = vec![vk::PipelineStageFlags::TRANSFER]; + let mut signal_sems = vec![render_sem]; + // The Vulkan-Video frame's timeline semaphore: wait for the decoder's value, + // signal value+1 when our reads are done (FFmpeg's per-submission contract). + let mut wait_values = vec![0u64]; + let mut signal_values = vec![0u64]; + if let Some(sync) = &vk_sync { + let sem = vk::Semaphore::from_raw(sync.semaphore); + wait_sems.push(sem); + wait_stages.push(vk::PipelineStageFlags::FRAGMENT_SHADER); + wait_values.push(sync.sem_value); + signal_sems.push(sem); + signal_values.push(sync.sem_value + 1); + } + let mut timeline = vk::TimelineSemaphoreSubmitInfo::default() + .wait_semaphore_values(&wait_values) + .signal_semaphore_values(&signal_values); + let mut submit = vk::SubmitInfo::default() + .wait_semaphores(&wait_sems) + .wait_dst_stage_mask(&wait_stages) + .command_buffers(&cmd_bufs) + .signal_semaphores(&signal_sems); + if vk_sync.is_some() { + submit = submit.push_next(&mut timeline); + } + let submitted = self.device.queue_submit(self.queue, &[submit], self.fence); + // Write the new sync state back and release the frames lock REGARDLESS of + // the submit outcome (an abandoned lock would wedge the decoder). + if let Some(sync) = vk_sync.take() { + let ok = submitted.is_ok(); + unlock_vkframe( + vk_frame.as_ref().map(|(f, _)| f).expect("vk_sync implies vk_frame"), + &sync, + ok, + self.qfi, + ); + } + submitted?; self.submitted = true; // The hw frame is on the GPU now — park it until the fence proves the reads // done (destroyed at the next present's fence wait, or in Drop). - self.retired_hw = hw_frame.take(); + self.retired_hw = hw_frame + .take() + .map(Retired::Dmabuf) + .or_else(|| vk_frame.take().map(|(frame, views)| Retired::Vk { frame, views })); let swapchains = [self.swapchain]; let indices = [index]; @@ -878,6 +1087,122 @@ impl Presenter { } } + + /// Record the NV12→RGBA CSC pass into the video image (framebuffer): fullscreen + /// triangle, CICP-driven push-constant rows. Shared by the dmabuf and Vulkan-Video + /// paths — only the plane views bound beforehand differ. + /// + /// # Safety + /// `self.cmd_buf` must be in the recording state; the CSC descriptor set must point + /// at live plane views. + unsafe fn record_csc( + &self, + framebuffer: vk::Framebuffer, + extent: vk::Extent2D, + color: pf_client_core::video::ColorDesc, + ) { + unsafe { + self.device.cmd_begin_render_pass( + self.cmd_buf, + &vk::RenderPassBeginInfo::default() + .render_pass(self.csc.render_pass) + .framebuffer(framebuffer) + .render_area(vk::Rect2D { + offset: vk::Offset2D { x: 0, y: 0 }, + extent, + }), + vk::SubpassContents::INLINE, + ); + self.device.cmd_bind_pipeline( + self.cmd_buf, + vk::PipelineBindPoint::GRAPHICS, + self.csc.pipeline, + ); + self.device.cmd_set_viewport( + self.cmd_buf, + 0, + &[vk::Viewport { + x: 0.0, + y: 0.0, + width: extent.width as f32, + height: extent.height as f32, + min_depth: 0.0, + max_depth: 1.0, + }], + ); + self.device.cmd_set_scissor( + self.cmd_buf, + 0, + &[vk::Rect2D { + offset: vk::Offset2D { x: 0, y: 0 }, + extent, + }], + ); + self.device.cmd_bind_descriptor_sets( + self.cmd_buf, + vk::PipelineBindPoint::GRAPHICS, + self.csc.pipeline_layout, + 0, + &[self.csc.desc_set], + &[], + ); + let rows = csc_rows(color); + let bytes = std::slice::from_raw_parts(rows.as_ptr().cast::(), 48); + self.device.cmd_push_constants( + self.cmd_buf, + self.csc.pipeline_layout, + vk::ShaderStageFlags::FRAGMENT, + 0, + bytes, + ); + self.device.cmd_draw(self.cmd_buf, 3, 1, 0, 0); + self.device.cmd_end_render_pass(self.cmd_buf); + } + } + + /// Per-plane views over a Vulkan-Video frame's multiplanar image (R8 luma + + /// R8G8 chroma — the CSC pass's exact sampling contract; the frames pool was + /// created MUTABLE_FORMAT for this). NV12 only, like the rest of the pipeline. + fn vkframe_plane_views(&self, f: &VkVideoFrame) -> Result<[vk::ImageView; 2]> { + if f.vk_format != vk::Format::G8_B8R8_2PLANE_420_UNORM.as_raw() { + bail!( + "Vulkan-Video pool format {} unsupported (expected G8_B8R8_2PLANE_420)", + f.vk_format + ); + } + // img[0] is creation-constant (only the sync fields need the frames lock). + let image = vk::Image::from_raw( + unsafe { (*(f.vkframe as *const pf_ffvk::AVVkFrame)).img[0] } as u64, + ); + let make = |aspect: vk::ImageAspectFlags, format: vk::Format| { + unsafe { + self.device.create_image_view( + &vk::ImageViewCreateInfo::default() + .image(image) + .view_type(vk::ImageViewType::TYPE_2D) + .format(format) + .subresource_range( + vk::ImageSubresourceRange::default() + .aspect_mask(aspect) + .level_count(1) + .layer_count(1), + ), + None, + ) + } + .context("vk-frame plane view") + }; + let luma = make(vk::ImageAspectFlags::PLANE_0, vk::Format::R8_UNORM)?; + let chroma = match make(vk::ImageAspectFlags::PLANE_1, vk::Format::R8G8_UNORM) { + Ok(v) => v, + Err(e) => { + unsafe { self.device.destroy_image_view(luma, None) }; + return Err(e); + } + }; + Ok([luma, chroma]) + } + /// Copy the frame's RGBA into the staging buffer and (re)build the video image on a /// stream-size change. Rows keep their stride — `buffer_row_length` unpacks it. fn stage_frame(&mut self, f: &CpuFrame) -> Result<()> { @@ -947,34 +1272,30 @@ impl Presenter { let reqs = unsafe { self.device.get_image_memory_requirements(image) }; let memory = self.allocate(reqs, vk::MemoryPropertyFlags::DEVICE_LOCAL)?; unsafe { self.device.bind_image_memory(image, memory, 0) }?; - // The CSC pass renders into it — view + framebuffer, hw-capable devices only. - let (view, framebuffer) = if let Some(hw) = &self.hw { - let view = unsafe { - self.device.create_image_view( - &vk::ImageViewCreateInfo::default() - .image(image) - .view_type(vk::ImageViewType::TYPE_2D) - .format(vk::Format::R8G8B8A8_UNORM) - .subresource_range(subresource_range()), - None, - ) - }?; - let attachments = [view]; - let framebuffer = unsafe { - self.device.create_framebuffer( - &vk::FramebufferCreateInfo::default() - .render_pass(hw.csc.render_pass) - .attachments(&attachments) - .width(width) - .height(height) - .layers(1), - None, - ) - }?; - (view, framebuffer) - } else { - (vk::ImageView::null(), vk::Framebuffer::null()) - }; + // The CSC pass renders into it — view + framebuffer, unconditional (Vulkan-Video + // frames need the pass on every device, dmabuf-capable or not). + let view = unsafe { + self.device.create_image_view( + &vk::ImageViewCreateInfo::default() + .image(image) + .view_type(vk::ImageViewType::TYPE_2D) + .format(vk::Format::R8G8B8A8_UNORM) + .subresource_range(subresource_range()), + None, + ) + }?; + let attachments = [view]; + let framebuffer = unsafe { + self.device.create_framebuffer( + &vk::FramebufferCreateInfo::default() + .render_pass(self.csc.render_pass) + .attachments(&attachments) + .width(width) + .height(height) + .layers(1), + None, + ) + }?; self.video = Some(VideoImage { image, memory, @@ -1071,9 +1392,8 @@ impl Drop for Presenter { self.device.destroy_image(v.image, None); self.device.free_memory(v.memory, None); } - if let Some(hw) = self.hw.take() { - hw.csc.destroy(&self.device); - } + self.hw.take(); + self.csc.destroy(&self.device); self.overlay_pipe.destroy(&self.device); for s in self.render_sems.drain(..) { self.device.destroy_semaphore(s, None); @@ -1195,6 +1515,95 @@ fn subresource_range() -> vk::ImageSubresourceRange { .layer_count(1) } + +/// The live sync state of an `AVVkFrame`, snapshotted under the frames lock. +struct VkFrameSync { + image: u64, + semaphore: u64, + sem_value: u64, + layout: i32, + queue_family: u32, +} + +/// Lock the frame and read its live sync state (the presenter's submit must wait +/// `sem_value` and signal `sem_value + 1`). The lock is held until [`unlock_vkframe`]. +fn lock_vkframe(f: &VkVideoFrame) -> VkFrameSync { + unsafe { + let lock: unsafe extern "C" fn(*mut pf_ffvk::AVHWFramesContext, *mut pf_ffvk::AVVkFrame) = + std::mem::transmute(f.lock_frame); + let fc = f.frames_ctx as *mut pf_ffvk::AVHWFramesContext; + let vkf = f.vkframe as *mut pf_ffvk::AVVkFrame; + lock(fc, vkf); + VkFrameSync { + image: (*vkf).img[0] as u64, + semaphore: (*vkf).sem[0] as u64, + sem_value: (*vkf).sem_value[0], + layout: (*vkf).layout[0] as i32, + queue_family: (*vkf).queue_family[0], + } + } +} + +/// Write the post-submission state back (FFmpeg waits these on its next use of the +/// frame) and release the lock. On a failed submit only the lock is released. +fn unlock_vkframe(f: &VkVideoFrame, sync: &VkFrameSync, submitted: bool, graphics_qf: u32) { + unsafe { + let vkf = f.vkframe as *mut pf_ffvk::AVVkFrame; + if submitted { + (*vkf).sem_value[0] = sync.sem_value + 1; + (*vkf).layout[0] = + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL.as_raw() as pf_ffvk::VkImageLayout; + if sync.queue_family != vk::QUEUE_FAMILY_IGNORED { + (*vkf).queue_family[0] = graphics_qf; + } + } + let unlock: unsafe extern "C" fn( + *mut pf_ffvk::AVHWFramesContext, + *mut pf_ffvk::AVVkFrame, + ) = std::mem::transmute(f.unlock_frame); + unlock(f.frames_ctx as *mut pf_ffvk::AVHWFramesContext, vkf); + } +} + +/// Acquire a Vulkan-Video frame's image from the decode queue family (EXCLUSIVE +/// sharing) and transition it for sampling. `src_qf == dst_qf` (or IGNORED/CONCURRENT) +/// degrades to a plain layout transition. The matching decode-side acquire happens in +/// FFmpeg, keyed off the queue_family we write back after submission. +fn vkframe_acquire_barrier( + device: &ash::Device, + cmd: vk::CommandBuffer, + image: vk::Image, + old_layout: vk::ImageLayout, + src_qf: u32, + dst_qf: u32, +) { + let (src, dst) = if src_qf == dst_qf || src_qf == vk::QUEUE_FAMILY_IGNORED { + (vk::QUEUE_FAMILY_IGNORED, vk::QUEUE_FAMILY_IGNORED) + } else { + (src_qf, dst_qf) + }; + let b = vk::ImageMemoryBarrier::default() + .src_access_mask(vk::AccessFlags::empty()) + .dst_access_mask(vk::AccessFlags::SHADER_READ) + .old_layout(old_layout) + .new_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) + .src_queue_family_index(src) + .dst_queue_family_index(dst) + .image(image) + .subresource_range(subresource_range()); + unsafe { + device.cmd_pipeline_barrier( + cmd, + vk::PipelineStageFlags::TOP_OF_PIPE, + vk::PipelineStageFlags::FRAGMENT_SHADER, + vk::DependencyFlags::empty(), + &[], + &[], + &[b], + ); + } +} + /// Acquire a dmabuf plane image from its foreign owner (the VAAPI decoder): queue-family /// transfer FOREIGN → ours, UNDEFINED → SHADER_READ_ONLY (content is preserved across /// the transfer regardless of the UNDEFINED old-layout, per the external-memory rules).