feat(video): Vulkan Video decode on the presenter's device (NVIDIA hw decode)

FFmpeg's Vulkan Video decoder now runs on the PRESENTER's own VkDevice —
the decoded VkImage feeds the existing CICP CSC pass directly: zero
copy, no interop, and NVIDIA gets hardware decode for the first time
(its VAAPI is unusable by design). One decode architecture for every
vendor going forward; VAAPI-dmabuf and software remain the fallbacks
(auto: vulkan → vaapi → software; PUNKTFUNK_DECODER=vulkan pins it).

Presenter: instance 1.3; probes VK_KHR_video_queue/decode_queue + codec
extensions, a VIDEO_DECODE queue family (+ its codec caps via
QueueFamilyVideoPropertiesKHR), and the samplerYcbcrConversion/
timelineSemaphore/synchronization2 features — all enabled at device
creation when present, exported as a VulkanDecodeDevice handle bundle.

Decoder: AVVulkanDeviceContext built over those handles via pf-ffvk
(features chain, extension lists, deprecated queue indices + the qf[]
map); get_format supplies OUR frames context with MUTABLE_FORMAT so the
presenter's per-plane views are legal; output is DecodedImage::VkFrame
carrying AVVkFrame/frames-ctx pointers plus the lock fns.

Present: R8+R8G8 plane views over the multiplanar image, the live sync
state read under the AVVulkanFramesContext lock, a timeline-semaphore
wait(sem_value)/signal(sem_value+1) folded into the submit, layout/
queue-family/sem_value written back per FFmpeg's contract, and the frame
guard parked in the retire queue until the fence. CSC pass + video
framebuffer are now unconditional (NVIDIA has no dmabuf-import path).

Verified on the RTX 5070 Ti: device creates with decode_qf=3,
caps=DECODE_H264|H265|AV1|VP9; swapchain unaffected. Live stream
validation next.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-07 21:46:34 +02:00
parent 58ccd530fc
commit c78ddc40cb
10 changed files with 1087 additions and 191 deletions
+69 -12
View File
@@ -16,6 +16,7 @@ use crate::overlay::{FrameCtx, Overlay, OverlayAction, OverlayFrame, SessionPhas
use crate::vk::{FrameInput, Presenter};
use anyhow::{Context as _, Result};
use pf_client_core::gamepad::GamepadService;
use pf_client_core::video::VulkanDecodeDevice;
use pf_client_core::session::{self, SessionEvent, SessionHandle, SessionParams, Stats};
use pf_client_core::video::{DecodedFrame, DecodedImage};
use punktfunk_core::client::NativeClient;
@@ -58,8 +59,8 @@ pub enum ActionOutcome {
/// Consumed binary-side (a Retry respawned the fetch, …).
Handled,
/// Start this session (a Launch action; `force_software` from the callback args is
/// wired into these params).
Start(SessionParams),
/// wired into these params). Boxed: SessionParams is large next to the unit variants.
Start(Box<SessionParams>),
/// Quit the launcher.
Quit,
}
@@ -67,13 +68,13 @@ pub enum ActionOutcome {
/// One `--connect` stream session; returns when it ends (the shell↔session contract).
pub fn run_session<F>(opts: SessionOpts, build_params: F) -> Result<Outcome>
where
F: FnOnce(&GamepadService, Mode, Arc<AtomicBool>) -> SessionParams,
F: FnOnce(&GamepadService, Mode, Arc<AtomicBool>, Option<VulkanDecodeDevice>) -> SessionParams,
{
let mut build = Some(build_params);
run_inner(
opts,
ModeCtl::Single(Box::new(move |gp, native, fs| {
(build.take().expect("single build runs once"))(gp, native, fs)
ModeCtl::Single(Box::new(move |gp, native, fs, vk| {
(build.take().expect("single build runs once"))(gp, native, fs, vk)
})),
)
.map(|o| o.expect("single mode always yields an outcome"))
@@ -85,7 +86,13 @@ where
/// per-session `force_software` flag.
pub fn run_browse<F>(opts: SessionOpts, on_action: F) -> Result<()>
where
F: FnMut(OverlayAction, &GamepadService, Mode, Arc<AtomicBool>) -> ActionOutcome,
F: FnMut(
OverlayAction,
&GamepadService,
Mode,
Arc<AtomicBool>,
Option<VulkanDecodeDevice>,
) -> ActionOutcome,
{
anyhow::ensure!(
opts.overlay.is_some(),
@@ -95,10 +102,21 @@ where
}
/// Params builder for the one single-mode session (called exactly once, post-setup).
type BuildParams<'a> = Box<dyn FnMut(&GamepadService, Mode, Arc<AtomicBool>) -> SessionParams + 'a>;
type BuildParams<'a> = Box<
dyn FnMut(&GamepadService, Mode, Arc<AtomicBool>, Option<VulkanDecodeDevice>) -> SessionParams
+ 'a,
>;
/// The browse-mode action callback (Launch → params, Retry/Quit → outcome).
type OnAction<'a> =
Box<dyn FnMut(OverlayAction, &GamepadService, Mode, Arc<AtomicBool>) -> ActionOutcome + 'a>;
type OnAction<'a> = Box<
dyn FnMut(
OverlayAction,
&GamepadService,
Mode,
Arc<AtomicBool>,
Option<VulkanDecodeDevice>,
) -> ActionOutcome
+ 'a,
>;
/// The two run modes, type-erased so one loop serves both.
enum ModeCtl<'a> {
@@ -227,7 +245,12 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
let mut stream: Option<StreamState> = match &mut mode {
ModeCtl::Single(build) => {
let force_software = Arc::new(AtomicBool::new(false));
let params = build(&gamepad, native, force_software.clone());
let params = build(
&gamepad,
native,
force_software.clone(),
presenter.vulkan_decode(),
);
Some(StreamState::new(params, force_software))
}
ModeCtl::Browse(_) => None,
@@ -416,10 +439,16 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
}
if let Some(action) = overlay.as_mut().and_then(|o| o.take_action()) {
let force_software = Arc::new(AtomicBool::new(false));
match on_action(action, &gamepad, native, force_software.clone()) {
match on_action(
action,
&gamepad,
native,
force_software.clone(),
presenter.vulkan_decode(),
) {
ActionOutcome::Handled => {}
ActionOutcome::Start(params) => {
stream = Some(StreamState::new(params, force_software));
stream = Some(StreamState::new(*params, force_software));
if let Some(o) = overlay.as_mut() {
o.session_phase(SessionPhase::Connecting);
}
@@ -605,6 +634,34 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
}
false
}
// Vulkan-Video: decoded on the presenter's own device — present is
// views + CSC, no import step to gate on. Same failure-streak
// demotion contract as the dmabuf path.
DecodedImage::VkFrame(v) if !st.dmabuf_demoted => {
st.hdr = v.color.is_pq();
match presenter.present(
&window,
FrameInput::VkFrame(v),
overlay_frame.as_ref(),
) {
Ok(p) => {
st.hw_fails = 0;
p
}
Err(e) => {
st.hw_fails += 1;
tracing::warn!(error = %format!("{e:#}"), fails = st.hw_fails,
"vulkan-video present failed");
if st.hw_fails >= 3 {
st.dmabuf_demoted = true;
tracing::warn!("demoting the decoder to software");
st.force_software.store(true, Ordering::Relaxed);
}
false
}
}
}
DecodedImage::VkFrame(_) => false, // demoted — drain until rebuild
};
if did_present {
presented_video = true;