feat(video): Vulkan Video decode on the presenter's device (NVIDIA hw decode)
FFmpeg's Vulkan Video decoder now runs on the PRESENTER's own VkDevice — the decoded VkImage feeds the existing CICP CSC pass directly: zero copy, no interop, and NVIDIA gets hardware decode for the first time (its VAAPI is unusable by design). One decode architecture for every vendor going forward; VAAPI-dmabuf and software remain the fallbacks (auto: vulkan → vaapi → software; PUNKTFUNK_DECODER=vulkan pins it). Presenter: instance 1.3; probes VK_KHR_video_queue/decode_queue + codec extensions, a VIDEO_DECODE queue family (+ its codec caps via QueueFamilyVideoPropertiesKHR), and the samplerYcbcrConversion/ timelineSemaphore/synchronization2 features — all enabled at device creation when present, exported as a VulkanDecodeDevice handle bundle. Decoder: AVVulkanDeviceContext built over those handles via pf-ffvk (features chain, extension lists, deprecated queue indices + the qf[] map); get_format supplies OUR frames context with MUTABLE_FORMAT so the presenter's per-plane views are legal; output is DecodedImage::VkFrame carrying AVVkFrame/frames-ctx pointers plus the lock fns. Present: R8+R8G8 plane views over the multiplanar image, the live sync state read under the AVVulkanFramesContext lock, a timeline-semaphore wait(sem_value)/signal(sem_value+1) folded into the submit, layout/ queue-family/sem_value written back per FFmpeg's contract, and the frame guard parked in the retire queue until the fence. CSC pass + video framebuffer are now unconditional (NVIDIA has no dmabuf-import path). Verified on the RTX 5070 Ti: device creates with decode_qf=3, caps=DECODE_H264|H265|AV1|VP9; swapchain unaffected. Live stream validation next. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Generated
+2
@@ -2774,6 +2774,7 @@ dependencies = [
|
||||
"ffmpeg-next",
|
||||
"mdns-sd",
|
||||
"opus",
|
||||
"pf-ffvk",
|
||||
"pipewire",
|
||||
"punktfunk-core",
|
||||
"rustls",
|
||||
@@ -2821,6 +2822,7 @@ dependencies = [
|
||||
"ash",
|
||||
"async-channel",
|
||||
"pf-client-core",
|
||||
"pf-ffvk",
|
||||
"punktfunk-core",
|
||||
"sdl3",
|
||||
"tracing",
|
||||
|
||||
@@ -66,7 +66,7 @@ pub fn run(target: &str) -> u8 {
|
||||
overlay: Some(Box::new(overlay)),
|
||||
};
|
||||
|
||||
let result = pf_presenter::run_browse(opts, |action, gamepad, native, force_software| {
|
||||
let result = pf_presenter::run_browse(opts, |action, gamepad, native, force_software, vulkan| {
|
||||
match action {
|
||||
OverlayAction::Launch { id, title } => {
|
||||
// The carousel only renders for a paired host, so the pin exists; the
|
||||
@@ -76,7 +76,7 @@ pub fn run(target: &str) -> u8 {
|
||||
return ActionOutcome::Handled;
|
||||
};
|
||||
tracing::info!(%id, %title, "launching from the library");
|
||||
ActionOutcome::Start(session_params(
|
||||
ActionOutcome::Start(Box::new(session_params(
|
||||
&settings,
|
||||
addr.clone(),
|
||||
port,
|
||||
@@ -86,7 +86,8 @@ pub fn run(target: &str) -> u8 {
|
||||
gamepad,
|
||||
native,
|
||||
force_software,
|
||||
))
|
||||
vulkan,
|
||||
)))
|
||||
}
|
||||
OverlayAction::Retry => {
|
||||
spawn_fetch(shared.clone(), addr.clone(), mgmt, identity.clone(), pin);
|
||||
|
||||
@@ -89,6 +89,7 @@ mod session_main {
|
||||
gamepad: &GamepadService,
|
||||
native: Mode,
|
||||
force_software: Arc<AtomicBool>,
|
||||
vulkan: Option<pf_client_core::video::VulkanDecodeDevice>,
|
||||
) -> SessionParams {
|
||||
let mode = Mode {
|
||||
width: if settings.width == 0 {
|
||||
@@ -126,6 +127,7 @@ mod session_main {
|
||||
// PUNKTFUNK_DECODER still overrides inside the decoder for bisects.
|
||||
decoder: settings.decoder.clone(),
|
||||
launch,
|
||||
vulkan,
|
||||
pin: Some(pin),
|
||||
identity,
|
||||
connect_timeout: connect_timeout(),
|
||||
@@ -259,19 +261,21 @@ mod session_main {
|
||||
overlay: None,
|
||||
};
|
||||
|
||||
let outcome = pf_presenter::run_session(opts, move |gamepad, native, force_software| {
|
||||
session_params(
|
||||
&settings,
|
||||
addr,
|
||||
port,
|
||||
pin,
|
||||
identity,
|
||||
launch,
|
||||
gamepad,
|
||||
native,
|
||||
force_software,
|
||||
)
|
||||
});
|
||||
let outcome =
|
||||
pf_presenter::run_session(opts, move |gamepad, native, force_software, vulkan| {
|
||||
session_params(
|
||||
&settings,
|
||||
addr,
|
||||
port,
|
||||
pin,
|
||||
identity,
|
||||
launch,
|
||||
gamepad,
|
||||
native,
|
||||
force_software,
|
||||
vulkan,
|
||||
)
|
||||
});
|
||||
|
||||
match outcome {
|
||||
Ok(pf_presenter::Outcome::Ended(None)) => 0,
|
||||
|
||||
@@ -13,6 +13,8 @@ repository.workspace = true
|
||||
# emptiness. `wol` is pure std and stays cross-platform, matching the old main.rs.
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
punktfunk-core = { path = "../punktfunk-core", features = ["quic"] }
|
||||
# FFmpeg's Vulkan hwcontext surface (Vulkan Video decode on the presenter's device).
|
||||
pf-ffvk = { path = "../pf-ffvk" }
|
||||
async-channel = "2"
|
||||
|
||||
# Video decode (same FFmpeg pin as the host) and audio.
|
||||
|
||||
@@ -35,6 +35,9 @@ pub struct SessionParams {
|
||||
/// Library id for the host to launch this session (`"steam:570"`, from the library
|
||||
/// page); `None` = plain desktop session.
|
||||
pub launch: Option<String>,
|
||||
/// The presenter's shared Vulkan device, when its stack can run FFmpeg's Vulkan
|
||||
/// Video decoder (decode lands as VkImages the presenter samples directly).
|
||||
pub vulkan: Option<crate::video::VulkanDecodeDevice>,
|
||||
/// Pinned host fingerprint; `None` = trust on first use (caller persists the observed one).
|
||||
pub pin: Option<[u8; 32]>,
|
||||
pub identity: (String, String),
|
||||
@@ -241,7 +244,7 @@ fn pump(
|
||||
welcome_codec = connector.codec,
|
||||
"negotiated video codec"
|
||||
);
|
||||
let mut decoder = match Decoder::new(codec_id, ¶ms.decoder) {
|
||||
let mut decoder = match Decoder::new(codec_id, ¶ms.decoder, params.vulkan.as_ref()) {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
let _ = ev_tx.send_blocking(SessionEvent::Ended(Some(format!("video decoder: {e}"))));
|
||||
@@ -314,11 +317,13 @@ fn pump(
|
||||
dec_path = match &image {
|
||||
DecodedImage::Cpu(_) => "software",
|
||||
DecodedImage::Dmabuf(_) => "vaapi",
|
||||
DecodedImage::VkFrame(_) => "vulkan",
|
||||
};
|
||||
if total_frames == 1 {
|
||||
let (w, h, path) = match &image {
|
||||
DecodedImage::Cpu(c) => (c.width, c.height, "software"),
|
||||
DecodedImage::Dmabuf(d) => (d.width, d.height, "vaapi-dmabuf"),
|
||||
DecodedImage::VkFrame(v) => (v.width, v.height, "vulkan-video"),
|
||||
};
|
||||
tracing::info!(width = w, height = h, path, "first frame decoded");
|
||||
}
|
||||
|
||||
@@ -39,6 +39,38 @@ pub struct DecodedFrame {
|
||||
pub enum DecodedImage {
|
||||
Cpu(CpuFrame),
|
||||
Dmabuf(DmabufFrame),
|
||||
/// FFmpeg Vulkan Video output: a VkImage already on the PRESENTER's device.
|
||||
VkFrame(VkVideoFrame),
|
||||
}
|
||||
|
||||
/// One Vulkan-decoded frame. The image lives on the presenter's own VkDevice (the
|
||||
/// decoder was built over its handles), so presenting is: plane views → CSC pass — no
|
||||
/// import, no copy. The live synchronization state (layout / timeline value / owning
|
||||
/// queue family) is deliberately NOT snapshotted here: FFmpeg updates it per submission,
|
||||
/// so the presenter reads it through `vkframe` under the frames-context lock at ITS
|
||||
/// submit time (the `AVVulkanFramesContext.lock_frame` contract).
|
||||
pub struct VkVideoFrame {
|
||||
/// `AVVkFrame*` — img[0] is the (multiplanar) image; sem/sem_value/layout/
|
||||
/// queue_family are the live sync state. Valid while `guard` lives.
|
||||
pub vkframe: usize,
|
||||
/// `AVHWFramesContext*` (FFmpeg's) — the first argument to the lock functions.
|
||||
/// Valid while `guard` lives.
|
||||
pub frames_ctx: usize,
|
||||
/// `AVVulkanFramesContext.lock_frame` / `.unlock_frame` (filled in by FFmpeg's
|
||||
/// init): the presenter MUST hold the lock while reading the live sync state and
|
||||
/// writing back the incremented semaphore value around its submission.
|
||||
pub lock_frame: usize,
|
||||
pub unlock_frame: usize,
|
||||
/// The frame pool's VkFormat (`AVVulkanFramesContext.format[0]`, raw i32) — the
|
||||
/// multiplanar format the presenter builds its per-plane views against.
|
||||
pub vk_format: i32,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub color: ColorDesc,
|
||||
/// Keeps the cloned AVFrame (and through it the VkImage + frames context) alive
|
||||
/// until the presenter's fence proves the GPU reads done — same mechanism as the
|
||||
/// VAAPI path's DRM guard.
|
||||
pub guard: DrmFrameGuard,
|
||||
}
|
||||
|
||||
/// The stream's colour signaling, read PER-FRAME from the decoder (HEVC VUI → the
|
||||
@@ -127,6 +159,7 @@ impl Drop for DrmFrameGuard {
|
||||
}
|
||||
|
||||
enum Backend {
|
||||
Vulkan(VulkanDecoder),
|
||||
Vaapi(VaapiDecoder),
|
||||
Software(SoftwareDecoder),
|
||||
}
|
||||
@@ -136,8 +169,9 @@ pub struct Decoder {
|
||||
/// The negotiated codec (from the host's Welcome), so a mid-session VAAPI→software demotion
|
||||
/// rebuilds the software decoder for the SAME codec.
|
||||
codec_id: ffmpeg::codec::Id,
|
||||
/// Consecutive VAAPI decode errors — a single transient failure (e.g. a reference-missing
|
||||
/// frame after packet loss) shouldn't cost the whole session its hardware decoder.
|
||||
/// Consecutive hardware decode errors (Vulkan or VAAPI) — a single transient failure
|
||||
/// (e.g. a reference-missing frame after packet loss) shouldn't cost the whole
|
||||
/// session its hardware decoder.
|
||||
vaapi_fails: u32,
|
||||
/// Set when the decoder needs a fresh IDR to resynchronize (after an error or a demotion).
|
||||
/// The pump drains it and asks the host — under the infinite GOP there is no periodic
|
||||
@@ -196,33 +230,64 @@ fn quiet_ffmpeg_log() {
|
||||
|
||||
impl Decoder {
|
||||
/// `codec_id` is the codec the host resolved in the Welcome (never assume HEVC).
|
||||
/// `pref` is the Settings "Video decoder" value (`auto`/`vaapi`/`software`).
|
||||
/// `pref` is the Settings "Video decoder" value (`auto`/`vulkan`/`vaapi`/`software`).
|
||||
/// `vk` is the presenter's shared Vulkan device when its stack can run FFmpeg's
|
||||
/// Vulkan Video decoder — decode lands as VkImages the presenter samples directly.
|
||||
/// Precedence: the `PUNKTFUNK_DECODER` env override wins (support/debug escape
|
||||
/// hatch, and the documented knob), then the setting; both default to auto
|
||||
/// (VAAPI → software).
|
||||
pub fn new(codec_id: ffmpeg::codec::Id, pref: &str) -> Result<Decoder> {
|
||||
/// (Vulkan → VAAPI → software).
|
||||
pub fn new(
|
||||
codec_id: ffmpeg::codec::Id,
|
||||
pref: &str,
|
||||
vk: Option<&VulkanDecodeDevice>,
|
||||
) -> Result<Decoder> {
|
||||
ffmpeg::init().context("ffmpeg init")?;
|
||||
quiet_ffmpeg_log();
|
||||
let choice = std::env::var("PUNKTFUNK_DECODER")
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty())
|
||||
.unwrap_or_else(|| pref.to_string());
|
||||
// Deck note: `auto` means VAAPI here too. GTK's tiled-NV12 dmabuf import is broken on
|
||||
// the Deck (Mesa ≥ 25.1 exports VCN surfaces TILED; artifacts/gray/washed-out), but the
|
||||
// presenter routes Deck frames through the in-process GL converter (`video_gl`) instead
|
||||
// of GdkDmabufTexture — and if THAT can't initialize, it demotes this decoder to
|
||||
// software mid-session via [`Decoder::force_software`]. The broken direct path is never
|
||||
// the fallback.
|
||||
if choice != "software" {
|
||||
let done = |backend| {
|
||||
Ok(Decoder {
|
||||
backend,
|
||||
codec_id,
|
||||
vaapi_fails: 0,
|
||||
want_keyframe: false,
|
||||
})
|
||||
};
|
||||
if matches!(choice.as_str(), "auto" | "" | "vulkan") {
|
||||
match vk {
|
||||
Some(vk) => match VulkanDecoder::new(codec_id, vk) {
|
||||
Ok(v) => {
|
||||
tracing::info!(
|
||||
?codec_id,
|
||||
"Vulkan Video hardware decode active (presenter-shared device)"
|
||||
);
|
||||
return done(Backend::Vulkan(v));
|
||||
}
|
||||
Err(e) => {
|
||||
if choice == "vulkan" {
|
||||
return Err(e.context("PUNKTFUNK_DECODER=vulkan but it failed"));
|
||||
}
|
||||
tracing::info!(reason = %format!("{e:#}"),
|
||||
"Vulkan Video unavailable — trying VAAPI");
|
||||
}
|
||||
},
|
||||
None if choice == "vulkan" => {
|
||||
bail!("PUNKTFUNK_DECODER=vulkan but the presenter's device can't (missing \
|
||||
video extensions/queue) — see the presenter log")
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
// Deck note: `auto` reaches VAAPI when Vulkan Video isn't available. A presenter
|
||||
// that can't display the dmabufs demotes this decoder to software mid-session
|
||||
// via [`Decoder::force_software`].
|
||||
if choice != "software" && choice != "vulkan" {
|
||||
match VaapiDecoder::new(codec_id) {
|
||||
Ok(v) => {
|
||||
tracing::info!(?codec_id, "VAAPI hardware decode active (zero-copy dmabuf)");
|
||||
return Ok(Decoder {
|
||||
backend: Backend::Vaapi(v),
|
||||
codec_id,
|
||||
vaapi_fails: 0,
|
||||
want_keyframe: false,
|
||||
});
|
||||
return done(Backend::Vaapi(v));
|
||||
}
|
||||
Err(e) => {
|
||||
if choice == "vaapi" {
|
||||
@@ -232,12 +297,7 @@ impl Decoder {
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Decoder {
|
||||
backend: Backend::Software(SoftwareDecoder::new(codec_id)?),
|
||||
codec_id,
|
||||
vaapi_fails: 0,
|
||||
want_keyframe: false,
|
||||
})
|
||||
done(Backend::Software(SoftwareDecoder::new(codec_id)?))
|
||||
}
|
||||
|
||||
/// Drain the "please ask the host for an IDR" flag — the pump calls this each iteration
|
||||
@@ -269,28 +329,34 @@ impl Decoder {
|
||||
/// pump asks the host for a fresh IDR — under the infinite GOP nothing else resyncs a
|
||||
/// rebuilt/erroring decoder, so skipping this leaves the picture gray/frozen for good.
|
||||
pub fn decode(&mut self, au: &[u8]) -> Result<Option<DecodedImage>> {
|
||||
match &mut self.backend {
|
||||
Backend::Vaapi(v) => match v.decode(au) {
|
||||
Ok(f) => {
|
||||
let result = match &mut self.backend {
|
||||
Backend::Vulkan(v) => v.decode(au).map(|f| f.map(DecodedImage::VkFrame)),
|
||||
Backend::Vaapi(v) => v.decode(au).map(|f| f.map(DecodedImage::Dmabuf)),
|
||||
Backend::Software(s) => return Ok(s.decode(au)?.map(DecodedImage::Cpu)),
|
||||
};
|
||||
match result {
|
||||
Ok(f) => {
|
||||
self.vaapi_fails = 0;
|
||||
Ok(f)
|
||||
}
|
||||
Err(e) => {
|
||||
let which = match self.backend {
|
||||
Backend::Vulkan(_) => "Vulkan Video",
|
||||
_ => "VAAPI",
|
||||
};
|
||||
self.vaapi_fails += 1;
|
||||
self.want_keyframe = true;
|
||||
if self.vaapi_fails >= VAAPI_DEMOTE_AFTER {
|
||||
tracing::warn!(error = %e, fails = self.vaapi_fails,
|
||||
"{which} decode failing repeatedly — demoting to software");
|
||||
self.backend = Backend::Software(SoftwareDecoder::new(self.codec_id)?);
|
||||
self.vaapi_fails = 0;
|
||||
Ok(f.map(DecodedImage::Dmabuf))
|
||||
} else {
|
||||
tracing::warn!(error = %e,
|
||||
"{which} decode error — requesting keyframe, keeping hardware decode");
|
||||
}
|
||||
Err(e) => {
|
||||
self.vaapi_fails += 1;
|
||||
self.want_keyframe = true;
|
||||
if self.vaapi_fails >= VAAPI_DEMOTE_AFTER {
|
||||
tracing::warn!(error = %e, fails = self.vaapi_fails,
|
||||
"VAAPI decode failing repeatedly — demoting to software");
|
||||
self.backend = Backend::Software(SoftwareDecoder::new(self.codec_id)?);
|
||||
self.vaapi_fails = 0;
|
||||
} else {
|
||||
tracing::warn!(error = %e,
|
||||
"VAAPI decode error — requesting keyframe, keeping hardware decode");
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
},
|
||||
Backend::Software(s) => Ok(s.decode(au)?.map(DecodedImage::Cpu)),
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -628,6 +694,39 @@ impl VaapiDecoder {
|
||||
}
|
||||
}
|
||||
|
||||
/// The presenter's Vulkan device handles, exported so FFmpeg's Vulkan Video decoder
|
||||
/// runs on the SAME device the presenter samples from — the whole point: the decoded
|
||||
/// VkImage is composited directly, no interop, no copy (plan: Vulkan Video phase).
|
||||
///
|
||||
/// Plain integers/strings on purpose: pf-client-core has no ash dependency; pf-ffvk
|
||||
/// casts these into vulkan.h handle types when filling `AVVulkanDeviceContext`. All
|
||||
/// handles stay valid for the presenter's lifetime, which outlives every session pump
|
||||
/// (the run loop tears the pump down before the presenter).
|
||||
#[derive(Clone)]
|
||||
pub struct VulkanDecodeDevice {
|
||||
/// `PFN_vkGetInstanceProcAddr` from the loader — FFmpeg resolves everything else.
|
||||
pub get_instance_proc_addr: usize,
|
||||
pub instance: usize,
|
||||
pub physical_device: usize,
|
||||
pub device: usize,
|
||||
/// The presenter's graphics+present family (FFmpeg's "required" tx/comp family too).
|
||||
pub graphics_qf: u32,
|
||||
/// Raw `VkQueueFlags` of that family (the qf[] entry wants the real capabilities).
|
||||
pub graphics_queue_flags: u32,
|
||||
/// The video-decode family (may equal `graphics_qf` on some hardware).
|
||||
pub decode_qf: u32,
|
||||
/// Raw `VkVideoCodecOperationFlagsKHR` the decode family advertises.
|
||||
pub decode_video_caps: u32,
|
||||
/// Everything enabled at instance/device creation — FFmpeg keys code paths off the
|
||||
/// extension STRINGS, so the lists must match reality exactly.
|
||||
pub instance_extensions: Vec<std::ffi::CString>,
|
||||
pub device_extensions: Vec<std::ffi::CString>,
|
||||
/// Features enabled at device creation (reported via `device_features`).
|
||||
pub f_sampler_ycbcr: bool,
|
||||
pub f_timeline_semaphore: bool,
|
||||
pub f_synchronization2: bool,
|
||||
}
|
||||
|
||||
/// `fourcc(a,b,c,d)` — the DRM FourCC packing (little-endian, `a | b<<8 | c<<16 | d<<24`).
|
||||
const fn fourcc(a: u8, b: u8, c: u8, d: u8) -> u32 {
|
||||
(a as u32) | ((b as u32) << 8) | ((c as u32) << 16) | ((d as u32) << 24)
|
||||
@@ -684,6 +783,314 @@ impl Drop for VaapiDecoder {
|
||||
}
|
||||
}
|
||||
|
||||
// --- Vulkan Video backend -------------------------------------------------------------
|
||||
|
||||
/// FFmpeg's Vulkan Video decoder over the PRESENTER's device: the hwdevice context is
|
||||
/// built from [`VulkanDecodeDevice`]'s handles (not `av_hwdevice_ctx_create`, which
|
||||
/// would make FFmpeg create its own device the presenter can't sample from). Output
|
||||
/// frames are `AVVkFrame`s whose VkImage the presenter feeds straight to its CSC pass.
|
||||
struct VulkanDecoder {
|
||||
ctx: *mut ffmpeg::ffi::AVCodecContext,
|
||||
hw_device: *mut ffmpeg::ffi::AVBufferRef,
|
||||
packet: *mut ffmpeg::ffi::AVPacket,
|
||||
frame: *mut ffmpeg::ffi::AVFrame,
|
||||
/// Storage `AVVulkanDeviceContext` points into (extension string arrays + the
|
||||
/// feature chain) — FFmpeg reads the extension lists past init (frames-context
|
||||
/// setup keys code paths off them), so this lives exactly as long as `hw_device`.
|
||||
_ctx_storage: Box<VkCtxStorage>,
|
||||
}
|
||||
|
||||
// Single-owner pointers, only touched from the session pump thread.
|
||||
unsafe impl Send for VulkanDecoder {}
|
||||
|
||||
struct VkCtxStorage {
|
||||
_inst: Vec<std::ffi::CString>,
|
||||
inst_ptrs: Vec<*const std::os::raw::c_char>,
|
||||
_dev: Vec<std::ffi::CString>,
|
||||
dev_ptrs: Vec<*const std::os::raw::c_char>,
|
||||
f11: pf_ffvk::VkPhysicalDeviceVulkan11Features,
|
||||
f12: pf_ffvk::VkPhysicalDeviceVulkan12Features,
|
||||
f13: pf_ffvk::VkPhysicalDeviceVulkan13Features,
|
||||
}
|
||||
|
||||
impl VulkanDecoder {
|
||||
fn new(codec_id: ffmpeg::codec::Id, vk: &VulkanDecodeDevice) -> Result<VulkanDecoder> {
|
||||
use ffmpeg::ffi;
|
||||
unsafe {
|
||||
let mut hw_device =
|
||||
ffi::av_hwdevice_ctx_alloc(ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VULKAN);
|
||||
if hw_device.is_null() {
|
||||
bail!("av_hwdevice_ctx_alloc(VULKAN) failed (FFmpeg built without Vulkan?)");
|
||||
}
|
||||
let devctx = (*hw_device).data as *mut ffi::AVHWDeviceContext;
|
||||
let hwctx = (*devctx).hwctx as *mut pf_ffvk::AVVulkanDeviceContext;
|
||||
|
||||
// Pinned storage for everything the context points into.
|
||||
let mut store = Box::new(VkCtxStorage {
|
||||
_inst: vk.instance_extensions.clone(),
|
||||
inst_ptrs: Vec::new(),
|
||||
_dev: vk.device_extensions.clone(),
|
||||
dev_ptrs: Vec::new(),
|
||||
f11: std::mem::zeroed(),
|
||||
f12: std::mem::zeroed(),
|
||||
f13: std::mem::zeroed(),
|
||||
});
|
||||
store.inst_ptrs = store._inst.iter().map(|c| c.as_ptr()).collect();
|
||||
store.dev_ptrs = store._dev.iter().map(|c| c.as_ptr()).collect();
|
||||
// The features enabled at device creation, as the 1.1/1.2/1.3 chain FFmpeg
|
||||
// walks to learn what it may use (sType values are vulkan.h constants).
|
||||
store.f11.sType =
|
||||
pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES;
|
||||
store.f11.samplerYcbcrConversion = vk.f_sampler_ycbcr as u32;
|
||||
store.f12.sType =
|
||||
pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES;
|
||||
store.f12.timelineSemaphore = vk.f_timeline_semaphore as u32;
|
||||
store.f13.sType =
|
||||
pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES;
|
||||
store.f13.synchronization2 = vk.f_synchronization2 as u32;
|
||||
store.f11.pNext = &mut store.f12 as *mut _ as *mut std::ffi::c_void;
|
||||
store.f12.pNext = &mut store.f13 as *mut _ as *mut std::ffi::c_void;
|
||||
|
||||
(*hwctx).get_proc_addr =
|
||||
std::mem::transmute::<usize, pf_ffvk::PFN_vkGetInstanceProcAddr>(
|
||||
vk.get_instance_proc_addr,
|
||||
);
|
||||
(*hwctx).inst = vk.instance as pf_ffvk::VkInstance;
|
||||
(*hwctx).phys_dev = vk.physical_device as pf_ffvk::VkPhysicalDevice;
|
||||
(*hwctx).act_dev = vk.device as pf_ffvk::VkDevice;
|
||||
(*hwctx).device_features.sType =
|
||||
pf_ffvk::VkStructureType_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;
|
||||
(*hwctx).device_features.pNext =
|
||||
&mut store.f11 as *mut _ as *mut std::ffi::c_void;
|
||||
(*hwctx).enabled_inst_extensions = store.inst_ptrs.as_ptr();
|
||||
(*hwctx).nb_enabled_inst_extensions = store.inst_ptrs.len() as i32;
|
||||
(*hwctx).enabled_dev_extensions = store.dev_ptrs.as_ptr();
|
||||
(*hwctx).nb_enabled_dev_extensions = store.dev_ptrs.len() as i32;
|
||||
|
||||
// Queue map: the deprecated per-role indices (tx/comp are "Required") plus
|
||||
// the qf[] list, which per the header must also carry every family named
|
||||
// above. One merged entry when decode shares the graphics family.
|
||||
let g = vk.graphics_qf as i32;
|
||||
let d = vk.decode_qf as i32;
|
||||
(*hwctx).queue_family_index = g;
|
||||
(*hwctx).nb_graphics_queues = 1;
|
||||
(*hwctx).queue_family_tx_index = g;
|
||||
(*hwctx).nb_tx_queues = 1;
|
||||
(*hwctx).queue_family_comp_index = g;
|
||||
(*hwctx).nb_comp_queues = 1;
|
||||
(*hwctx).queue_family_encode_index = -1;
|
||||
(*hwctx).nb_encode_queues = 0;
|
||||
(*hwctx).queue_family_decode_index = d;
|
||||
(*hwctx).nb_decode_queues = 1;
|
||||
const VIDEO_DECODE_BIT: u32 = 0x20; // VK_QUEUE_VIDEO_DECODE_BIT_KHR
|
||||
if g == d {
|
||||
(*hwctx).qf[0] = pf_ffvk::AVVulkanDeviceQueueFamily {
|
||||
idx: g,
|
||||
num: 1,
|
||||
flags: vk.graphics_queue_flags | VIDEO_DECODE_BIT,
|
||||
video_caps: vk.decode_video_caps,
|
||||
};
|
||||
(*hwctx).nb_qf = 1;
|
||||
} else {
|
||||
(*hwctx).qf[0] = pf_ffvk::AVVulkanDeviceQueueFamily {
|
||||
idx: g,
|
||||
num: 1,
|
||||
flags: vk.graphics_queue_flags,
|
||||
video_caps: 0,
|
||||
};
|
||||
(*hwctx).qf[1] = pf_ffvk::AVVulkanDeviceQueueFamily {
|
||||
idx: d,
|
||||
num: 1,
|
||||
flags: VIDEO_DECODE_BIT,
|
||||
video_caps: vk.decode_video_caps,
|
||||
};
|
||||
(*hwctx).nb_qf = 2;
|
||||
}
|
||||
|
||||
let r = ffi::av_hwdevice_ctx_init(hw_device);
|
||||
if r < 0 {
|
||||
ffi::av_buffer_unref(&mut hw_device);
|
||||
return Err(averr("av_hwdevice_ctx_init(VULKAN)", r));
|
||||
}
|
||||
|
||||
let codec = ffi::avcodec_find_decoder(codec_id.into());
|
||||
if codec.is_null() {
|
||||
ffi::av_buffer_unref(&mut hw_device);
|
||||
bail!("no {codec_id:?} decoder");
|
||||
}
|
||||
let ctx = ffi::avcodec_alloc_context3(codec);
|
||||
(*ctx).hw_device_ctx = ffi::av_buffer_ref(hw_device);
|
||||
(*ctx).get_format = Some(pick_vulkan);
|
||||
(*ctx).flags |= ffi::AV_CODEC_FLAG_LOW_DELAY as i32;
|
||||
(*ctx).thread_count = 1; // hwaccel: threads only add latency
|
||||
// Same pool headroom rationale as VAAPI: the presenter pins the on-screen
|
||||
// frame + the newest in flight past receive_frame.
|
||||
(*ctx).extra_hw_frames = 4;
|
||||
let r = ffi::avcodec_open2(ctx, codec, ptr::null_mut());
|
||||
if r < 0 {
|
||||
let mut ctx = ctx;
|
||||
ffi::avcodec_free_context(&mut ctx);
|
||||
ffi::av_buffer_unref(&mut hw_device);
|
||||
return Err(averr("avcodec_open2 (vulkan)", r));
|
||||
}
|
||||
Ok(VulkanDecoder {
|
||||
ctx,
|
||||
hw_device,
|
||||
packet: ffi::av_packet_alloc(),
|
||||
frame: ffi::av_frame_alloc(),
|
||||
_ctx_storage: store,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn decode(&mut self, au: &[u8]) -> Result<Option<VkVideoFrame>> {
|
||||
use ffmpeg::ffi;
|
||||
unsafe {
|
||||
let r = ffi::av_new_packet(self.packet, au.len() as i32);
|
||||
if r < 0 {
|
||||
return Err(averr("av_new_packet", r));
|
||||
}
|
||||
ptr::copy_nonoverlapping(au.as_ptr(), (*self.packet).data, au.len());
|
||||
let r = ffi::avcodec_send_packet(self.ctx, self.packet);
|
||||
ffi::av_packet_unref(self.packet);
|
||||
if r < 0 {
|
||||
return Err(averr("send_packet", r));
|
||||
}
|
||||
let mut out = None;
|
||||
loop {
|
||||
let r = ffi::avcodec_receive_frame(self.ctx, self.frame);
|
||||
if r == AVERROR_EAGAIN {
|
||||
break;
|
||||
}
|
||||
if r < 0 {
|
||||
return Err(averr("receive_frame", r));
|
||||
}
|
||||
out = Some(self.extract()?); // newest wins; older guards drop here
|
||||
ffi::av_frame_unref(self.frame);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Lift the decoded `AVVkFrame` into a [`VkVideoFrame`]: clone the AVFrame (the
|
||||
/// guard — keeps the image + frames context alive through present) and ship the
|
||||
/// POINTERS; the presenter reads the live sync state under the frames-context lock
|
||||
/// at its own submit time.
|
||||
unsafe fn extract(&mut self) -> Result<VkVideoFrame> {
|
||||
use ffmpeg::ffi;
|
||||
unsafe {
|
||||
if (*self.frame).format != ffi::AVPixelFormat::AV_PIX_FMT_VULKAN as i32 {
|
||||
bail!("decoder returned a non-Vulkan frame");
|
||||
}
|
||||
let hwfc_ref = (*self.frame).hw_frames_ctx;
|
||||
if hwfc_ref.is_null() {
|
||||
bail!("Vulkan frame without a hardware frames context");
|
||||
}
|
||||
let fc = (*hwfc_ref).data as *mut ffi::AVHWFramesContext;
|
||||
let sw = (*fc).sw_format;
|
||||
if sw != ffi::AVPixelFormat::AV_PIX_FMT_NV12 {
|
||||
bail!("Vulkan decode output {sw:?} unsupported (NV12 only for now)");
|
||||
}
|
||||
let vkfc = (*fc).hwctx as *const pf_ffvk::AVVulkanFramesContext;
|
||||
let vk_format = (*vkfc).format[0] as i32;
|
||||
let lock_frame = (*vkfc).lock_frame.map_or(0, |f| f as usize);
|
||||
let unlock_frame = (*vkfc).unlock_frame.map_or(0, |f| f as usize);
|
||||
if lock_frame == 0 || unlock_frame == 0 {
|
||||
bail!("Vulkan frames context without lock functions");
|
||||
}
|
||||
|
||||
let clone = ffi::av_frame_clone(self.frame);
|
||||
if clone.is_null() {
|
||||
bail!("av_frame_clone failed");
|
||||
}
|
||||
let vkf = (*clone).data[0] as *mut pf_ffvk::AVVkFrame;
|
||||
// v1 handles the (default) single multiplanar image; a disjoint/multi-image
|
||||
// pool would need per-plane images — bail so the session demotes cleanly.
|
||||
if !(*vkf).img[1].is_null() {
|
||||
let mut clone = clone;
|
||||
ffi::av_frame_free(&mut clone);
|
||||
bail!("multi-image Vulkan frames unsupported (disjoint pool)");
|
||||
}
|
||||
Ok(VkVideoFrame {
|
||||
vkframe: vkf as usize,
|
||||
frames_ctx: fc as usize,
|
||||
lock_frame,
|
||||
unlock_frame,
|
||||
vk_format,
|
||||
width: (*self.frame).width as u32,
|
||||
height: (*self.frame).height as u32,
|
||||
color: ColorDesc::from_raw(self.frame),
|
||||
guard: DrmFrameGuard(clone),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for VulkanDecoder {
|
||||
fn drop(&mut self) {
|
||||
use ffmpeg::ffi;
|
||||
unsafe {
|
||||
ffi::av_packet_free(&mut self.packet);
|
||||
ffi::av_frame_free(&mut self.frame);
|
||||
ffi::avcodec_free_context(&mut self.ctx);
|
||||
ffi::av_buffer_unref(&mut self.hw_device);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// libavcodec offers the formats it can decode into; pick the Vulkan hw surface and
|
||||
/// hand the decoder OUR frames context — the default one lacks
|
||||
/// `VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT`, without which the presenter can't create the
|
||||
/// per-plane views its CSC pass samples. Returning NONE (over the software entry) keeps
|
||||
/// failures loud: the session demotes explicitly instead of silently CPU-decoding.
|
||||
unsafe extern "C" fn pick_vulkan(
|
||||
ctx: *mut ffmpeg::ffi::AVCodecContext,
|
||||
mut list: *const ffmpeg::ffi::AVPixelFormat,
|
||||
) -> ffmpeg::ffi::AVPixelFormat {
|
||||
use ffmpeg::ffi;
|
||||
unsafe {
|
||||
let mut offered = false;
|
||||
while *list != ffi::AVPixelFormat::AV_PIX_FMT_NONE {
|
||||
if *list == ffi::AVPixelFormat::AV_PIX_FMT_VULKAN {
|
||||
offered = true;
|
||||
break;
|
||||
}
|
||||
list = list.add(1);
|
||||
}
|
||||
if !offered {
|
||||
return ffi::AVPixelFormat::AV_PIX_FMT_NONE;
|
||||
}
|
||||
let mut fr: *mut ffi::AVBufferRef = ptr::null_mut();
|
||||
let r = ffi::avcodec_get_hw_frames_parameters(
|
||||
ctx,
|
||||
(*ctx).hw_device_ctx,
|
||||
ffi::AVPixelFormat::AV_PIX_FMT_VULKAN,
|
||||
&mut fr,
|
||||
);
|
||||
if r < 0 || fr.is_null() {
|
||||
tracing::warn!("avcodec_get_hw_frames_parameters(VULKAN) failed ({r})");
|
||||
return ffi::AVPixelFormat::AV_PIX_FMT_NONE;
|
||||
}
|
||||
let fc = (*fr).data as *mut ffi::AVHWFramesContext;
|
||||
let vkfc = (*fc).hwctx as *mut pf_ffvk::AVVulkanFramesContext;
|
||||
// MUTABLE_FORMAT: per-plane views (spec requirement); ALIAS is FFmpeg's default.
|
||||
(*vkfc).img_flags = pf_ffvk::VkImageCreateFlagBits_VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT
|
||||
| pf_ffvk::VkImageCreateFlagBits_VK_IMAGE_CREATE_ALIAS_BIT;
|
||||
let r = ffi::av_hwframe_ctx_init(fr);
|
||||
if r < 0 {
|
||||
tracing::warn!("av_hwframe_ctx_init(VULKAN) failed ({r})");
|
||||
let mut fr = fr;
|
||||
ffi::av_buffer_unref(&mut fr);
|
||||
return ffi::AVPixelFormat::AV_PIX_FMT_NONE;
|
||||
}
|
||||
if !(*ctx).hw_frames_ctx.is_null() {
|
||||
ffi::av_buffer_unref(&mut (*ctx).hw_frames_ctx);
|
||||
}
|
||||
(*ctx).hw_frames_ctx = fr; // the codec owns our ref now
|
||||
ffi::AVPixelFormat::AV_PIX_FMT_VULKAN
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -50,6 +50,13 @@ fn main() {
|
||||
.allowlist_type("AVVkFrame.*")
|
||||
.allowlist_function("av_vk_frame_alloc")
|
||||
.allowlist_function("av_vkfmt_from_pixfmt")
|
||||
// The feature structs chained into AVVulkanDeviceContext.device_features (plain
|
||||
// vulkan.h types; generating them here keeps the chain in one type system).
|
||||
.allowlist_type("VkPhysicalDeviceVulkan11Features")
|
||||
.allowlist_type("VkPhysicalDeviceVulkan12Features")
|
||||
.allowlist_type("VkPhysicalDeviceVulkan13Features")
|
||||
// AVVulkanFramesContext.img_flags values (plane views need MUTABLE_FORMAT).
|
||||
.allowlist_type("VkImageCreateFlagBits")
|
||||
// …plus nothing else of FFmpeg: the core types these structs reference only
|
||||
// ever appear behind pointers here, so keep them opaque instead of duplicating
|
||||
// ffmpeg-sys-next's definitions (callers cast pointers between the crates).
|
||||
|
||||
@@ -11,6 +11,8 @@ repository.workspace = true
|
||||
# Same Linux gating as the rest of the client stack.
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
pf-client-core = { path = "../pf-client-core" }
|
||||
# AVVkFrame access (Vulkan Video frames: live sync state under the frames lock).
|
||||
pf-ffvk = { path = "../pf-ffvk" }
|
||||
punktfunk-core = { path = "../punktfunk-core", features = ["quic"] }
|
||||
|
||||
# `loaded` dlopens libvulkan at runtime (no link-time dependency — GPU-less boxes still
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::overlay::{FrameCtx, Overlay, OverlayAction, OverlayFrame, SessionPhas
|
||||
use crate::vk::{FrameInput, Presenter};
|
||||
use anyhow::{Context as _, Result};
|
||||
use pf_client_core::gamepad::GamepadService;
|
||||
use pf_client_core::video::VulkanDecodeDevice;
|
||||
use pf_client_core::session::{self, SessionEvent, SessionHandle, SessionParams, Stats};
|
||||
use pf_client_core::video::{DecodedFrame, DecodedImage};
|
||||
use punktfunk_core::client::NativeClient;
|
||||
@@ -58,8 +59,8 @@ pub enum ActionOutcome {
|
||||
/// Consumed binary-side (a Retry respawned the fetch, …).
|
||||
Handled,
|
||||
/// Start this session (a Launch action; `force_software` from the callback args is
|
||||
/// wired into these params).
|
||||
Start(SessionParams),
|
||||
/// wired into these params). Boxed: SessionParams is large next to the unit variants.
|
||||
Start(Box<SessionParams>),
|
||||
/// Quit the launcher.
|
||||
Quit,
|
||||
}
|
||||
@@ -67,13 +68,13 @@ pub enum ActionOutcome {
|
||||
/// One `--connect` stream session; returns when it ends (the shell↔session contract).
|
||||
pub fn run_session<F>(opts: SessionOpts, build_params: F) -> Result<Outcome>
|
||||
where
|
||||
F: FnOnce(&GamepadService, Mode, Arc<AtomicBool>) -> SessionParams,
|
||||
F: FnOnce(&GamepadService, Mode, Arc<AtomicBool>, Option<VulkanDecodeDevice>) -> SessionParams,
|
||||
{
|
||||
let mut build = Some(build_params);
|
||||
run_inner(
|
||||
opts,
|
||||
ModeCtl::Single(Box::new(move |gp, native, fs| {
|
||||
(build.take().expect("single build runs once"))(gp, native, fs)
|
||||
ModeCtl::Single(Box::new(move |gp, native, fs, vk| {
|
||||
(build.take().expect("single build runs once"))(gp, native, fs, vk)
|
||||
})),
|
||||
)
|
||||
.map(|o| o.expect("single mode always yields an outcome"))
|
||||
@@ -85,7 +86,13 @@ where
|
||||
/// per-session `force_software` flag.
|
||||
pub fn run_browse<F>(opts: SessionOpts, on_action: F) -> Result<()>
|
||||
where
|
||||
F: FnMut(OverlayAction, &GamepadService, Mode, Arc<AtomicBool>) -> ActionOutcome,
|
||||
F: FnMut(
|
||||
OverlayAction,
|
||||
&GamepadService,
|
||||
Mode,
|
||||
Arc<AtomicBool>,
|
||||
Option<VulkanDecodeDevice>,
|
||||
) -> ActionOutcome,
|
||||
{
|
||||
anyhow::ensure!(
|
||||
opts.overlay.is_some(),
|
||||
@@ -95,10 +102,21 @@ where
|
||||
}
|
||||
|
||||
/// Params builder for the one single-mode session (called exactly once, post-setup).
|
||||
type BuildParams<'a> = Box<dyn FnMut(&GamepadService, Mode, Arc<AtomicBool>) -> SessionParams + 'a>;
|
||||
type BuildParams<'a> = Box<
|
||||
dyn FnMut(&GamepadService, Mode, Arc<AtomicBool>, Option<VulkanDecodeDevice>) -> SessionParams
|
||||
+ 'a,
|
||||
>;
|
||||
/// The browse-mode action callback (Launch → params, Retry/Quit → outcome).
|
||||
type OnAction<'a> =
|
||||
Box<dyn FnMut(OverlayAction, &GamepadService, Mode, Arc<AtomicBool>) -> ActionOutcome + 'a>;
|
||||
type OnAction<'a> = Box<
|
||||
dyn FnMut(
|
||||
OverlayAction,
|
||||
&GamepadService,
|
||||
Mode,
|
||||
Arc<AtomicBool>,
|
||||
Option<VulkanDecodeDevice>,
|
||||
) -> ActionOutcome
|
||||
+ 'a,
|
||||
>;
|
||||
|
||||
/// The two run modes, type-erased so one loop serves both.
|
||||
enum ModeCtl<'a> {
|
||||
@@ -227,7 +245,12 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
|
||||
let mut stream: Option<StreamState> = match &mut mode {
|
||||
ModeCtl::Single(build) => {
|
||||
let force_software = Arc::new(AtomicBool::new(false));
|
||||
let params = build(&gamepad, native, force_software.clone());
|
||||
let params = build(
|
||||
&gamepad,
|
||||
native,
|
||||
force_software.clone(),
|
||||
presenter.vulkan_decode(),
|
||||
);
|
||||
Some(StreamState::new(params, force_software))
|
||||
}
|
||||
ModeCtl::Browse(_) => None,
|
||||
@@ -416,10 +439,16 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
|
||||
}
|
||||
if let Some(action) = overlay.as_mut().and_then(|o| o.take_action()) {
|
||||
let force_software = Arc::new(AtomicBool::new(false));
|
||||
match on_action(action, &gamepad, native, force_software.clone()) {
|
||||
match on_action(
|
||||
action,
|
||||
&gamepad,
|
||||
native,
|
||||
force_software.clone(),
|
||||
presenter.vulkan_decode(),
|
||||
) {
|
||||
ActionOutcome::Handled => {}
|
||||
ActionOutcome::Start(params) => {
|
||||
stream = Some(StreamState::new(params, force_software));
|
||||
stream = Some(StreamState::new(*params, force_software));
|
||||
if let Some(o) = overlay.as_mut() {
|
||||
o.session_phase(SessionPhase::Connecting);
|
||||
}
|
||||
@@ -605,6 +634,34 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
|
||||
}
|
||||
false
|
||||
}
|
||||
// Vulkan-Video: decoded on the presenter's own device — present is
|
||||
// views + CSC, no import step to gate on. Same failure-streak
|
||||
// demotion contract as the dmabuf path.
|
||||
DecodedImage::VkFrame(v) if !st.dmabuf_demoted => {
|
||||
st.hdr = v.color.is_pq();
|
||||
match presenter.present(
|
||||
&window,
|
||||
FrameInput::VkFrame(v),
|
||||
overlay_frame.as_ref(),
|
||||
) {
|
||||
Ok(p) => {
|
||||
st.hw_fails = 0;
|
||||
p
|
||||
}
|
||||
Err(e) => {
|
||||
st.hw_fails += 1;
|
||||
tracing::warn!(error = %format!("{e:#}"), fails = st.hw_fails,
|
||||
"vulkan-video present failed");
|
||||
if st.hw_fails >= 3 {
|
||||
st.dmabuf_demoted = true;
|
||||
tracing::warn!("demoting the decoder to software");
|
||||
st.force_software.store(true, Ordering::Relaxed);
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
DecodedImage::VkFrame(_) => false, // demoted — drain until rebuild
|
||||
};
|
||||
if did_present {
|
||||
presented_video = true;
|
||||
|
||||
+527
-118
@@ -19,7 +19,8 @@ use crate::dmabuf::{self, HwFrame};
|
||||
use crate::overlay::{OverlayFrame, SharedDevice};
|
||||
use anyhow::{anyhow, bail, Context as _, Result};
|
||||
use ash::vk;
|
||||
use pf_client_core::video::{CpuFrame, DmabufFrame};
|
||||
use ash::vk::Handle as _;
|
||||
use pf_client_core::video::{CpuFrame, DmabufFrame, VkVideoFrame};
|
||||
use std::ffi::CString;
|
||||
|
||||
/// One presenter iteration's video input.
|
||||
@@ -28,12 +29,40 @@ pub enum FrameInput<'a> {
|
||||
Redraw,
|
||||
Cpu(&'a CpuFrame),
|
||||
Dmabuf(DmabufFrame),
|
||||
/// FFmpeg Vulkan Video output — a VkImage already on THIS device (zero copy).
|
||||
VkFrame(VkVideoFrame),
|
||||
}
|
||||
|
||||
/// The dmabuf/CSC machinery, present only when the device carries the import extensions.
|
||||
struct HwCtx {
|
||||
ext_mem_fd: ash::khr::external_memory_fd::Device,
|
||||
csc: CscPass,
|
||||
}
|
||||
|
||||
/// A submitted hardware frame parked until the in-flight fence proves the GPU reads
|
||||
/// done: imported dmabuf planes, or a Vulkan-Video frame (FFmpeg's image — we own only
|
||||
/// the plane views; dropping the frame's guard releases the AVFrame back to the pool).
|
||||
enum Retired {
|
||||
Dmabuf(HwFrame),
|
||||
Vk {
|
||||
frame: VkVideoFrame,
|
||||
views: [vk::ImageView; 2],
|
||||
},
|
||||
}
|
||||
|
||||
impl Retired {
|
||||
fn destroy(self, device: &ash::Device) {
|
||||
match self {
|
||||
Retired::Dmabuf(f) => f.destroy(device),
|
||||
Retired::Vk { frame, views } => {
|
||||
unsafe {
|
||||
for v in views {
|
||||
device.destroy_image_view(v, None);
|
||||
}
|
||||
}
|
||||
drop(frame); // guard drops here — AVFrame (and the VkImage) released
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The overlay composite: one premultiplied-alpha quad blended over the swapchain image
|
||||
@@ -251,13 +280,18 @@ pub struct Presenter {
|
||||
swap_d: ash::khr::swapchain::Device,
|
||||
queue: vk::Queue,
|
||||
qfi: u32,
|
||||
/// Dmabuf import + CSC — `None` when the device lacks the import extensions.
|
||||
/// Dmabuf import — `None` when the device lacks the import extensions (the CSC
|
||||
/// pass itself is unconditional: Vulkan-Video frames need it everywhere).
|
||||
hw: Option<HwCtx>,
|
||||
csc: CscPass,
|
||||
/// FFmpeg Vulkan Video decode handles — `None` when the stack can't do it.
|
||||
video_export: Option<pf_client_core::video::VulkanDecodeDevice>,
|
||||
/// The console-UI composite quad (§6.1's presenter half).
|
||||
overlay_pipe: OverlayPipe,
|
||||
/// The submitted hw frame (plane images + decoder-surface guard): its GPU reads end
|
||||
/// with the in-flight fence, so it's destroyed right after the next fence wait.
|
||||
retired_hw: Option<HwFrame>,
|
||||
/// The submitted hardware frame (dmabuf plane images + guard, or a Vulkan-Video
|
||||
/// frame + our plane views): its GPU reads end with the in-flight fence, so it's
|
||||
/// destroyed right after the next fence wait.
|
||||
retired_hw: Option<Retired>,
|
||||
format: vk::SurfaceFormatKHR,
|
||||
present_mode: vk::PresentModeKHR,
|
||||
swapchain: vk::SwapchainKHR,
|
||||
@@ -285,9 +319,12 @@ impl Presenter {
|
||||
let entry = unsafe { ash::Entry::load() }.context("libvulkan not loadable")?;
|
||||
|
||||
let app_name = CString::new("punktfunk-session").unwrap();
|
||||
// 1.3: FFmpeg's Vulkan hwcontext requires an instance of at least 1.3 (any
|
||||
// current loader accepts it regardless of device support; device-level gating
|
||||
// happens below).
|
||||
let app_info = vk::ApplicationInfo::default()
|
||||
.application_name(&app_name)
|
||||
.api_version(vk::API_VERSION_1_2);
|
||||
.api_version(vk::API_VERSION_1_3);
|
||||
let ext_cstrings: Vec<CString> = instance_extensions
|
||||
.iter()
|
||||
.map(|e| CString::new(e.as_str()).unwrap())
|
||||
@@ -318,11 +355,8 @@ impl Presenter {
|
||||
tracing::info!(device = %name, queue_family = qfi, "vulkan device");
|
||||
}
|
||||
|
||||
let queue_info = [vk::DeviceQueueCreateInfo::default()
|
||||
.queue_family_index(qfi)
|
||||
.queue_priorities(&[1.0])];
|
||||
// The dmabuf import set is optional: enabled when the device offers all four,
|
||||
// else the presenter is software-only (`supports_dmabuf() == false`).
|
||||
// else that path is off (`supports_dmabuf() == false`).
|
||||
let available = unsafe { instance.enumerate_device_extension_properties(pdev) }?;
|
||||
let has = |name: &std::ffi::CStr| {
|
||||
available
|
||||
@@ -335,8 +369,123 @@ impl Presenter {
|
||||
dev_exts.extend(dmabuf::DEVICE_EXTENSIONS.iter().map(|n| n.as_ptr()));
|
||||
} else {
|
||||
tracing::info!(
|
||||
"device lacks the dmabuf import extensions — hardware frames unavailable \
|
||||
(software decode only)"
|
||||
"device lacks the dmabuf import extensions — VAAPI hardware frames \
|
||||
unavailable"
|
||||
);
|
||||
}
|
||||
|
||||
// --- Vulkan Video decode (the FFmpeg-on-our-device path) ---------------------
|
||||
// Probed, never required: a capable stack gets the video extensions, a second
|
||||
// (decode) queue, and the features FFmpeg's decoder needs; anything less means
|
||||
// `vulkan_decode() == None` and the decoder chain falls back (VAAPI/software).
|
||||
let dev_props = unsafe { instance.get_physical_device_properties(pdev) };
|
||||
let dev_is_13 = vk::api_version_major(dev_props.api_version) > 1
|
||||
|| vk::api_version_minor(dev_props.api_version) >= 3;
|
||||
let mut have_f11 = vk::PhysicalDeviceVulkan11Features::default();
|
||||
let mut have_f12 = vk::PhysicalDeviceVulkan12Features::default();
|
||||
let mut have_f13 = vk::PhysicalDeviceVulkan13Features::default();
|
||||
let mut have_f2 = vk::PhysicalDeviceFeatures2::default()
|
||||
.push_next(&mut have_f11)
|
||||
.push_next(&mut have_f12)
|
||||
.push_next(&mut have_f13);
|
||||
unsafe { instance.get_physical_device_features2(pdev, &mut have_f2) };
|
||||
let features_ok = have_f11.sampler_ycbcr_conversion == vk::TRUE
|
||||
&& have_f12.timeline_semaphore == vk::TRUE
|
||||
&& have_f13.synchronization2 == vk::TRUE;
|
||||
|
||||
// The decode queue family + which codec operations it can run.
|
||||
let decode_family: Option<(u32, vk::VideoCodecOperationFlagsKHR)> = {
|
||||
let n = unsafe { instance.get_physical_device_queue_family_properties2_len(pdev) };
|
||||
let mut video: Vec<vk::QueueFamilyVideoPropertiesKHR> =
|
||||
vec![vk::QueueFamilyVideoPropertiesKHR::default(); n];
|
||||
let mut props: Vec<vk::QueueFamilyProperties2> = video
|
||||
.iter_mut()
|
||||
.map(|v| vk::QueueFamilyProperties2::default().push_next(v))
|
||||
.collect();
|
||||
unsafe { instance.get_physical_device_queue_family_properties2(pdev, &mut props) };
|
||||
// `props` mutably borrows `video` (push_next); copy the flags out, then
|
||||
// read the driver-filled video properties directly.
|
||||
let flags: Vec<vk::QueueFlags> = props
|
||||
.iter()
|
||||
.map(|p| p.queue_family_properties.queue_flags)
|
||||
.collect();
|
||||
drop(props);
|
||||
flags
|
||||
.iter()
|
||||
.zip(&video)
|
||||
.enumerate()
|
||||
.find(|(_, (f, _))| f.contains(vk::QueueFlags::VIDEO_DECODE_KHR))
|
||||
.map(|(i, (_, v))| (i as u32, v.video_codec_operations))
|
||||
};
|
||||
|
||||
const VIDEO_BASE: [&std::ffi::CStr; 2] = [
|
||||
ash::khr::video_queue::NAME,
|
||||
ash::khr::video_decode_queue::NAME,
|
||||
];
|
||||
const VIDEO_CODECS: [&std::ffi::CStr; 3] = [
|
||||
ash::khr::video_decode_h264::NAME,
|
||||
ash::khr::video_decode_h265::NAME,
|
||||
c"VK_KHR_video_decode_av1",
|
||||
];
|
||||
let codec_exts: Vec<&std::ffi::CStr> = VIDEO_CODECS
|
||||
.into_iter()
|
||||
.filter(|n| has(n))
|
||||
.collect();
|
||||
let video_ok = dev_is_13
|
||||
&& features_ok
|
||||
&& decode_family.is_some()
|
||||
&& VIDEO_BASE.iter().all(|n| has(n))
|
||||
&& !codec_exts.is_empty();
|
||||
|
||||
let (decode_qf, decode_caps) = decode_family.unwrap_or((qfi, Default::default()));
|
||||
let mut video_ext_names: Vec<&std::ffi::CStr> = Vec::new();
|
||||
if video_ok {
|
||||
video_ext_names.extend(VIDEO_BASE);
|
||||
video_ext_names.extend(&codec_exts);
|
||||
// Optional decoder niceties FFmpeg uses when present.
|
||||
for opt in [c"VK_KHR_video_maintenance1", c"VK_KHR_video_maintenance2"] {
|
||||
if has(opt) {
|
||||
video_ext_names.push(opt);
|
||||
}
|
||||
}
|
||||
dev_exts.extend(video_ext_names.iter().map(|n| n.as_ptr()));
|
||||
tracing::info!(
|
||||
decode_qf,
|
||||
caps = ?decode_caps,
|
||||
exts = ?video_ext_names,
|
||||
"Vulkan Video decode available on this device"
|
||||
);
|
||||
} else {
|
||||
tracing::info!(
|
||||
dev_is_13,
|
||||
features_ok,
|
||||
decode_family = decode_family.is_some(),
|
||||
"Vulkan Video decode unavailable — decoder falls back (VAAPI/software)"
|
||||
);
|
||||
}
|
||||
|
||||
// Enable only the features the video path needs, and only where supported
|
||||
// (harmless when the path is off; reported to FFmpeg via device_features).
|
||||
let mut en_f11 = vk::PhysicalDeviceVulkan11Features::default()
|
||||
.sampler_ycbcr_conversion(have_f11.sampler_ycbcr_conversion == vk::TRUE);
|
||||
let mut en_f12 = vk::PhysicalDeviceVulkan12Features::default()
|
||||
.timeline_semaphore(have_f12.timeline_semaphore == vk::TRUE);
|
||||
let mut en_f13 = vk::PhysicalDeviceVulkan13Features::default()
|
||||
.synchronization2(have_f13.synchronization2 == vk::TRUE);
|
||||
let mut en_f2 = vk::PhysicalDeviceFeatures2::default()
|
||||
.push_next(&mut en_f11)
|
||||
.push_next(&mut en_f12)
|
||||
.push_next(&mut en_f13);
|
||||
|
||||
let priorities = [1.0f32];
|
||||
let mut queue_info = vec![vk::DeviceQueueCreateInfo::default()
|
||||
.queue_family_index(qfi)
|
||||
.queue_priorities(&priorities)];
|
||||
if video_ok && decode_qf != qfi {
|
||||
queue_info.push(
|
||||
vk::DeviceQueueCreateInfo::default()
|
||||
.queue_family_index(decode_qf)
|
||||
.queue_priorities(&priorities),
|
||||
);
|
||||
}
|
||||
let device = unsafe {
|
||||
@@ -344,7 +493,8 @@ impl Presenter {
|
||||
pdev,
|
||||
&vk::DeviceCreateInfo::default()
|
||||
.queue_create_infos(&queue_info)
|
||||
.enabled_extension_names(&dev_exts),
|
||||
.enabled_extension_names(&dev_exts)
|
||||
.push_next(&mut en_f2),
|
||||
None,
|
||||
)
|
||||
}
|
||||
@@ -354,7 +504,41 @@ impl Presenter {
|
||||
let hw = if hw_capable {
|
||||
Some(HwCtx {
|
||||
ext_mem_fd: ash::khr::external_memory_fd::Device::new(&instance, &device),
|
||||
csc: CscPass::new(&device)?,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let csc = CscPass::new(&device)?;
|
||||
|
||||
// The exported handle bundle for FFmpeg's Vulkan Video decoder (None = the
|
||||
// decoder chain skips straight to VAAPI/software). Extension lists must mirror
|
||||
// creation exactly — FFmpeg keys its code paths off the strings.
|
||||
let video_export = if video_ok {
|
||||
let qf_props = unsafe { instance.get_physical_device_queue_family_properties(pdev) };
|
||||
let mut device_extensions: Vec<CString> =
|
||||
vec![CString::from(ash::khr::swapchain::NAME)];
|
||||
if hw_capable {
|
||||
device_extensions
|
||||
.extend(dmabuf::DEVICE_EXTENSIONS.iter().map(|n| CString::from(*n)));
|
||||
}
|
||||
device_extensions.extend(video_ext_names.iter().map(|n| CString::from(*n)));
|
||||
Some(pf_client_core::video::VulkanDecodeDevice {
|
||||
get_instance_proc_addr: entry.static_fn().get_instance_proc_addr as usize,
|
||||
instance: instance.handle().as_raw() as usize,
|
||||
physical_device: pdev.as_raw() as usize,
|
||||
device: device.handle().as_raw() as usize,
|
||||
graphics_qf: qfi,
|
||||
graphics_queue_flags: qf_props[qfi as usize].queue_flags.as_raw(),
|
||||
decode_qf,
|
||||
decode_video_caps: decode_caps.as_raw(),
|
||||
instance_extensions: instance_extensions
|
||||
.iter()
|
||||
.map(|e| CString::new(e.as_str()).unwrap())
|
||||
.collect(),
|
||||
device_extensions,
|
||||
f_sampler_ycbcr: true,
|
||||
f_timeline_semaphore: true,
|
||||
f_synchronization2: true,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
@@ -402,6 +586,8 @@ impl Presenter {
|
||||
queue,
|
||||
qfi,
|
||||
hw,
|
||||
csc,
|
||||
video_export,
|
||||
overlay_pipe,
|
||||
retired_hw: None,
|
||||
format,
|
||||
@@ -502,6 +688,13 @@ impl Presenter {
|
||||
self.hw.is_some()
|
||||
}
|
||||
|
||||
/// The FFmpeg Vulkan Video decode handle bundle — `None` when this stack can't
|
||||
/// (device < 1.3, missing video extensions/queue/features). The decoder chain
|
||||
/// falls back to VAAPI/software then.
|
||||
pub fn vulkan_decode(&self) -> Option<pf_client_core::video::VulkanDecodeDevice> {
|
||||
self.video_export.clone()
|
||||
}
|
||||
|
||||
/// Quiesce the queue — the run loop calls this before dropping the overlay so
|
||||
/// nothing in flight still references its images.
|
||||
pub fn wait_idle(&self) {
|
||||
@@ -535,9 +728,11 @@ impl Presenter {
|
||||
if self.extent.width == 0 || self.extent.height == 0 {
|
||||
return Ok(true); // minimized — nothing to do
|
||||
}
|
||||
// A dmabuf frame imports before anything touches the queue: an import the driver
|
||||
// rejects must fail out here, before this present consumed the acquire semaphore.
|
||||
// Hardware frames prepare before anything touches the queue: an import/view the
|
||||
// driver rejects must fail out here, before this present consumed the acquire
|
||||
// semaphore.
|
||||
let mut hw_frame: Option<HwFrame> = None;
|
||||
let mut vk_frame: Option<(VkVideoFrame, [vk::ImageView; 2])> = None;
|
||||
let cpu_frame = match input {
|
||||
FrameInput::Redraw => None,
|
||||
FrameInput::Cpu(f) => Some(f),
|
||||
@@ -549,6 +744,11 @@ impl Presenter {
|
||||
hw_frame = Some(dmabuf::import(&self.device, &hw.ext_mem_fd, d)?);
|
||||
None
|
||||
}
|
||||
FrameInput::VkFrame(v) => {
|
||||
let views = self.vkframe_plane_views(&v)?;
|
||||
vk_frame = Some((v, views));
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
// One frame in flight: the fence covers the command buffer, the staging buffer
|
||||
@@ -577,8 +777,18 @@ impl Presenter {
|
||||
tracing::info!(width = f.width, height = f.height, "video image (re)built");
|
||||
}
|
||||
// Safe while nothing in flight references the set — the fence wait above.
|
||||
let hw = self.hw.as_ref().unwrap();
|
||||
hw.csc.bind_planes(&self.device, f.luma_view, f.chroma_view);
|
||||
self.csc.bind_planes(&self.device, f.luma_view, f.chroma_view);
|
||||
}
|
||||
if let Some((f, views)) = &vk_frame {
|
||||
if self
|
||||
.video
|
||||
.as_ref()
|
||||
.is_none_or(|v| v.width != f.width || v.height != f.height)
|
||||
{
|
||||
self.rebuild_video_image(f.width, f.height)?;
|
||||
tracing::info!(width = f.width, height = f.height, "video image (re)built");
|
||||
}
|
||||
self.csc.bind_planes(&self.device, views[0], views[1]);
|
||||
}
|
||||
if let Some(o) = overlay {
|
||||
// Point the composite at this overlay image (same fence-wait safety).
|
||||
@@ -621,10 +831,10 @@ impl Presenter {
|
||||
.flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT),
|
||||
)?;
|
||||
|
||||
// Hardware frame: acquire the foreign planes, then the CSC pass renders
|
||||
// Dmabuf frame: acquire the foreign planes, then the CSC pass renders
|
||||
// NV12→RGBA into the video image (render pass ends it in TRANSFER_SRC for
|
||||
// the blit below).
|
||||
if let (Some(f), Some(hw), Some(v)) = (&hw_frame, &self.hw, &self.video) {
|
||||
if let (Some(f), Some(v)) = (&hw_frame, &self.video) {
|
||||
for view_image in [f.luma_image(), f.chroma_image()] {
|
||||
foreign_acquire_barrier(&self.device, self.cmd_buf, view_image, self.qfi);
|
||||
}
|
||||
@@ -632,61 +842,30 @@ impl Presenter {
|
||||
width: v.width,
|
||||
height: v.height,
|
||||
};
|
||||
self.device.cmd_begin_render_pass(
|
||||
self.record_csc(v.framebuffer, extent, f.color);
|
||||
}
|
||||
|
||||
// Vulkan-Video frame: the decoded image is already on THIS device. Read the
|
||||
// live sync state under the frames lock (held through submission — the
|
||||
// AVVulkanFramesContext contract), acquire from the decode queue family,
|
||||
// then the same CSC pass.
|
||||
let mut vk_sync: Option<VkFrameSync> = None;
|
||||
if let (Some((f, _)), Some(v)) = (&vk_frame, &self.video) {
|
||||
let sync = lock_vkframe(f);
|
||||
vkframe_acquire_barrier(
|
||||
&self.device,
|
||||
self.cmd_buf,
|
||||
&vk::RenderPassBeginInfo::default()
|
||||
.render_pass(hw.csc.render_pass)
|
||||
.framebuffer(v.framebuffer)
|
||||
.render_area(vk::Rect2D {
|
||||
offset: vk::Offset2D { x: 0, y: 0 },
|
||||
extent,
|
||||
}),
|
||||
vk::SubpassContents::INLINE,
|
||||
vk::Image::from_raw(sync.image),
|
||||
vk::ImageLayout::from_raw(sync.layout),
|
||||
sync.queue_family,
|
||||
self.qfi,
|
||||
);
|
||||
self.device.cmd_bind_pipeline(
|
||||
self.cmd_buf,
|
||||
vk::PipelineBindPoint::GRAPHICS,
|
||||
hw.csc.pipeline,
|
||||
);
|
||||
self.device.cmd_set_viewport(
|
||||
self.cmd_buf,
|
||||
0,
|
||||
&[vk::Viewport {
|
||||
x: 0.0,
|
||||
y: 0.0,
|
||||
width: extent.width as f32,
|
||||
height: extent.height as f32,
|
||||
min_depth: 0.0,
|
||||
max_depth: 1.0,
|
||||
}],
|
||||
);
|
||||
self.device.cmd_set_scissor(
|
||||
self.cmd_buf,
|
||||
0,
|
||||
&[vk::Rect2D {
|
||||
offset: vk::Offset2D { x: 0, y: 0 },
|
||||
extent,
|
||||
}],
|
||||
);
|
||||
self.device.cmd_bind_descriptor_sets(
|
||||
self.cmd_buf,
|
||||
vk::PipelineBindPoint::GRAPHICS,
|
||||
hw.csc.pipeline_layout,
|
||||
0,
|
||||
&[hw.csc.desc_set],
|
||||
&[],
|
||||
);
|
||||
let rows = csc_rows(f.color);
|
||||
let bytes = std::slice::from_raw_parts(rows.as_ptr().cast::<u8>(), 48);
|
||||
self.device.cmd_push_constants(
|
||||
self.cmd_buf,
|
||||
hw.csc.pipeline_layout,
|
||||
vk::ShaderStageFlags::FRAGMENT,
|
||||
0,
|
||||
bytes,
|
||||
);
|
||||
self.device.cmd_draw(self.cmd_buf, 3, 1, 0, 0);
|
||||
self.device.cmd_end_render_pass(self.cmd_buf);
|
||||
let extent = vk::Extent2D {
|
||||
width: v.width,
|
||||
height: v.height,
|
||||
};
|
||||
self.record_csc(v.framebuffer, extent, f.color);
|
||||
vk_sync = Some(sync);
|
||||
}
|
||||
|
||||
// New frame: staging → video image (stride carried by buffer_row_length).
|
||||
@@ -840,23 +1019,53 @@ impl Presenter {
|
||||
self.device.end_command_buffer(self.cmd_buf)?;
|
||||
|
||||
let render_sem = self.render_sems[index as usize];
|
||||
let wait_sems = [self.acquire_sem];
|
||||
let wait_stages = [vk::PipelineStageFlags::TRANSFER];
|
||||
let cmd_bufs = [self.cmd_buf];
|
||||
let signal_sems = [render_sem];
|
||||
self.device.queue_submit(
|
||||
self.queue,
|
||||
&[vk::SubmitInfo::default()
|
||||
.wait_semaphores(&wait_sems)
|
||||
.wait_dst_stage_mask(&wait_stages)
|
||||
.command_buffers(&cmd_bufs)
|
||||
.signal_semaphores(&signal_sems)],
|
||||
self.fence,
|
||||
)?;
|
||||
let mut wait_sems = vec![self.acquire_sem];
|
||||
let mut wait_stages = vec![vk::PipelineStageFlags::TRANSFER];
|
||||
let mut signal_sems = vec![render_sem];
|
||||
// The Vulkan-Video frame's timeline semaphore: wait for the decoder's value,
|
||||
// signal value+1 when our reads are done (FFmpeg's per-submission contract).
|
||||
let mut wait_values = vec![0u64];
|
||||
let mut signal_values = vec![0u64];
|
||||
if let Some(sync) = &vk_sync {
|
||||
let sem = vk::Semaphore::from_raw(sync.semaphore);
|
||||
wait_sems.push(sem);
|
||||
wait_stages.push(vk::PipelineStageFlags::FRAGMENT_SHADER);
|
||||
wait_values.push(sync.sem_value);
|
||||
signal_sems.push(sem);
|
||||
signal_values.push(sync.sem_value + 1);
|
||||
}
|
||||
let mut timeline = vk::TimelineSemaphoreSubmitInfo::default()
|
||||
.wait_semaphore_values(&wait_values)
|
||||
.signal_semaphore_values(&signal_values);
|
||||
let mut submit = vk::SubmitInfo::default()
|
||||
.wait_semaphores(&wait_sems)
|
||||
.wait_dst_stage_mask(&wait_stages)
|
||||
.command_buffers(&cmd_bufs)
|
||||
.signal_semaphores(&signal_sems);
|
||||
if vk_sync.is_some() {
|
||||
submit = submit.push_next(&mut timeline);
|
||||
}
|
||||
let submitted = self.device.queue_submit(self.queue, &[submit], self.fence);
|
||||
// Write the new sync state back and release the frames lock REGARDLESS of
|
||||
// the submit outcome (an abandoned lock would wedge the decoder).
|
||||
if let Some(sync) = vk_sync.take() {
|
||||
let ok = submitted.is_ok();
|
||||
unlock_vkframe(
|
||||
vk_frame.as_ref().map(|(f, _)| f).expect("vk_sync implies vk_frame"),
|
||||
&sync,
|
||||
ok,
|
||||
self.qfi,
|
||||
);
|
||||
}
|
||||
submitted?;
|
||||
self.submitted = true;
|
||||
// The hw frame is on the GPU now — park it until the fence proves the reads
|
||||
// done (destroyed at the next present's fence wait, or in Drop).
|
||||
self.retired_hw = hw_frame.take();
|
||||
self.retired_hw = hw_frame
|
||||
.take()
|
||||
.map(Retired::Dmabuf)
|
||||
.or_else(|| vk_frame.take().map(|(frame, views)| Retired::Vk { frame, views }));
|
||||
|
||||
let swapchains = [self.swapchain];
|
||||
let indices = [index];
|
||||
@@ -878,6 +1087,122 @@ impl Presenter {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Record the NV12→RGBA CSC pass into the video image (framebuffer): fullscreen
|
||||
/// triangle, CICP-driven push-constant rows. Shared by the dmabuf and Vulkan-Video
|
||||
/// paths — only the plane views bound beforehand differ.
|
||||
///
|
||||
/// # Safety
|
||||
/// `self.cmd_buf` must be in the recording state; the CSC descriptor set must point
|
||||
/// at live plane views.
|
||||
unsafe fn record_csc(
|
||||
&self,
|
||||
framebuffer: vk::Framebuffer,
|
||||
extent: vk::Extent2D,
|
||||
color: pf_client_core::video::ColorDesc,
|
||||
) {
|
||||
unsafe {
|
||||
self.device.cmd_begin_render_pass(
|
||||
self.cmd_buf,
|
||||
&vk::RenderPassBeginInfo::default()
|
||||
.render_pass(self.csc.render_pass)
|
||||
.framebuffer(framebuffer)
|
||||
.render_area(vk::Rect2D {
|
||||
offset: vk::Offset2D { x: 0, y: 0 },
|
||||
extent,
|
||||
}),
|
||||
vk::SubpassContents::INLINE,
|
||||
);
|
||||
self.device.cmd_bind_pipeline(
|
||||
self.cmd_buf,
|
||||
vk::PipelineBindPoint::GRAPHICS,
|
||||
self.csc.pipeline,
|
||||
);
|
||||
self.device.cmd_set_viewport(
|
||||
self.cmd_buf,
|
||||
0,
|
||||
&[vk::Viewport {
|
||||
x: 0.0,
|
||||
y: 0.0,
|
||||
width: extent.width as f32,
|
||||
height: extent.height as f32,
|
||||
min_depth: 0.0,
|
||||
max_depth: 1.0,
|
||||
}],
|
||||
);
|
||||
self.device.cmd_set_scissor(
|
||||
self.cmd_buf,
|
||||
0,
|
||||
&[vk::Rect2D {
|
||||
offset: vk::Offset2D { x: 0, y: 0 },
|
||||
extent,
|
||||
}],
|
||||
);
|
||||
self.device.cmd_bind_descriptor_sets(
|
||||
self.cmd_buf,
|
||||
vk::PipelineBindPoint::GRAPHICS,
|
||||
self.csc.pipeline_layout,
|
||||
0,
|
||||
&[self.csc.desc_set],
|
||||
&[],
|
||||
);
|
||||
let rows = csc_rows(color);
|
||||
let bytes = std::slice::from_raw_parts(rows.as_ptr().cast::<u8>(), 48);
|
||||
self.device.cmd_push_constants(
|
||||
self.cmd_buf,
|
||||
self.csc.pipeline_layout,
|
||||
vk::ShaderStageFlags::FRAGMENT,
|
||||
0,
|
||||
bytes,
|
||||
);
|
||||
self.device.cmd_draw(self.cmd_buf, 3, 1, 0, 0);
|
||||
self.device.cmd_end_render_pass(self.cmd_buf);
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-plane views over a Vulkan-Video frame's multiplanar image (R8 luma +
|
||||
/// R8G8 chroma — the CSC pass's exact sampling contract; the frames pool was
|
||||
/// created MUTABLE_FORMAT for this). NV12 only, like the rest of the pipeline.
|
||||
fn vkframe_plane_views(&self, f: &VkVideoFrame) -> Result<[vk::ImageView; 2]> {
|
||||
if f.vk_format != vk::Format::G8_B8R8_2PLANE_420_UNORM.as_raw() {
|
||||
bail!(
|
||||
"Vulkan-Video pool format {} unsupported (expected G8_B8R8_2PLANE_420)",
|
||||
f.vk_format
|
||||
);
|
||||
}
|
||||
// img[0] is creation-constant (only the sync fields need the frames lock).
|
||||
let image = vk::Image::from_raw(
|
||||
unsafe { (*(f.vkframe as *const pf_ffvk::AVVkFrame)).img[0] } as u64,
|
||||
);
|
||||
let make = |aspect: vk::ImageAspectFlags, format: vk::Format| {
|
||||
unsafe {
|
||||
self.device.create_image_view(
|
||||
&vk::ImageViewCreateInfo::default()
|
||||
.image(image)
|
||||
.view_type(vk::ImageViewType::TYPE_2D)
|
||||
.format(format)
|
||||
.subresource_range(
|
||||
vk::ImageSubresourceRange::default()
|
||||
.aspect_mask(aspect)
|
||||
.level_count(1)
|
||||
.layer_count(1),
|
||||
),
|
||||
None,
|
||||
)
|
||||
}
|
||||
.context("vk-frame plane view")
|
||||
};
|
||||
let luma = make(vk::ImageAspectFlags::PLANE_0, vk::Format::R8_UNORM)?;
|
||||
let chroma = match make(vk::ImageAspectFlags::PLANE_1, vk::Format::R8G8_UNORM) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
unsafe { self.device.destroy_image_view(luma, None) };
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
Ok([luma, chroma])
|
||||
}
|
||||
|
||||
/// Copy the frame's RGBA into the staging buffer and (re)build the video image on a
|
||||
/// stream-size change. Rows keep their stride — `buffer_row_length` unpacks it.
|
||||
fn stage_frame(&mut self, f: &CpuFrame) -> Result<()> {
|
||||
@@ -947,34 +1272,30 @@ impl Presenter {
|
||||
let reqs = unsafe { self.device.get_image_memory_requirements(image) };
|
||||
let memory = self.allocate(reqs, vk::MemoryPropertyFlags::DEVICE_LOCAL)?;
|
||||
unsafe { self.device.bind_image_memory(image, memory, 0) }?;
|
||||
// The CSC pass renders into it — view + framebuffer, hw-capable devices only.
|
||||
let (view, framebuffer) = if let Some(hw) = &self.hw {
|
||||
let view = unsafe {
|
||||
self.device.create_image_view(
|
||||
&vk::ImageViewCreateInfo::default()
|
||||
.image(image)
|
||||
.view_type(vk::ImageViewType::TYPE_2D)
|
||||
.format(vk::Format::R8G8B8A8_UNORM)
|
||||
.subresource_range(subresource_range()),
|
||||
None,
|
||||
)
|
||||
}?;
|
||||
let attachments = [view];
|
||||
let framebuffer = unsafe {
|
||||
self.device.create_framebuffer(
|
||||
&vk::FramebufferCreateInfo::default()
|
||||
.render_pass(hw.csc.render_pass)
|
||||
.attachments(&attachments)
|
||||
.width(width)
|
||||
.height(height)
|
||||
.layers(1),
|
||||
None,
|
||||
)
|
||||
}?;
|
||||
(view, framebuffer)
|
||||
} else {
|
||||
(vk::ImageView::null(), vk::Framebuffer::null())
|
||||
};
|
||||
// The CSC pass renders into it — view + framebuffer, unconditional (Vulkan-Video
|
||||
// frames need the pass on every device, dmabuf-capable or not).
|
||||
let view = unsafe {
|
||||
self.device.create_image_view(
|
||||
&vk::ImageViewCreateInfo::default()
|
||||
.image(image)
|
||||
.view_type(vk::ImageViewType::TYPE_2D)
|
||||
.format(vk::Format::R8G8B8A8_UNORM)
|
||||
.subresource_range(subresource_range()),
|
||||
None,
|
||||
)
|
||||
}?;
|
||||
let attachments = [view];
|
||||
let framebuffer = unsafe {
|
||||
self.device.create_framebuffer(
|
||||
&vk::FramebufferCreateInfo::default()
|
||||
.render_pass(self.csc.render_pass)
|
||||
.attachments(&attachments)
|
||||
.width(width)
|
||||
.height(height)
|
||||
.layers(1),
|
||||
None,
|
||||
)
|
||||
}?;
|
||||
self.video = Some(VideoImage {
|
||||
image,
|
||||
memory,
|
||||
@@ -1071,9 +1392,8 @@ impl Drop for Presenter {
|
||||
self.device.destroy_image(v.image, None);
|
||||
self.device.free_memory(v.memory, None);
|
||||
}
|
||||
if let Some(hw) = self.hw.take() {
|
||||
hw.csc.destroy(&self.device);
|
||||
}
|
||||
self.hw.take();
|
||||
self.csc.destroy(&self.device);
|
||||
self.overlay_pipe.destroy(&self.device);
|
||||
for s in self.render_sems.drain(..) {
|
||||
self.device.destroy_semaphore(s, None);
|
||||
@@ -1195,6 +1515,95 @@ fn subresource_range() -> vk::ImageSubresourceRange {
|
||||
.layer_count(1)
|
||||
}
|
||||
|
||||
|
||||
/// The live sync state of an `AVVkFrame`, snapshotted under the frames lock.
|
||||
struct VkFrameSync {
|
||||
image: u64,
|
||||
semaphore: u64,
|
||||
sem_value: u64,
|
||||
layout: i32,
|
||||
queue_family: u32,
|
||||
}
|
||||
|
||||
/// Lock the frame and read its live sync state (the presenter's submit must wait
|
||||
/// `sem_value` and signal `sem_value + 1`). The lock is held until [`unlock_vkframe`].
|
||||
fn lock_vkframe(f: &VkVideoFrame) -> VkFrameSync {
|
||||
unsafe {
|
||||
let lock: unsafe extern "C" fn(*mut pf_ffvk::AVHWFramesContext, *mut pf_ffvk::AVVkFrame) =
|
||||
std::mem::transmute(f.lock_frame);
|
||||
let fc = f.frames_ctx as *mut pf_ffvk::AVHWFramesContext;
|
||||
let vkf = f.vkframe as *mut pf_ffvk::AVVkFrame;
|
||||
lock(fc, vkf);
|
||||
VkFrameSync {
|
||||
image: (*vkf).img[0] as u64,
|
||||
semaphore: (*vkf).sem[0] as u64,
|
||||
sem_value: (*vkf).sem_value[0],
|
||||
layout: (*vkf).layout[0] as i32,
|
||||
queue_family: (*vkf).queue_family[0],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Write the post-submission state back (FFmpeg waits these on its next use of the
|
||||
/// frame) and release the lock. On a failed submit only the lock is released.
|
||||
fn unlock_vkframe(f: &VkVideoFrame, sync: &VkFrameSync, submitted: bool, graphics_qf: u32) {
|
||||
unsafe {
|
||||
let vkf = f.vkframe as *mut pf_ffvk::AVVkFrame;
|
||||
if submitted {
|
||||
(*vkf).sem_value[0] = sync.sem_value + 1;
|
||||
(*vkf).layout[0] =
|
||||
vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL.as_raw() as pf_ffvk::VkImageLayout;
|
||||
if sync.queue_family != vk::QUEUE_FAMILY_IGNORED {
|
||||
(*vkf).queue_family[0] = graphics_qf;
|
||||
}
|
||||
}
|
||||
let unlock: unsafe extern "C" fn(
|
||||
*mut pf_ffvk::AVHWFramesContext,
|
||||
*mut pf_ffvk::AVVkFrame,
|
||||
) = std::mem::transmute(f.unlock_frame);
|
||||
unlock(f.frames_ctx as *mut pf_ffvk::AVHWFramesContext, vkf);
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquire a Vulkan-Video frame's image from the decode queue family (EXCLUSIVE
|
||||
/// sharing) and transition it for sampling. `src_qf == dst_qf` (or IGNORED/CONCURRENT)
|
||||
/// degrades to a plain layout transition. The matching decode-side acquire happens in
|
||||
/// FFmpeg, keyed off the queue_family we write back after submission.
|
||||
fn vkframe_acquire_barrier(
|
||||
device: &ash::Device,
|
||||
cmd: vk::CommandBuffer,
|
||||
image: vk::Image,
|
||||
old_layout: vk::ImageLayout,
|
||||
src_qf: u32,
|
||||
dst_qf: u32,
|
||||
) {
|
||||
let (src, dst) = if src_qf == dst_qf || src_qf == vk::QUEUE_FAMILY_IGNORED {
|
||||
(vk::QUEUE_FAMILY_IGNORED, vk::QUEUE_FAMILY_IGNORED)
|
||||
} else {
|
||||
(src_qf, dst_qf)
|
||||
};
|
||||
let b = vk::ImageMemoryBarrier::default()
|
||||
.src_access_mask(vk::AccessFlags::empty())
|
||||
.dst_access_mask(vk::AccessFlags::SHADER_READ)
|
||||
.old_layout(old_layout)
|
||||
.new_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL)
|
||||
.src_queue_family_index(src)
|
||||
.dst_queue_family_index(dst)
|
||||
.image(image)
|
||||
.subresource_range(subresource_range());
|
||||
unsafe {
|
||||
device.cmd_pipeline_barrier(
|
||||
cmd,
|
||||
vk::PipelineStageFlags::TOP_OF_PIPE,
|
||||
vk::PipelineStageFlags::FRAGMENT_SHADER,
|
||||
vk::DependencyFlags::empty(),
|
||||
&[],
|
||||
&[],
|
||||
&[b],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquire a dmabuf plane image from its foreign owner (the VAAPI decoder): queue-family
|
||||
/// transfer FOREIGN → ours, UNDEFINED → SHADER_READ_ONLY (content is preserved across
|
||||
/// the transfer regardless of the UNDEFINED old-layout, per the external-memory rules).
|
||||
|
||||
Reference in New Issue
Block a user