diff --git a/Cargo.lock b/Cargo.lock index 7043fb23..726b786d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2994,6 +2994,7 @@ dependencies = [ "opus", "pf-ffvk", "pf-update-check", + "pf-vkdecode", "pipewire", "punktfunk-core", "pyrowave-sys", diff --git a/crates/pf-client-core/Cargo.toml b/crates/pf-client-core/Cargo.toml index 40212ef7..07a19ee5 100644 --- a/crates/pf-client-core/Cargo.toml +++ b/crates/pf-client-core/Cargo.toml @@ -17,6 +17,10 @@ repository.workspace = true punktfunk-core = { path = "../punktfunk-core", features = ["quic"] } # FFmpeg's Vulkan hwcontext surface (Vulkan Video decode on the presenter's device). pf-ffvk = { path = "../pf-ffvk" } +# Native Vulkan Video H.264 decode (WP-C of the native-decode program): the opt-in +# `PUNKTFUNK_DECODER=native-vulkan` backend in video_vk_native.rs, running pf-vkdecode's +# VkH264Decoder on the presenter's shared device. +pf-vkdecode = { path = "../pf-vkdecode" } async-channel = "2" # Video decode (same FFmpeg pin as the host) and Opus for the audio planes. diff --git a/crates/pf-client-core/src/lib.rs b/crates/pf-client-core/src/lib.rs index e8ef8fbc..e286d931 100644 --- a/crates/pf-client-core/src/lib.rs +++ b/crates/pf-client-core/src/lib.rs @@ -77,6 +77,11 @@ mod video_software; mod video_libav; #[cfg(target_os = "linux")] mod video_vaapi; +// Native Vulkan Video H.264 decode (WP-C of the native-decode program): pf-vkdecode's +// decoder on the presenter's shared device, behind the `PUNKTFUNK_DECODER=native-vulkan` +// runtime opt-in only. +#[cfg(any(target_os = "linux", windows))] +mod video_vk_native; #[cfg(any(target_os = "linux", windows))] mod video_vulkan; // The OS-clipboard bridge for the shared clipboard (design/clipboard-and-file-transfer.md §5). diff --git a/crates/pf-client-core/src/session.rs b/crates/pf-client-core/src/session.rs index 0c23df98..f9bd8312 100644 --- a/crates/pf-client-core/src/session.rs +++ b/crates/pf-client-core/src/session.rs @@ -771,6 +771,7 @@ fn pump( DecodedImage::D3d11(_) => "d3d11va", #[cfg(all(any(target_os = "linux", windows), feature = "pyrowave"))] DecodedImage::PyroWave(_) => "pyrowave", + DecodedImage::NativeVk(_) => "native-vulkan", }; if total_frames == 1 { let (w, h, path) = match &image { @@ -785,6 +786,7 @@ fn pump( feature = "pyrowave" ))] DecodedImage::PyroWave(f) => (f.width, f.height, "pyrowave"), + DecodedImage::NativeVk(f) => (f.width, f.height, "native-vulkan"), }; tracing::info!(width = w, height = h, path, "first frame decoded"); } diff --git a/crates/pf-client-core/src/video.rs b/crates/pf-client-core/src/video.rs index 74a71965..b1836b51 100644 --- a/crates/pf-client-core/src/video.rs +++ b/crates/pf-client-core/src/video.rs @@ -4,7 +4,10 @@ //! see [`VulkanDecodeDevice::prefer_vulkan_first`]. Linux: vaapi → vulkan → software on //! desktop Mesa, vulkan first on NVIDIA/VanGogh. Windows: d3d11va → vulkan → software on //! Intel/unknown, vulkan first on NVIDIA/AMD. -//! Override: `PUNKTFUNK_DECODER=vulkan|vaapi|d3d11va|software`): +//! Override: `PUNKTFUNK_DECODER=vulkan|vaapi|d3d11va|software`; additionally +//! `native-vulkan` — the pf-vkdecode H.264 decoder on the presenter's device +//! (`video_vk_native`), runtime-opt-in ONLY until WP-D's A/B verdict admits it to the +//! ladder — see [`native_vulkan_gate`]): //! //! * **Vulkan Video**: FFmpeg's Vulkan decoder running on the PRESENTER's own VkDevice //! (its handles arrive via [`VulkanDecodeDevice`]) — the decoded VkImage feeds the @@ -44,6 +47,7 @@ pub use crate::video_color::{csc_rows, ColorDesc}; use crate::video_software::SoftwareDecoder; #[cfg(target_os = "linux")] use crate::video_vaapi::VaapiDecoder; +use crate::video_vk_native::NativeVulkanDecoder; use crate::video_vulkan::VulkanDecoder; /// One decoded frame headed for the presenter, carrying the host capture timestamp so the @@ -79,6 +83,13 @@ pub enum DecodedImage { /// samples them directly (BT.709 limited, the codec's fixed colour contract). #[cfg(all(any(target_os = "linux", windows), feature = "pyrowave"))] PyroWave(crate::video_pyrowave::PyroWavePlanarFrame), + /// Native Vulkan Video output (pf-vkdecode, `PUNKTFUNK_DECODER=native-vulkan`): + /// an NV12 image + per-plane views already on the PRESENTER's device — same + /// zero-copy contract as [`DecodedImage::VkFrame`], no FFmpeg involved. The + /// presenter waits the frame's timeline pair, transitions the layer for sampling + /// and BACK to [`NativeVkFrame::layout`], and releases the decoder's slot by + /// dropping the frame (its guard sends the release token). + NativeVk(NativeVkFrame), } /// One Vulkan-decoded frame. The image lives on the presenter's own VkDevice (the @@ -136,6 +147,109 @@ pub struct VkVideoFrame { pub guard: DrmFrameGuard, } +/// The layout a [`NativeVkFrame`]'s image layer is in when its semaphore signals — +/// pf-client-core's ash-free mirror of the two decode layouts, so the presenter can +/// transition for sampling and back without this crate naming `vk::ImageLayout`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NativeVkLayout { + /// `VIDEO_DECODE_DST_KHR` — distinct-mode output; the layer holds ONLY this + /// picture and the next decode into the slot discards it (UNDEFINED-old-layout). + DecodeDst, + /// `VIDEO_DECODE_DPB_KHR` — coincide-mode output: the picture IS a DPB slot and + /// may still be a live reference, so a consumer that transitions it for sampling + /// MUST transition it back to this layout in the same submission. + DecodeDpb, +} + +/// The release token a presented/dropped [`NativeVkFrame`] hands back to the native +/// decode backend: `seq` names the shipped frame, `generation` the decoder session it +/// belongs to (a stale generation releases nothing — the pools it indexed are gone). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NativeReleaseToken { + pub seq: u64, + pub generation: u64, +} + +/// Sends the frame's [`NativeReleaseToken`] exactly once, on drop — the native path's +/// analog of the VAAPI/VkFrame `DrmFrameGuard`s. The presenter holds the frame (and so +/// this guard) until its sampling submission's fence has been waited, which makes +/// "guard dropped" equal "the GPU is done with the image"; a frame dropped UNPRESENTED +/// (newest-wins displacement, demotion drain) releases through the very same drop. A +/// dead channel (the backend was demoted/rebuilt) is ignored — the decoder that owned +/// the slot is gone. +pub struct NativeReleaseGuard { + tx: std::sync::mpsc::Sender, + token: Option, +} + +impl NativeReleaseGuard { + pub(crate) fn new( + tx: std::sync::mpsc::Sender, + token: NativeReleaseToken, + ) -> Self { + Self { + tx, + token: Some(token), + } + } +} + +impl Drop for NativeReleaseGuard { + fn drop(&mut self) { + if let Some(token) = self.token.take() { + let _ = self.tx.send(token); + } + } +} + +/// One natively decoded frame (pf-vkdecode). Everything is raw `u64`/plain data — this +/// crate stays ash-free, exactly like [`VulkanDecodeDevice`]. The handles BORROW the +/// decoder's pools: valid until the frame is released (the guard's drop) AND the +/// decoder generation they carry is current — the backend keeps the decoder alive +/// until every shipped frame's token has come back (bounded), so the presenter never +/// has to validate liveness itself. +pub struct NativeVkFrame { + /// The decode image (raw `VkImage`); the picture occupies array layer [`Self::layer`]. + pub image: u64, + /// `R8`/`R8G8` per-plane views (raw `VkImageView`s) — the presenter's planar CSC + /// sampling contract, same shape as the FFmpeg path's derived plane views. + pub plane_views: [u64; 2], + pub layer: u32, + /// The layout the layer is in when the semaphore signals; the presenter must + /// return it there after sampling (see [`NativeVkLayout`]). + pub layout: NativeVkLayout, + /// Timeline pair (raw `VkSemaphore` + value): pixels are ready when the semaphore + /// reaches the value — the presenter waits it on the GPU (submit wait list, like + /// the AVVkFrame path), never on the host. + pub semaphore: u64, + pub semaphore_value: u64, + /// The decoder session generation the handles belong to (rides the release token). + pub generation: u64, + /// Display size (the conformance-window crop) — what [`DecodedImage::dimensions`] + /// reports and what the presenter shows. + pub width: u32, + pub height: u32, + /// The image's allocated/coded extent (`>=` display) — the presenter scales its + /// sampling UVs by display/coded per axis or the alignment padding smears into + /// view (the 1088-row lesson; same contract as [`VkVideoFrame::coded_width`]). + pub coded_width: u32, + pub coded_height: u32, + /// Crop origin within the coded picture. Punktfunk hosts emit origin crops only; + /// the presenter's UV-scale path assumes (0,0) and a nonzero origin would show the + /// wrong window — carried so that assumption is checkable, not silent. + pub crop_x: u32, + pub crop_y: u32, + /// Colour signalling. The native H.264 path serves the SDR envelope; the backend + /// fills H.273 "unspecified" code points, which every consumer already resolves to + /// the BT.709-limited SDR default (`csc_rows`' documented fallback). + pub color: ColorDesc, + /// IDR — the stream's re-anchor point (the pump's post-loss resume signal). + pub keyframe: bool, + pub poc: i32, + /// Sends the release token on drop — see [`NativeReleaseGuard`]. + pub guard: NativeReleaseGuard, +} + /// True if the decoder tagged this frame as a full IDR keyframe — a guaranteed clean re-anchor /// after which the picture is loss-free, so the pump can lift a post-loss display freeze here. /// @@ -172,6 +286,7 @@ impl DecodedImage { DecodedImage::D3d11(f) => f.keyframe, #[cfg(all(any(target_os = "linux", windows), feature = "pyrowave"))] DecodedImage::PyroWave(f) => f.keyframe, + DecodedImage::NativeVk(f) => f.keyframe, } } @@ -188,6 +303,7 @@ impl DecodedImage { DecodedImage::D3d11(f) => (f.width, f.height), #[cfg(all(any(target_os = "linux", windows), feature = "pyrowave"))] DecodedImage::PyroWave(f) => (f.width, f.height), + DecodedImage::NativeVk(f) => (f.width, f.height), } } } @@ -254,6 +370,13 @@ impl Drop for DrmFrameGuard { enum Backend { Vulkan(VulkanDecoder), + /// Native Vulkan Video H.264 (pf-vkdecode) on the presenter's device — runtime + /// opt-in only (`PUNKTFUNK_DECODER=native-vulkan`, see [`native_vulkan_gate`]); + /// not in the automatic ladder until WP-D's A/B verdict. Errors ride the SAME + /// streak/demotion machinery as the FFmpeg-Vulkan rung. + /// Boxed: the decoder (planner + shipped-frame ledger) dwarfs the other variants, + /// same as PyroWave below. + NativeVulkan(Box), #[cfg(target_os = "linux")] Vaapi(VaapiDecoder), #[cfg(windows)] @@ -312,6 +435,17 @@ const VAAPI_DEMOTE_AFTER: u32 = 3; /// software before the first requested IDR could even arrive. const HW_DEMOTE_MIN_STREAK: std::time::Duration = std::time::Duration::from_millis(1000); +/// The native Vulkan Video opt-in gate (WP-C of the native-decode program): the +/// pf-vkdecode backend engages ONLY when the operator asked for it by name +/// (`PUNKTFUNK_DECODER=native-vulkan` — `choice` is env-first, so that's what carries +/// it), the negotiated wire codec is H.264 (the one codec pf-vkdecode speaks), and the +/// presenter's device actually advertises Vulkan Video decode. Deliberately NOT an +/// `auto` rung: entering the automatic ladder is WP-D's A/B verdict. Pure so the +/// decision is CPU-testable. +fn native_vulkan_gate(choice: &str, codec_id: ffmpeg::codec::Id, video_decode: bool) -> bool { + choice == "native-vulkan" && codec_id == ffmpeg::codec::Id::H264 && video_decode +} + /// Map a negotiated `quic` codec bit to the FFmpeg decoder id the client opens. pub fn ffmpeg_codec_id(wire: u8) -> ffmpeg::codec::Id { match wire { @@ -500,6 +634,38 @@ impl Decoder { d3d11_hdr10, }) }; + // Native Vulkan Video (pf-vkdecode) — strictly the runtime opt-in + // (`PUNKTFUNK_DECODER=native-vulkan`); [`native_vulkan_gate`] is the whole + // decision. Any refusal or init failure logs and DEMOTES to the standard + // ladder below exactly as if the native rung errored (choice reads as `auto` + // from here on) — a native failure must never be quieter, or land somewhere + // other, than the FFmpeg rungs' failures do. + let mut choice = choice; + if choice == "native-vulkan" { + if native_vulkan_gate(&choice, codec_id, vk.is_some_and(|v| v.video_decode)) { + let vk = vk.expect("gate demands video_decode, so vk is Some"); + match NativeVulkanDecoder::new(vk) { + Ok(n) => { + tracing::info!( + ?codec_id, + "native Vulkan Video hardware decode active \ + (pf-vkdecode, presenter-shared device)" + ); + return done(Backend::NativeVulkan(Box::new(n))); + } + Err(e) => tracing::warn!(reason = %format!("{e:#}"), + "native Vulkan decode init failed — demoting to the standard ladder"), + } + } else { + tracing::warn!( + ?codec_id, + video_decode = vk.is_some_and(|v| v.video_decode), + "PUNKTFUNK_DECODER=native-vulkan refused (needs an H.264 session and a \ + Vulkan-Video-capable presenter device) — standard ladder" + ); + } + choice = "auto".to_string(); + } // Linux `auto`: try VAAPI FIRST unless this device is one where Vulkan Video is // the established right answer (NVIDIA — no usable VAAPI; VanGogh — VAAPI // chroma-fringes). Mesa now exposes decode queues by default (and the session @@ -774,6 +940,10 @@ impl Decoder { debug_assert!(complete, "partial AUs are pyrowave-only"); v.decode(au).map(|f| f.map(DecodedImage::VkFrame)) } + Backend::NativeVulkan(n) => { + debug_assert!(complete, "partial AUs are pyrowave-only"); + n.decode(au).map(|f| f.map(DecodedImage::NativeVk)) + } #[cfg(target_os = "linux")] Backend::Vaapi(v) => v.decode(au).map(|f| f.map(DecodedImage::Dmabuf)), #[cfg(windows)] @@ -799,6 +969,7 @@ impl Decoder { Err(e) => { let which = match self.backend { Backend::Vulkan(_) => "Vulkan Video", + Backend::NativeVulkan(_) => "native Vulkan Video", #[cfg(windows)] Backend::D3d11va(_) => "D3D11VA", _ => "VAAPI", @@ -808,12 +979,14 @@ impl Decoder { let first = *self.first_fail.get_or_insert_with(std::time::Instant::now); if self.vaapi_fails >= VAAPI_DEMOTE_AFTER && first.elapsed() >= HW_DEMOTE_MIN_STREAK { - // A failing Vulkan backend still has a hardware rung below it on - // Linux — demote to VAAPI first (user-reported: FFmpeg-Vulkan-on-Mesa - // error-streaking where VAAPI streams perfectly); only when that - // can't be built either does the session land on software. + // A failing Vulkan backend (FFmpeg or native — the native rung + // demotes exactly like the FFmpeg one) still has a hardware rung + // below it on Linux — demote to VAAPI first (user-reported: + // FFmpeg-Vulkan-on-Mesa error-streaking where VAAPI streams + // perfectly); only when that can't be built either does the + // session land on software. #[cfg(target_os = "linux")] - if matches!(self.backend, Backend::Vulkan(_)) { + if matches!(self.backend, Backend::Vulkan(_) | Backend::NativeVulkan(_)) { match VaapiDecoder::new(self.codec_id) { Ok(v) => { tracing::warn!(error = %e, fails = self.vaapi_fails, @@ -828,10 +1001,13 @@ impl Decoder { "VAAPI unavailable for demotion — software decode"), } } - // Windows' hardware rung below Vulkan is D3D11VA (a 4K120 stream is - // not survivable on software) — same-GPU rebuild via the stashed LUID. + // Windows' hardware rung below Vulkan (FFmpeg or native) is D3D11VA + // (a 4K120 stream is not survivable on software) — same-GPU rebuild + // via the stashed LUID. #[cfg(windows)] - if matches!(self.backend, Backend::Vulkan(_)) && self.d3d11_import { + if matches!(self.backend, Backend::Vulkan(_) | Backend::NativeVulkan(_)) + && self.d3d11_import + { match crate::video_d3d11::D3d11vaDecoder::new( self.codec_id, self.adapter_luid, @@ -1132,6 +1308,24 @@ mod tests { assert!(!decode_device(0x8086, "Intel(R) Arc(TM) Pro Graphics").prefer_vulkan_first()); } + /// The native-Vulkan opt-in gate (WP-C): by name only, H.264 only, and only on a + /// device that really decodes — every other combination takes the standard ladder. + /// Pinned so "native enters auto" can only ever be a deliberate WP-D change. + #[test] + fn native_vulkan_gate_is_by_name_h264_and_capable_device_only() { + use ffmpeg::codec::Id; + assert!(native_vulkan_gate("native-vulkan", Id::H264, true)); + // Never by any other preference — it is not an auto rung yet. + for choice in ["auto", "", "hardware", "vulkan", "software"] { + assert!(!native_vulkan_gate(choice, Id::H264, true), "{choice:?}"); + } + // The one codec pf-vkdecode speaks. + assert!(!native_vulkan_gate("native-vulkan", Id::HEVC, true)); + assert!(!native_vulkan_gate("native-vulkan", Id::AV1, true)); + // No Vulkan-Video-capable presenter device. + assert!(!native_vulkan_gate("native-vulkan", Id::H264, false)); + } + /// Lock the DRM FourCC magic numbers against typos — these are the exact values /// `` defines, and a wrong one is what painted the Steam Deck green. #[test] diff --git a/crates/pf-client-core/src/video_vk_native.rs b/crates/pf-client-core/src/video_vk_native.rs new file mode 100644 index 00000000..a085dd17 --- /dev/null +++ b/crates/pf-client-core/src/video_vk_native.rs @@ -0,0 +1,558 @@ +//! Native Vulkan Video H.264 decode backend (WP-C of the native-decode program): +//! pf-vkdecode's [`VkH264Decoder`] running on the PRESENTER's own VkDevice — the same +//! zero-copy shape as the FFmpeg-Vulkan backend, with no FFmpeg in the path. Strictly +//! the `PUNKTFUNK_DECODER=native-vulkan` runtime opt-in (`video::native_vulkan_gate`); +//! the automatic ladder stays FFmpeg's until WP-D's A/B verdict. +//! +//! **Queue lock:** pf-vkdecode submits on queue 0 of the decode family +//! ([`DECODE_QUEUE_INDEX`] — the presenter creates exactly one queue per family). When +//! the decode family IS the presenter's graphics family, that is the very `VkQueue` the +//! presenter/Skia/overlay submit and present on, so every decode submit must hold the +//! device's shared [`video::QueueLock`] (`vkQueueSubmit` external sync — the 2026-07-09 +//! `VK_ERROR_DEVICE_LOST` class). When the families differ, the decode queue has exactly +//! one submitter (this backend, on the pump thread) and locking would serialize decode +//! against present for nothing — [`submit_queues_collide`] is the whole decision. (The +//! FFmpeg path locks on every family only because `lock_queue` is one callback pair for +//! the whole device; the collision it exists to prevent is the shared-queue one.) +//! +//! **Release lifecycle** (decode → present → retire → release): each delivered frame +//! ships as a [`NativeVkFrame`] whose [`NativeReleaseGuard`] sends a token (seq + +//! generation) into this backend's channel on drop. The presenter drops the frame only +//! after the sampling submission's fence has been waited (its retired-frame slot), so a +//! returned token proves the GPU is done with the image; a frame dropped UNPRESENTED +//! (newest-wins displacement, post-demotion drain) releases through the same drop. The +//! backend drains the channel at every `decode` entry and calls +//! [`VkH264Decoder::release_frame`] — but only once the frame's decode-status query has +//! also been read (the slot stays pinned meanwhile, which is what makes re-polling the +//! query safe: an unreleased slot can never be recycled under the poll). +//! +//! **Status queries:** every decode op carries a `RESULT_STATUS_ONLY` query — +//! [`VkH264Decoder::poll_status`], read non-blockingly here at each decode entry. A +//! `Failed` verdict is driver-reported decode corruption, the class FFmpeg's +//! `vulkan_decode.c` (`nb_queries = 0`) architecturally cannot see — the Xbox Ally X +//! field case. It surfaces as an `Err` from the CURRENT `decode_frame` call so the +//! existing streak/reanchor machinery fires exactly as it does for FFmpeg errors. +//! +//! **Teardown:** dropping this backend (demotion, session end) waits — bounded — for +//! every shipped frame's token before dropping the decoder, because the decoder's Drop +//! destroys the pool images and its own drain only covers DECODE work, not the +//! presenter's in-flight sampling. Tokens arrive as the presenter's fence waits/drops +//! displace the frames; a presenter wedged past [`TEARDOWN_BUDGET`] forfeits (warned). + +use crate::video::{ + ColorDesc, NativeReleaseGuard, NativeReleaseToken, NativeVkFrame, NativeVkLayout, + VulkanDecodeDevice, +}; +use anyhow::{anyhow, bail, Result}; +use pf_vkdecode::ash::vk; +use pf_vkdecode::ash::vk::Handle as _; +use pf_vkdecode::{DecodeStatus, DecodedVkFrame, DeviceHandles, VkH264Decoder}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +/// The queue index this backend submits on within the decode family: the presenter +/// creates exactly ONE queue (index 0) per family it enables (`vk/setup.rs` — one +/// `VkDeviceQueueCreateInfo` per family, `queue_count = 1`), so 0 is the only queue +/// that exists. +const DECODE_QUEUE_INDEX: u32 = 0; + +/// Teardown budget for the presenter to hand back every outstanding frame token (its +/// next present's fence wait, typically one frame). Generous against a paused stream, +/// finite against a wedged presenter — after this the pools are destroyed anyway +/// (warned; the realistic residue is a logically-held frame, not in-flight GPU work). +const TEARDOWN_BUDGET: Duration = Duration::from_millis(500); + +/// Query-poll belt: a frame whose token has returned had its decode op complete on the +/// GPU (the presenter's submit waited the decode timeline), so its status query MUST be +/// readable — if it still reads Pending after this many polls, give the slot back +/// anyway rather than strand it (debug-logged; the status is then simply unknown). +const MAX_POLLS_AFTER_RELEASE: u32 = 3; + +/// Do the presenter's and the decoder's submit queues collide? Both sides use queue +/// index 0 of their family by construction (the presenter's graphics queue is +/// `get_device_queue(qfi, 0)`, the decoder's is [`DECODE_QUEUE_INDEX`] of `decode_qf`), +/// so the collision test is family equality. Pure — the queue-lock decision is +/// CPU-testable. +fn submit_queues_collide(graphics_qf: u32, decode_qf: u32) -> bool { + graphics_qf == decode_qf +} + +/// [`pf_vkdecode::QueueLock`] over the device's shared [`crate::video::QueueLock`] — +/// or over nothing, when the decode queue provably has no other submitter (see the +/// module doc's queue-lock section). +enum NativeQueueLock { + /// Decode shares the presenter's graphics queue: serialize with everyone. + Shared(std::sync::Arc), + /// A separate decode family/queue: this backend is its only submitter. + Uncontended, +} + +impl pf_vkdecode::QueueLock for NativeQueueLock { + fn lock(&self) { + if let NativeQueueLock::Shared(l) = self { + l.lock(); + } + } + fn unlock(&self) { + if let NativeQueueLock::Shared(l) = self { + l.unlock(); + } + } +} + +/// One frame shipped to the presenter and not yet fully settled: settled = its release +/// token came back (GPU reads proven done) AND its status query was read. +struct Shipped { + seq: u64, + frame: DecodedVkFrame, + /// The presenter (or a drop on the way there) returned the token. + released: bool, + /// The status query read a conclusive verdict (or the poll belt expired). + resolved: bool, + /// Polls attempted after the token returned — see [`MAX_POLLS_AFTER_RELEASE`]. + polls_after_release: u32, +} + +/// Mark the shipped entry a token names as released. Returns false when nothing +/// matches (a late token from before a demotion drain — benign). Pure bookkeeping, +/// split out so the channel-drain behavior is CPU-testable. +fn note_token(outstanding: &mut [Shipped], token: NativeReleaseToken) -> bool { + match outstanding.iter_mut().find(|s| s.seq == token.seq) { + Some(s) => { + debug_assert_eq!( + s.frame.generation, token.generation, + "a token's generation always matches the frame it rode on" + ); + s.released = true; + true + } + None => false, + } +} + +/// The native backend: the decoder plus the shipped-frame ledger and release channel. +pub(crate) struct NativeVulkanDecoder { + dec: VkH264Decoder, + /// Cloned into every shipped frame's guard. + release_tx: mpsc::Sender, + release_rx: mpsc::Receiver, + outstanding: Vec, + next_seq: u64, +} + +// SAFETY: the decoder is used strictly serially through `&mut self` from whichever +// single thread owns the enclosing `Decoder` (the session pump) — `Send` only moves +// that ownership. The `Rc`s inside pf-vkdecode's planner never escape it, so they all +// move together; every queue submission runs under the collision-aware queue lock; the +// mpsc endpoints are `Send`. Same contract, same shape as the `VulkanDecoder` and +// `PyroWaveDecoder` impls above/beside it. Deliberately NOT `Sync`. +unsafe impl Send for NativeVulkanDecoder {} + +impl NativeVulkanDecoder { + pub(crate) fn new(vk: &VulkanDecodeDevice) -> Result { + if !vk.video_decode { + bail!("presenter device lacks Vulkan Video decode"); + } + let lock: Box = + if submit_queues_collide(vk.graphics_qf, vk.decode_qf) { + Box::new(NativeQueueLock::Shared(vk.queue_lock.clone())) + } else { + Box::new(NativeQueueLock::Uncontended) + }; + let handles = DeviceHandles { + get_instance_proc_addr: vk.get_instance_proc_addr, + instance: vk.instance, + physical_device: vk.physical_device, + device: vk.device, + decode_qf: vk.decode_qf, + decode_queue_index: DECODE_QUEUE_INDEX, + graphics_qf: vk.graphics_qf, + }; + // SAFETY: the handles are the presenter's live instance/device, which outlives + // every session pump (the run loop tears the pump — and with it this decoder — + // down first: the exact liveness contract the FFmpeg and PyroWave backends + // already rely on over the same bundle). `video_decode` (checked above) is set + // only when the presenter enabled the Vulkan Video decode extension stack + + // synchronization2/timelineSemaphore at device creation, and + // `decode_qf`/`graphics_qf` mirror the families it created queues for (one + // queue, index 0, each). + let dec = unsafe { VkH264Decoder::new(&handles, lock) } + .map_err(|e| anyhow!("VkH264Decoder init: {e}"))?; + let (release_tx, release_rx) = mpsc::channel(); + Ok(NativeVulkanDecoder { + dec, + release_tx, + release_rx, + outstanding: Vec::new(), + next_seq: 0, + }) + } + + /// Feed one complete access unit. `Ok(None)` = no display-ready picture (the pump's + /// no-output/reanchor machinery reads that exactly as it does for FFmpeg). `Err` = + /// decode trouble — a decoder error, a plan that needed concealment, or a + /// driver-reported corrupt PREVIOUS frame — routed through the caller's shared + /// streak/demotion machinery. + pub(crate) fn decode(&mut self, au: &[u8]) -> Result> { + self.drain_releases(); + let corrupt = self.settle_statuses(); + if corrupt > 0 { + // Driver-reported decode corruption on an already-delivered frame — the + // Ally X class, invisible to FFmpeg's query-less decoder. The frame is on + // (or past) the glass; erroring THIS call is what arms the reanchor gate + // and gets the IDR that replaces the corrupt content. + return Err(anyhow!( + "driver reported decode corruption on {corrupt} prior frame(s) \ + (RESULT_STATUS_ONLY query) — re-anchor needed" + )); + } + + let delivered = self.dec.decode(au).map_err(|e| anyhow!("decode: {e}"))?; + let warnings = self.dec.take_warnings(); + if !warnings.is_empty() { + // The AU was planned around missing/damaged references: the picture + // decodes, but its content is concealed. Release it unshown and surface + // the AU as decode trouble — same path, same volume as an FFmpeg + // reference-miss error (never quieter). + if let Some(frame) = &delivered { + if let Err(e) = self.dec.release_frame(frame) { + tracing::debug!(error = %e, "releasing a concealed frame failed"); + } + } + tracing::warn!( + ?warnings, + "native decode planned with concealment — dropping the frame, \ + requesting re-anchor" + ); + bail!( + "AU planned with concealment ({} warning(s))", + warnings.len() + ); + } + + Ok(delivered.map(|frame| self.ship(frame))) + } + + /// Wrap a delivered [`DecodedVkFrame`] for the presenter and enter it into the + /// shipped ledger (the original stays here — release/poll need it). + fn ship(&mut self, frame: DecodedVkFrame) -> NativeVkFrame { + let seq = self.next_seq; + self.next_seq += 1; + let token = NativeReleaseToken { + seq, + generation: frame.generation, + }; + let native = NativeVkFrame { + image: frame.image.as_raw(), + plane_views: [frame.plane_views[0].as_raw(), frame.plane_views[1].as_raw()], + layer: frame.layer, + layout: if frame.layout == vk::ImageLayout::VIDEO_DECODE_DPB_KHR { + NativeVkLayout::DecodeDpb + } else { + NativeVkLayout::DecodeDst + }, + semaphore: frame.semaphore.as_raw(), + semaphore_value: frame.value, + generation: frame.generation, + width: frame.crop.width, + height: frame.crop.height, + coded_width: frame.coded_width, + coded_height: frame.coded_height, + crop_x: frame.crop.x, + crop_y: frame.crop.y, + // H.273 "unspecified" — every consumer resolves it to the BT.709-limited + // SDR default, the native H.264 envelope's colour contract (`csc_rows`). + color: ColorDesc { + primaries: 2, + transfer: 2, + matrix: 2, + full_range: false, + }, + keyframe: frame.is_idr, + poc: frame.poc, + guard: NativeReleaseGuard::new(self.release_tx.clone(), token), + }; + self.outstanding.push(Shipped { + seq, + frame, + released: false, + resolved: false, + polls_after_release: 0, + }); + native + } + + /// Drain the release channel, marking returned frames (release itself waits for + /// the status read — see [`Self::settle_statuses`]). + fn drain_releases(&mut self) { + while let Ok(token) = self.release_rx.try_recv() { + if !note_token(&mut self.outstanding, token) { + tracing::debug!( + seq = token.seq, + generation = token.generation, + "release token without an outstanding frame" + ); + } + } + } + + /// Poll the status query of every unresolved shipped frame (non-blocking) and + /// release the ones that are both status-settled and token-returned. Returns how + /// many frames NEWLY read `Failed` — driver-reported corruption. + /// + /// Polling an unreleased frame is always sound: its slot is pinned until + /// `release_frame`, so the query slot it names cannot have been recycled under it + /// (the false-`Failed` a recycled slot would read). + fn settle_statuses(&mut self) -> u32 { + let mut corrupt = 0u32; + let Self { + dec, outstanding, .. + } = self; + for s in outstanding.iter_mut() { + if s.resolved { + continue; + } + // A session rebuild (stream renegotiation) already made this frame stale: + // its pools are gone and a status poll would read the conservative Failed + // — which is NOT driver corruption. Resolve it quietly; the rebuild rode + // an IDR, so the stream has its re-anchor already. + if s.frame.generation != dec.generation() { + tracing::debug!( + poc = s.frame.poc, + frame_generation = s.frame.generation, + "outstanding frame outlived its session generation — status unknowable" + ); + s.resolved = true; + continue; + } + match dec.poll_status(&s.frame) { + DecodeStatus::Ok => s.resolved = true, + DecodeStatus::Failed => { + s.resolved = true; + corrupt += 1; + tracing::warn!( + poc = s.frame.poc, + slot = s.frame.query_slot, + "decode status query: Failed (driver-reported corruption)" + ); + } + DecodeStatus::Pending => { + if s.released { + // Token back ⇒ the decode op completed before the presenter's + // sampling ⇒ the query should be readable. Belt, not a path. + s.polls_after_release += 1; + if s.polls_after_release >= MAX_POLLS_AFTER_RELEASE { + tracing::debug!( + poc = s.frame.poc, + "status query still pending after release — giving \ + the slot back with an unknown verdict" + ); + s.resolved = true; + } + } + } + } + } + outstanding.retain(|s| { + if !(s.released && s.resolved) { + return true; + } + match dec.release_frame(&s.frame) { + Ok(()) => {} + // A session rebuild (stream renegotiation) already dropped the pools + // this frame indexed — nothing left to release. + Err(e) => tracing::debug!(error = %e, "release_frame: {e}"), + } + false + }); + corrupt + } +} + +impl Drop for NativeVulkanDecoder { + fn drop(&mut self) { + // Wait (bounded) for the presenter to hand back every shipped frame before the + // decoder's Drop destroys the pool images: a returned token proves the + // sampling submission's fence was waited, i.e. no GPU work of the presenter's + // still reads the pools (the decoder's own drain covers only decode work). + let deadline = Instant::now() + TEARDOWN_BUDGET; + loop { + self.drain_releases(); + self.outstanding.retain(|s| !s.released); + if self.outstanding.is_empty() { + break; + } + let now = Instant::now(); + if now >= deadline { + tracing::warn!( + outstanding = self.outstanding.len(), + "native decode teardown: presenter still holds frames past the \ + budget — destroying the pools anyway" + ); + break; + } + // `self` holds a Sender, so the channel can't disconnect — only time out. + match self + .release_rx + .recv_timeout((deadline - now).min(Duration::from_millis(50))) + { + Ok(token) => { + note_token(&mut self.outstanding, token); + } + Err(_) => continue, + } + } + // `self.dec` drops after this body: it drains its own decode-side GPU work. + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A shipped-ledger entry with inert handles — the bookkeeping under test is pure. + fn shipped(seq: u64, generation: u64) -> Shipped { + Shipped { + seq, + frame: DecodedVkFrame { + image: vk::Image::null(), + view: vk::ImageView::null(), + plane_views: [vk::ImageView::null(); 2], + layer: 0, + layout: vk::ImageLayout::VIDEO_DECODE_DST_KHR, + coded_width: 1920, + coded_height: 1088, + crop: pf_bitstream_crop(1920, 1080), + semaphore: vk::Semaphore::null(), + value: 0, + poc: 0, + is_idr: false, + query_slot: 0, + generation, + }, + released: false, + resolved: false, + polls_after_release: 0, + } + } + + fn pf_bitstream_crop(width: u32, height: u32) -> pf_vkdecode::DisplayCrop { + pf_vkdecode::DisplayCrop { + x: 0, + y: 0, + width, + height, + } + } + + #[test] + fn release_tokens_mark_their_frame_and_tolerate_strays() { + let mut outstanding = vec![shipped(0, 1), shipped(1, 1)]; + assert!(note_token( + &mut outstanding, + NativeReleaseToken { + seq: 1, + generation: 1 + } + )); + assert!(!outstanding[0].released); + assert!(outstanding[1].released); + // A stray token (frame already settled away — e.g. a post-demotion drain) + // matches nothing and must not panic or mis-mark. + assert!(!note_token( + &mut outstanding, + NativeReleaseToken { + seq: 7, + generation: 1 + } + )); + assert!(!outstanding[0].released); + } + + #[test] + fn the_guard_sends_its_token_exactly_once_on_drop() { + let (tx, rx) = mpsc::channel(); + let token = NativeReleaseToken { + seq: 42, + generation: 3, + }; + let guard = NativeReleaseGuard::new(tx, token); + assert!( + rx.try_recv().is_err(), + "nothing is sent while the frame lives" + ); + drop(guard); + assert_eq!(rx.try_recv().ok(), Some(token), "drop sends the token"); + assert!(rx.try_recv().is_err(), "exactly once"); + } + + #[test] + fn a_dropped_unpresented_frame_still_releases_through_the_same_guard() { + // The newest-wins channel/store displacement path: the frame never reaches a + // present, but dropping it must still return its slot. + let (tx, rx) = mpsc::channel(); + let frame = NativeVkFrame { + image: 0, + plane_views: [0; 2], + layer: 0, + layout: NativeVkLayout::DecodeDst, + semaphore: 0, + semaphore_value: 0, + generation: 5, + width: 1920, + height: 1080, + coded_width: 1920, + coded_height: 1088, + crop_x: 0, + crop_y: 0, + color: ColorDesc { + primaries: 2, + transfer: 2, + matrix: 2, + full_range: false, + }, + keyframe: true, + poc: 0, + guard: NativeReleaseGuard::new( + tx, + NativeReleaseToken { + seq: 9, + generation: 5, + }, + ), + }; + drop(frame); + assert_eq!( + rx.try_recv().ok(), + Some(NativeReleaseToken { + seq: 9, + generation: 5 + }) + ); + } + + #[test] + fn a_dead_channel_is_ignored_not_fatal() { + // Demotion mid-stream: the backend (and its Receiver) are gone while the + // presenter still holds a frame — its drop must be a no-op, not a panic. + let (tx, rx) = mpsc::channel(); + drop(rx); + let guard = NativeReleaseGuard::new( + tx, + NativeReleaseToken { + seq: 1, + generation: 1, + }, + ); + drop(guard); // must not panic + } + + #[test] + fn the_queue_lock_is_shared_only_when_the_families_collide() { + // Same family ⇒ same VkQueue (both sides use index 0) ⇒ shared lock. + assert!(submit_queues_collide(0, 0)); + assert!(submit_queues_collide(2, 2)); + // A separate decode family has exactly one submitter — no lock. + assert!(!submit_queues_collide(0, 3)); + } +} diff --git a/crates/pf-presenter/src/run.rs b/crates/pf-presenter/src/run.rs index b91127ab..a587fda2 100644 --- a/crates/pf-presenter/src/run.rs +++ b/crates/pf-presenter/src/run.rs @@ -1767,6 +1767,41 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result } } DecodedImage::VkFrame(_) => false, // demoted — drain until rebuild + // Native (pf-vkdecode) frames: decoded on the presenter's own + // device, same present shape and the same failure-streak demotion + // contract as the VkFrame arm. A drained/demoted frame drops here + // — its guard still returns the decoder's slot. + DecodedImage::NativeVk(v) if !st.dmabuf_demoted => { + st.hdr = v.color.is_pq(); + st.hdr_untonemapped = false; + match presenter.present( + &window, + FrameInput::NativeVk(v), + overlay_frame.as_ref(), + ) { + Ok(p) => { + st.hw_fails = 0; + p + } + Err(e) => { + // Lost device ⇒ unrecoverable, never demote ([`device_lost`]). + if device_lost(&e) { + return Err(e) + .context("GPU device lost — the session cannot continue"); + } + st.hw_fails += 1; + tracing::warn!(error = %format!("{e:#}"), fails = st.hw_fails, + "native vulkan present failed"); + if st.hw_fails >= 3 { + st.dmabuf_demoted = true; + tracing::warn!("demoting the decoder to software"); + st.force_software.store(true, Ordering::Relaxed); + } + false + } + } + } + DecodedImage::NativeVk(_) => false, // demoted — drain until rebuild }; if did_present { presented_video = true; diff --git a/crates/pf-presenter/src/vk/gpu.rs b/crates/pf-presenter/src/vk/gpu.rs index d371eb2e..744927c8 100644 --- a/crates/pf-presenter/src/vk/gpu.rs +++ b/crates/pf-presenter/src/vk/gpu.rs @@ -132,6 +132,56 @@ pub(super) fn vkframe_acquire_barrier( } } +/// Layout round-trip for one LAYER of a native (pf-vkdecode) decode image: decode +/// layout → SHADER_READ_ONLY before the CSC pass, and back after it. Layer-scoped — +/// the pool is an image array and the other layers are live DPB state that must not +/// be touched. No queue-family transfer: the pool is created CONCURRENT across the +/// graphics+decode families. +/// +/// Both scopes are FRAGMENT_SHADER for the same dependency-chain reason as +/// [`vkframe_acquire_barrier`]: the submit waits the frame's decode-complete timeline +/// with `wait_dst_stage_mask = FRAGMENT_SHADER`, and only a barrier whose first sync +/// scope intersects that mask chains with the wait — with TOP_OF_PIPE the transition +/// could execute while the decode queue still writes (the RADV green-block class). +pub(super) fn native_layer_barrier( + device: &ash::Device, + cmd: vk::CommandBuffer, + image: vk::Image, + layer: u32, + from: vk::ImageLayout, + to: vk::ImageLayout, +) { + let b = vk::ImageMemoryBarrier::default() + .src_access_mask(vk::AccessFlags::empty()) + .dst_access_mask(vk::AccessFlags::SHADER_READ) + .old_layout(from) + .new_layout(to) + .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) + .image(image) + .subresource_range( + vk::ImageSubresourceRange::default() + .aspect_mask(vk::ImageAspectFlags::COLOR) + .level_count(1) + .base_array_layer(layer) + .layer_count(1), + ); + // SAFETY: per the Vulkan contract above - recorded into a command buffer this code owns and + // has begun, referencing handles it also owns; nothing is submitted until the recording is + // ended. + unsafe { + device.cmd_pipeline_barrier( + cmd, + vk::PipelineStageFlags::FRAGMENT_SHADER, + vk::PipelineStageFlags::FRAGMENT_SHADER, + vk::DependencyFlags::empty(), + &[], + &[], + &[b], + ); + } +} + /// Acquire an imported D3D11 texture from the EXTERNAL queue family as a copy source. /// The keyed mutex on the submit is the actual cross-API ordering; per the /// external-memory rules an UNDEFINED-old-layout transition on externally-bound memory diff --git a/crates/pf-presenter/src/vk/mod.rs b/crates/pf-presenter/src/vk/mod.rs index 31379352..3d0f53f3 100644 --- a/crates/pf-presenter/src/vk/mod.rs +++ b/crates/pf-presenter/src/vk/mod.rs @@ -23,7 +23,7 @@ use crate::overlay::SharedDevice; use ash::vk; #[cfg(target_os = "linux")] use pf_client_core::video::DmabufFrame; -use pf_client_core::video::{CpuFrame, VkVideoFrame}; +use pf_client_core::video::{CpuFrame, NativeVkFrame, VkVideoFrame}; mod gpu; mod overlay_pipe; @@ -51,6 +51,12 @@ pub enum FrameInput<'a> { /// fence-complete, GENERAL layout (`pf_client_core::video_pyrowave`). #[cfg(all(any(target_os = "linux", windows), feature = "pyrowave"))] PyroWave(pf_client_core::video_pyrowave::PyroWavePlanarFrame), + /// Native Vulkan Video output (pf-vkdecode) — an NV12 image + plane views already + /// on THIS device: wait the frame's timeline pair on the submit, transition its + /// layer for sampling and BACK to its decode layout, CSC with the coded-vs-display + /// UV scale. Dropping the frame (after the sampling fence) releases the decoder's + /// slot via its guard. + NativeVk(NativeVkFrame), } /// The dmabuf/CSC machinery, present only when the device carries the import extensions. @@ -78,6 +84,10 @@ enum Retired { frame: VkVideoFrame, views: [vk::ImageView; 2], }, + /// A native (pf-vkdecode) frame: image + views are the DECODER's — nothing to + /// destroy here; dropping the frame after the fence wait sends its release token, + /// which is what returns the decode slot (the release-after-fence contract). + NativeVk(NativeVkFrame), } /// The overlay composite: one premultiplied-alpha quad blended over the swapchain image diff --git a/crates/pf-presenter/src/vk/present.rs b/crates/pf-presenter/src/vk/present.rs index a8b5bd12..b0ff98f2 100644 --- a/crates/pf-presenter/src/vk/present.rs +++ b/crates/pf-presenter/src/vk/present.rs @@ -9,7 +9,7 @@ use crate::overlay::OverlayFrame; use anyhow::{bail, Context as _, Result}; use ash::vk; use ash::vk::Handle as _; -use pf_client_core::video::VkVideoFrame; +use pf_client_core::video::{NativeVkFrame, NativeVkLayout, VkVideoFrame}; impl Presenter { /// Present one frame: route `input` into the video image (staging upload or dmabuf @@ -68,6 +68,7 @@ impl Presenter { FrameInput::D3d11(d) => Some(d.color.is_pq()), #[cfg(all(any(target_os = "linux", windows), feature = "pyrowave"))] FrameInput::PyroWave(f) => Some(f.color.is_pq()), + FrameInput::NativeVk(f) => Some(f.color.is_pq()), }; if let Some(pq) = frame_pq { // A PQ stream we can only tone-map (no HDR10 surface) is the silent failure behind @@ -96,6 +97,7 @@ impl Presenter { #[cfg(windows)] let mut win_frame: Option = None; let mut vk_frame: Option<(VkVideoFrame, [vk::ImageView; 2])> = None; + let mut native_frame: Option = None; #[cfg(all(any(target_os = "linux", windows), feature = "pyrowave"))] let mut pyro_frame: Option = None; let cpu_frame = match input { @@ -129,6 +131,12 @@ impl Presenter { pyro_frame = Some(f); None } + // Same device, and the decoder already made the per-plane views — no + // import, no view creation, nothing that can fail out here. + FrameInput::NativeVk(f) => { + native_frame = Some(f); + None + } }; // One frame in flight: the fence covers the command buffer, the staging buffer @@ -185,6 +193,38 @@ impl Presenter { } self.csc.bind_planes(&self.device, views[0], views[1]); } + if let Some(f) = &native_frame { + if self + .video + .as_ref() + .is_none_or(|v| v.width != f.width || v.height != f.height) + { + self.rebuild_video_image(f.width, f.height)?; + tracing::info!(width = f.width, height = f.height, "video image (re)built"); + } + // The UV-scale crop below assumes an origin crop (punktfunk hosts emit + // nothing else); a nonzero origin would display the wrong window — say so + // rather than be silently wrong. + if f.crop_x != 0 || f.crop_y != 0 { + use std::sync::atomic::{AtomicBool, Ordering}; + static WARNED: AtomicBool = AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + tracing::warn!( + crop_x = f.crop_x, + crop_y = f.crop_y, + "native frame carries a non-origin conformance crop — the UV \ + scale only handles origin crops; picture offset expected" + ); + } + } + // Decoder-owned plane views (R8 + R8G8), fence-wait above makes the set + // rebindable — same contract as the AVVkFrame arm. + self.csc.bind_planes( + &self.device, + vk::ImageView::from_raw(f.plane_views[0]), + vk::ImageView::from_raw(f.plane_views[1]), + ); + } #[cfg(all(any(target_os = "linux", windows), feature = "pyrowave"))] if let Some(f) = &pyro_frame { if self @@ -363,6 +403,60 @@ impl Presenter { vk_sync = Some(sync); } + // Native (pf-vkdecode) frame: same-device sampling like the AVVkFrame path, + // but the sync facts ride the frame itself (no frames lock — the decoder + // stamped layout/semaphore/value at delivery and nothing mutates them). + // Transition the picture's LAYER for sampling, run the same CSC pass with + // the coded-vs-display UV scale (the 1088-row lesson), then transition BACK + // to the decode layout: a coincide-mode picture is a live DPB slot the next + // decode must find in VIDEO_DECODE_DPB_KHR (distinct-mode DST images get the + // same round-trip — the decoder's reuse barrier discards via UNDEFINED, so + // the restore costs nothing and keeps one rule). The pool images are created + // CONCURRENT across the graphics+decode families, so these are plain layout + // transitions — no queue-family ownership transfer. + let mut native_wait: Option<(vk::Semaphore, u64)> = None; + if let (Some(f), Some(v)) = (&native_frame, &self.video) { + let image = vk::Image::from_raw(f.image); + let decode_layout = match f.layout { + NativeVkLayout::DecodeDst => vk::ImageLayout::VIDEO_DECODE_DST_KHR, + NativeVkLayout::DecodeDpb => vk::ImageLayout::VIDEO_DECODE_DPB_KHR, + }; + native_layer_barrier( + &self.device, + self.cmd_buf, + image, + f.layer, + decode_layout, + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + ); + let extent = vk::Extent2D { + width: v.width, + height: v.height, + }; + // The native path is the 8-bit H.264 envelope (NV12) — depth 8, no + // MSB packing; colour rides the frame (BT.709-limited SDR default). + self.record_csc( + v.framebuffer, + extent, + [ + f.width as f32 / f.coded_width as f32, + f.height as f32 / f.coded_height as f32, + ], + f.color, + 8, + false, + ); + native_layer_barrier( + &self.device, + self.cmd_buf, + image, + f.layer, + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + decode_layout, + ); + native_wait = Some((vk::Semaphore::from_raw(f.semaphore), f.semaphore_value)); + } + // PyroWave frame: the planes are already on THIS device, decode // fence-complete and barriered to fragment sampling (GENERAL) by the // decoder — no acquire needed, just the planar CSC pass. @@ -542,6 +636,18 @@ impl Presenter { signal_sems.push(sem); signal_values.push(sync.sem_value + 1); } + // The native frame's decode-complete timeline: wait it at FRAGMENT_SHADER + // (chaining with the acquire barrier — the same dependency-chain rule as + // `vkframe_acquire_barrier`). Deliberately NO signal back on the decoder's + // timeline: its per-slot values are the DECODER's counter (a foreign signal + // would collide with its next decode's value) — the slot-return contract is + // the release token the frame's guard sends once our fence proves the reads + // done. + if let Some((sem, value)) = &native_wait { + wait_sems.push(*sem); + wait_stages.push(vk::PipelineStageFlags::FRAGMENT_SHADER); + wait_values.push(*value); + } let mut timeline = vk::TimelineSemaphoreSubmitInfo::default() .wait_semaphore_values(&wait_values) .signal_semaphore_values(&signal_values); @@ -550,7 +656,7 @@ impl Presenter { .wait_dst_stage_mask(&wait_stages) .command_buffers(&cmd_bufs) .signal_semaphores(&signal_sems); - if vk_sync.is_some() { + if vk_sync.is_some() || native_wait.is_some() { submit = submit.push_next(&mut timeline); } // D3D11 frame: bracket the submit in the shared texture's keyed mutex, key 0 @@ -615,6 +721,11 @@ impl Presenter { if let Some(f) = win_frame.take() { self.retired_hw = Some(Retired::D3d11(f)); } + // Native frame: parked until the fence proves the sampling reads done — its + // drop THEN sends the decoder's release token (never at record time). + if let Some(f) = native_frame.take() { + self.retired_hw = Some(Retired::NativeVk(f)); + } let swapchains = [self.swapchain]; let indices = [index]; diff --git a/crates/pf-presenter/src/vk/resources.rs b/crates/pf-presenter/src/vk/resources.rs index 24f14e0f..4627e137 100644 --- a/crates/pf-presenter/src/vk/resources.rs +++ b/crates/pf-presenter/src/vk/resources.rs @@ -24,6 +24,10 @@ impl Retired { } drop(frame); // guard drops here — AVFrame (and the VkImage) released } + // The image and plane views belong to the DECODER's pools — nothing of ours + // to destroy. The drop sends the release token (the caller reaches here only + // after the sampling fence, so the token honestly means "GPU reads done"). + Retired::NativeVk(frame) => drop(frame), } } } diff --git a/crates/pf-vkdecode/src/decoder.rs b/crates/pf-vkdecode/src/decoder.rs index 663015dd..5c46a292 100644 --- a/crates/pf-vkdecode/src/decoder.rs +++ b/crates/pf-vkdecode/src/decoder.rs @@ -29,6 +29,7 @@ use pf_bitstream::h264::DpbUpdate; use pf_bitstream::h264::H264Planner; use pf_bitstream::h264::PicId; use pf_bitstream::h264::PlanError; +use pf_bitstream::h264::PlanWarning; use tracing::debug; use tracing::trace; @@ -424,6 +425,10 @@ pub struct VkH264Decoder { /// ones are detectable ([`DecodedVkFrame::generation`]). generation: u64, device_lost: bool, + /// The last `decode` call's plan warnings (concealment signals), held for + /// [`Self::take_warnings`] — the integration layer's recovery hook. Cleared at + /// every `decode` entry so a warning is never attributed to the wrong AU. + last_warnings: Vec, } impl VkH264Decoder { @@ -451,6 +456,7 @@ impl VkH264Decoder { ready: VecDeque::new(), generation: 0, device_lost: false, + last_warnings: Vec::new(), }) } @@ -460,6 +466,7 @@ impl VkH264Decoder { /// Never panics. `VkDecodeError::DeviceLost` latches: every later call fails /// fast until the owner rebuilds the decoder on fresh handles. pub fn decode(&mut self, au: &[u8]) -> Result, VkDecodeError> { + self.last_warnings.clear(); if self.device_lost { return Err(VkDecodeError::DeviceLost); } @@ -470,12 +477,31 @@ impl VkH264Decoder { result } + /// The plan warnings (concealment signals) of the most recent [`Self::decode`] + /// call, taken. Non-empty means the AU was planned around missing/damaged + /// references — the picture decodes but its content is concealed, and the caller + /// should request a re-anchor (the recovery wiring the module doc reserves for + /// the integration layer). + pub fn take_warnings(&mut self) -> Vec { + std::mem::take(&mut self.last_warnings) + } + + /// The current session generation ([`DecodedVkFrame::generation`]'s counterpart): + /// lets a caller holding delivered frames tell a STALE frame (its session was + /// rebuilt — every decoder entry point would report it so) apart from a live one, + /// without tripping the conservative `Failed` a stale status poll returns. + pub fn generation(&self) -> u64 { + self.generation + } + fn decode_inner(&mut self, au: &[u8]) -> Result, VkDecodeError> { let plan = self.planner.plan_au(au)?; for warning in &plan.warnings { - // Concealment wiring (want_keyframe) is WP-C's; never silent though. + // The recovery verdict is the integration layer's ([`Self::take_warnings`]); + // never silent here though. trace!(?warning, "plan warning"); } + self.last_warnings = plan.warnings.clone(); self.ensure_state(&plan)?; let sps_id = plan.sps.seq_parameter_set_id; diff --git a/crates/pf-vkdecode/src/lib.rs b/crates/pf-vkdecode/src/lib.rs index 92944a2b..fdd38b98 100644 --- a/crates/pf-vkdecode/src/lib.rs +++ b/crates/pf-vkdecode/src/lib.rs @@ -53,6 +53,17 @@ pub mod ring; pub mod session; pub mod slots; +/// Re-exported for the integration layer (WP-C): [`DecodedVkFrame`]'s handle fields are +/// ash types, and the consumer (pf-client-core, whose own `ash` is optional/feature-gated) +/// flattens them to raw `u64`s through `ash::vk::Handle` — via THIS instance of ash, so +/// the versions can never skew. +pub use ash; +/// Re-exported so [`VkH264Decoder::take_warnings`] consumers name the warning type — +/// and [`DecodedVkFrame::crop`]'s type — without growing a pf-bitstream dependency of +/// their own. +pub use pf_bitstream::h264::DisplayCrop; +pub use pf_bitstream::h264::PlanWarning; + pub use caps::derive_caps; pub use caps::CapsError; pub use caps::DecodeCaps;