From 575975687ca2c6a977800e7c505cc43e3a269e89 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Wed, 15 Jul 2026 01:33:38 +0200 Subject: [PATCH] feat(client): PyroWave decode backend on the presenter's device (Phase 2b, part 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The presenter's device creation now probes + enables the PyroWave compute feature set alongside the Vulkan Video probe (shaderInt16, storageBuffer8BitAccess, subgroup size control — gated on support, harmless when unused) and exports the facts through VulkanDecodeDevice (pyrowave_decode capability + feature bools + apiVersion + the queue- family shape). pf-client-core (feature `pyrowave`, Linux): video_pyrowave.rs — the decoder runs pyrowave compute on the PRESENTER's own VkDevice, zero interop (plan §4.5): pinned content-equivalent create-info reconstruction satisfies pyrowave 0.4.0's lifetime rule without refactoring the presenter's creation; queue access rides the existing device-wide QueueLock (the FFmpeg/Skia contract); decode records into our command buffer, fence-synchronous (sub-ms), into a 4-deep ring of 3xR8 plane sets (decode REQUIRES storage usage + identity swizzles, so the encoder's RG8 trick doesn't apply). Backend::PyroWave + DecodedImage::PyroWave + Decoder::new_pyrowave + decodable_codecs_for (advertisement gated on the device probe) wired through the decode dispatch; no demote ladder (nothing else decodes it — fallback is session renegotiation, plan §4.6). Still to come for a live session: the presenter's planar-CSC render path for the new variant, pump/shell opt-in (preferred_codec) wiring, and the on-glass .21 run. Validated on .21: pf-client-core + pf-presenter compile with and without the feature, clippy clean, 26 client-core tests green. Co-Authored-By: Claude Fable 5 --- crates/pf-client-core/Cargo.toml | 10 + crates/pf-client-core/src/lib.rs | 4 + crates/pf-client-core/src/session.rs | 4 + crates/pf-client-core/src/video.rs | 74 +++ crates/pf-client-core/src/video_pyrowave.rs | 561 ++++++++++++++++++++ crates/pf-presenter/src/vk.rs | 35 +- 6 files changed, 684 insertions(+), 4 deletions(-) create mode 100644 crates/pf-client-core/src/video_pyrowave.rs diff --git a/crates/pf-client-core/Cargo.toml b/crates/pf-client-core/Cargo.toml index f452ad53..62c4b56d 100644 --- a/crates/pf-client-core/Cargo.toml +++ b/crates/pf-client-core/Cargo.toml @@ -40,6 +40,11 @@ tracing = "0.1" [target.'cfg(target_os = "linux")'.dependencies] pipewire = "0.9" sdl3 = { version = "0.18", features = ["hidapi"] } +# PyroWave decode (the opt-in wired-LAN wavelet codec, design/pyrowave-codec-plan.md +# §4.5) — pure Vulkan compute on the presenter's shared device. `ash` only wraps the +# presenter's existing raw handles (same pinned version as pf-presenter). +pyrowave-sys = { path = "../pyrowave-sys", optional = true } +ash = { version = "0.38", optional = true } [target.'cfg(windows)'.dependencies] wasapi = "0.23" @@ -57,3 +62,8 @@ windows = { git = "https://github.com/microsoft/windows-rs", rev = "a4f7b2cb7c63 # method itself is feature-gated behind this. "Win32_Security", ] } + +[features] +# PyroWave client decode — OFF by default (the flatpak/default builds stay unchanged); +# the Linux session client turns it on together with the host-side feature. +pyrowave = ["dep:pyrowave-sys", "dep:ash"] diff --git a/crates/pf-client-core/src/lib.rs b/crates/pf-client-core/src/lib.rs index 292eb16c..f406bc8e 100644 --- a/crates/pf-client-core/src/lib.rs +++ b/crates/pf-client-core/src/lib.rs @@ -33,7 +33,11 @@ pub mod session; pub mod trust; #[cfg(any(target_os = "linux", windows))] pub mod video; +// PyroWave decode — Linux + `pyrowave` feature only (plan §4.5; the Windows client's +// present-path decision and the Apple Metal port are their own phases). #[cfg(windows)] pub mod video_d3d11; +#[cfg(all(target_os = "linux", feature = "pyrowave"))] +pub mod video_pyrowave; pub mod wol; diff --git a/crates/pf-client-core/src/session.rs b/crates/pf-client-core/src/session.rs index ceee049b..eb371b13 100644 --- a/crates/pf-client-core/src/session.rs +++ b/crates/pf-client-core/src/session.rs @@ -413,6 +413,8 @@ fn pump( DecodedImage::VkFrame(_) => "vulkan", #[cfg(windows)] DecodedImage::D3d11(_) => "d3d11va", + #[cfg(all(target_os = "linux", feature = "pyrowave"))] + DecodedImage::PyroWave(_) => "pyrowave", }; if total_frames == 1 { let (w, h, path) = match &image { @@ -422,6 +424,8 @@ fn pump( DecodedImage::VkFrame(v) => (v.width, v.height, "vulkan-video"), #[cfg(windows)] DecodedImage::D3d11(d) => (d.width, d.height, "d3d11va"), + #[cfg(all(target_os = "linux", feature = "pyrowave"))] + DecodedImage::PyroWave(f) => (f.width, f.height, "pyrowave"), }; tracing::info!(width = w, height = h, path, "first frame decoded"); } diff --git a/crates/pf-client-core/src/video.rs b/crates/pf-client-core/src/video.rs index 23a2d738..05907148 100644 --- a/crates/pf-client-core/src/video.rs +++ b/crates/pf-client-core/src/video.rs @@ -68,6 +68,11 @@ pub enum DecodedImage { /// (Intel's Windows driver foremost). See `crate::video_d3d11`. #[cfg(windows)] D3d11(crate::video_d3d11::D3d11Frame), + /// PyroWave planar output: three R8 plane views on the presenter's own device, + /// decode already fence-complete, GENERAL layout — the presenter's planar CSC + /// samples them directly (BT.709 limited, the codec's fixed colour contract). + #[cfg(all(target_os = "linux", feature = "pyrowave"))] + PyroWave(crate::video_pyrowave::PyroWavePlanarFrame), } /// One Vulkan-decoded frame. The image lives on the presenter's own VkDevice (the @@ -183,6 +188,8 @@ impl DecodedImage { DecodedImage::VkFrame(f) => f.keyframe, #[cfg(windows)] DecodedImage::D3d11(f) => f.keyframe, + #[cfg(all(target_os = "linux", feature = "pyrowave"))] + DecodedImage::PyroWave(f) => f.keyframe, } } @@ -197,6 +204,8 @@ impl DecodedImage { DecodedImage::VkFrame(f) => (f.width, f.height), #[cfg(windows)] DecodedImage::D3d11(f) => (f.width, f.height), + #[cfg(all(target_os = "linux", feature = "pyrowave"))] + DecodedImage::PyroWave(f) => (f.width, f.height), } } } @@ -312,6 +321,10 @@ enum Backend { Vaapi(VaapiDecoder), #[cfg(windows)] D3d11va(crate::video_d3d11::D3d11vaDecoder), + /// PyroWave (wired-LAN wavelet codec): pyrowave compute on the presenter's device, + /// no FFmpeg involvement. No demotion rung — there is no other decoder for it. + #[cfg(all(target_os = "linux", feature = "pyrowave"))] + PyroWave(crate::video_pyrowave::PyroWaveDecoder), Software(SoftwareDecoder), } @@ -360,6 +373,21 @@ pub fn decodable_codecs() -> u8 { bits } +/// [`decodable_codecs`] plus the PyroWave bit when the presenter's device passed the +/// compute-feature probe. Advertisement-only: `resolve_codec` never auto-picks PyroWave — +/// the session must also name it `preferred_codec` (plan §3), which the client does only +/// under its explicit opt-in. +pub fn decodable_codecs_for(vk: Option<&VulkanDecodeDevice>) -> u8 { + let bits = decodable_codecs(); + #[cfg(all(target_os = "linux", feature = "pyrowave"))] + if vk.map(|v| v.pyrowave_decode).unwrap_or(false) { + return bits | punktfunk_core::quic::CODEC_PYROWAVE; + } + #[cfg(not(all(target_os = "linux", feature = "pyrowave")))] + let _ = vk; + bits +} + /// libavcodec logs reference-frame recovery to the process stderr very verbosely /// (`First slice in a frame missing`, `Could not find ref with POC …`, `Error /// constructing the frame RPS`) — normal chatter while the decoder waits for a keyframe @@ -539,6 +567,21 @@ impl Decoder { /// Drain the "please ask the host for an IDR" flag — the pump calls this each iteration /// (throttled) so a demoted/erroring decoder can resynchronize under the infinite GOP. + /// Open a PyroWave decoder for a `CODEC_PYROWAVE` session (plan §4.5): pyrowave + /// compute on the presenter's device, no FFmpeg. `codec_id` is irrelevant (kept as + /// HEVC so an — impossible — demotion path stays well-formed). + #[cfg(all(target_os = "linux", feature = "pyrowave"))] + pub fn new_pyrowave(vk: &VulkanDecodeDevice, width: u32, height: u32) -> Result { + Ok(Decoder { + backend: Backend::PyroWave(crate::video_pyrowave::PyroWaveDecoder::new( + vk, width, height, + )?), + codec_id: ffmpeg::codec::Id::HEVC, + vaapi_fails: 0, + want_keyframe: false, + }) + } + pub fn take_keyframe_request(&mut self) -> bool { std::mem::take(&mut self.want_keyframe) } @@ -572,6 +615,11 @@ impl Decoder { Backend::Vaapi(v) => v.decode(au).map(|f| f.map(DecodedImage::Dmabuf)), #[cfg(windows)] Backend::D3d11va(d) => d.decode(au).map(|f| f.map(DecodedImage::D3d11)), + // No demote ladder below PyroWave (nothing else decodes it): propagate the + // error; the pump surfaces it and the session falls back to HEVC by + // renegotiation (plan §4.6), not by decoder swap. + #[cfg(all(target_os = "linux", feature = "pyrowave"))] + Backend::PyroWave(p) => return Ok(p.decode(au)?.map(DecodedImage::PyroWave)), Backend::Software(s) => return Ok(s.decode(au)?.map(DecodedImage::Cpu)), }; match result { @@ -1077,6 +1125,24 @@ pub struct VulkanDecodeDevice { /// features). The bundle now exists even without it — Windows D3D11 interop rides the /// same struct — so consumers gate the FFmpeg-Vulkan decoder on THIS, not on `Some`. pub video_decode: bool, + /// PyroWave decode (the wired-LAN wavelet codec) is usable: Vulkan 1.3 + the compute + /// features its kernels need were present AND enabled at device creation + /// (`shaderInt16`, `storageBuffer8BitAccess`, subgroup size control). Gates the + /// `CODEC_PYROWAVE` advertisement and the pyrowave decoder backend. + pub pyrowave_decode: bool, + /// The feature facts + creation shape the pyrowave decoder's pinned create-info + /// reconstruction mirrors (pyrowave 0.4.0 requires the instance/device create infos — + /// content-accurate, kept alive — to share our VkDevice). + pub f_shader_int16: bool, + pub f_storage_buffer8: bool, + pub f_subgroup_size_control: bool, + pub f_compute_full_subgroups: bool, + pub f_shader_float16: bool, + /// `VkPhysicalDeviceProperties::apiVersion` of the presenter's device. + pub api_version: u32, + /// The queue families the device was created with (one `VkDeviceQueueCreateInfo` each, + /// one queue per family, priority 1.0) — mirrored by the reconstruction. + pub queue_families: Vec, /// The presenter enabled `VK_KHR_external_memory_win32` + `VK_KHR_win32_keyed_mutex`: /// D3D11 shared-texture frames can reach the screen. Always `false` off Windows. pub d3d11_import: bool, @@ -1598,6 +1664,14 @@ mod tests { f_sampler_ycbcr: true, f_timeline_semaphore: true, f_synchronization2: true, + f_shader_int16: false, + f_storage_buffer8: false, + f_subgroup_size_control: false, + f_compute_full_subgroups: false, + f_shader_float16: false, + api_version: 0, + queue_families: Vec::new(), + pyrowave_decode: false, video_decode: true, d3d11_import: false, adapter_luid: None, diff --git a/crates/pf-client-core/src/video_pyrowave.rs b/crates/pf-client-core/src/video_pyrowave.rs new file mode 100644 index 00000000..2ebf6ff9 --- /dev/null +++ b/crates/pf-client-core/src/video_pyrowave.rs @@ -0,0 +1,561 @@ +//! PyroWave client decode (design/pyrowave-codec-plan.md §4.5) — the wired-LAN wavelet +//! codec's decoder, running as plain Vulkan compute on the PRESENTER's own VkDevice (the +//! whole point: decode + CSC + present on one device, zero interop). Bypasses FFmpeg +//! entirely: the AU is one self-delimiting pyrowave packet; `push_packet` → ready → +//! `decode_gpu_buffer` recorded into OUR command buffer, submitted on the shared graphics +//! queue under the device's [`QueueLock`], fence-waited (sub-ms — Phase-0 measured +//! 0.067 ms GPU at 1080p on the RTX 5070 Ti). +//! +//! Output: three separate R8 planes (Y full-res, Cb/Cr half-res) — the decode path +//! requires STORAGE usage and IDENTITY/R swizzles, so the encoder's two-component +//! RG8 trick is not allowed here (pyrowave.h validation). The presenter samples them +//! with its planar CSC variant (BT.709 limited — the codec's fixed colour contract, +//! there is no VUI). A small ring of plane-sets keeps a decode from overwriting the set +//! the presenter is still sampling; the synchronous fence bounds decode-side reuse and +//! the ring depth covers present-side latency (≤ 1–2 frames in this pipeline). +//! +//! pyrowave 0.4.0 requires the instance/device create-infos to stay alive on the shared +//! device — the presenter doesn't pin its originals, so [`Hold`] reconstructs +//! content-equivalent ones from [`VulkanDecodeDevice`]'s exported extension lists, +//! feature facts and queue-family shape (pyrowave reads them for extension/feature +//! detection; pointer identity is not required). + +use crate::video::{ColorDesc, VulkanDecodeDevice}; +use anyhow::{bail, Context as _, Result}; +use ash::vk; +use ash::vk::Handle as _; +use pyrowave_sys as pw; +use std::ffi::{c_char, c_void, CString}; +use std::sync::Arc; + +/// Plane-set ring depth: decode writes slot N while the presenter may still sample +/// N-1/N-2 (its own submission raced ahead under the shared queue's FIFO order, so +/// same-queue execution ordering already serializes writes vs. reads per slot; the ring +/// keeps LOGICAL reuse far enough behind). +const RING: usize = 4; + +fn pw_check(r: pw::pyrowave_result, what: &str) -> Result<()> { + if r == pw::pyrowave_result_PYROWAVE_SUCCESS { + Ok(()) + } else { + bail!("pyrowave {what} failed: result {r}") + } +} + +/// Content-equivalent reconstruction of the presenter device's create-infos, pinned for +/// the lifetime of the `pyrowave_device` (heap boxes; moving `Hold` moves only pointers). +struct Hold { + _inst_ext_names: Vec, + _inst_ext_ptrs: Vec<*const c_char>, + _dev_ext_names: Vec, + _dev_ext_ptrs: Vec<*const c_char>, + _app_info: Box>, + instance_ci: Box>, + _queue_prio: Box<[f32; 1]>, + _queue_cis: Vec>, + _feat2: Box>, + _v11: Box>, + _v12: Box>, + _v13: Box>, + device_ci: Box>, +} + +impl Hold { + fn build(vkd: &VulkanDecodeDevice) -> Hold { + let inst_ext_names = vkd.instance_extensions.clone(); + let inst_ext_ptrs: Vec<*const c_char> = inst_ext_names.iter().map(|c| c.as_ptr()).collect(); + let dev_ext_names = vkd.device_extensions.clone(); + let dev_ext_ptrs: Vec<*const c_char> = dev_ext_names.iter().map(|c| c.as_ptr()).collect(); + + let mut app_info = + Box::new(vk::ApplicationInfo::default().api_version(vk::API_VERSION_1_3)); + let mut instance_ci = Box::new(vk::InstanceCreateInfo::default()); + instance_ci.p_application_info = &mut *app_info; + instance_ci.enabled_extension_count = inst_ext_ptrs.len() as u32; + instance_ci.pp_enabled_extension_names = if inst_ext_ptrs.is_empty() { + std::ptr::null() + } else { + inst_ext_ptrs.as_ptr() + }; + + let queue_prio = Box::new([1.0f32]); + let mut queue_cis: Vec> = vkd + .queue_families + .iter() + .map(|&fam| { + let mut ci = vk::DeviceQueueCreateInfo::default().queue_family_index(fam); + ci.queue_count = 1; + ci + }) + .collect(); + for ci in &mut queue_cis { + ci.p_queue_priorities = queue_prio.as_ptr(); + } + + // The feature facts the presenter enabled (VulkanDecodeDevice reports exactly + // what device creation turned on — pyrowave keys its paths off these). + let mut feat2 = Box::new(vk::PhysicalDeviceFeatures2::default()); + feat2.features.shader_int16 = vkd.f_shader_int16 as u32; + let mut v11 = Box::new( + vk::PhysicalDeviceVulkan11Features::default() + .sampler_ycbcr_conversion(vkd.f_sampler_ycbcr), + ); + let mut v12 = Box::new( + vk::PhysicalDeviceVulkan12Features::default() + .timeline_semaphore(vkd.f_timeline_semaphore) + .storage_buffer8_bit_access(vkd.f_storage_buffer8) + .shader_float16(vkd.f_shader_float16), + ); + let mut v13 = Box::new( + vk::PhysicalDeviceVulkan13Features::default() + .synchronization2(vkd.f_synchronization2) + .subgroup_size_control(vkd.f_subgroup_size_control) + .compute_full_subgroups(vkd.f_compute_full_subgroups), + ); + feat2.p_next = &mut *v11 as *mut _ as *mut c_void; + v11.p_next = &mut *v12 as *mut _ as *mut c_void; + v12.p_next = &mut *v13 as *mut _ as *mut c_void; + + let mut device_ci = Box::new(vk::DeviceCreateInfo::default()); + device_ci.p_next = &*feat2 as *const _ as *const c_void; + device_ci.queue_create_info_count = queue_cis.len() as u32; + device_ci.p_queue_create_infos = queue_cis.as_ptr(); + device_ci.enabled_extension_count = dev_ext_ptrs.len() as u32; + device_ci.pp_enabled_extension_names = dev_ext_ptrs.as_ptr(); + + Hold { + _inst_ext_names: inst_ext_names, + _inst_ext_ptrs: inst_ext_ptrs, + _dev_ext_names: dev_ext_names, + _dev_ext_ptrs: dev_ext_ptrs, + _app_info: app_info, + instance_ci, + _queue_prio: queue_prio, + _queue_cis: queue_cis, + _feat2: feat2, + _v11: v11, + _v12: v12, + _v13: v13, + device_ci, + } + } +} + +/// The queue-lock trampolines pyrowave calls around any internal queue use. `userdata` +/// is a raw pointer to the [`crate::video::QueueLock`] kept alive by the decoder's Arc. +unsafe extern "C" fn queue_lock_cb(ud: *mut c_void) { + // SAFETY: `ud` is the QueueLock the decoder's Arc pins; pyrowave only calls this + // while the decoder (and thus the Arc) lives. + unsafe { (*(ud as *const crate::video::QueueLock)).lock() } +} +unsafe extern "C" fn queue_unlock_cb(ud: *mut c_void) { + // SAFETY: as above. + unsafe { (*(ud as *const crate::video::QueueLock)).unlock() } +} + +/// One decoded PyroWave frame: three R8 plane images on the presenter's device, GENERAL +/// layout, decode-complete (the decoder fence-waits before handing it over). `slot` +/// identifies the ring entry; the images/views live as long as the decoder. +pub struct PyroWavePlanarFrame { + /// Raw `VkImageView`s (Y, Cb, Cr) for the presenter's planar CSC sampling. + pub views: [u64; 3], + pub width: u32, + pub height: u32, + pub color: ColorDesc, + /// Every PyroWave frame is independently decodable — always a clean re-anchor. + pub keyframe: bool, +} + +struct PlaneSet { + imgs: [vk::Image; 3], + mems: [vk::DeviceMemory; 3], + views: [vk::ImageView; 3], + /// First use transitions from UNDEFINED; afterwards GENERAL→GENERAL. + initialized: bool, +} + +pub struct PyroWaveDecoder { + // ash wrappers reconstructed over the presenter's raw handles (not owned — the + // presenter outlives the decoder; Drop destroys only what this struct created). + device: ash::Device, + queue: vk::Queue, + _hold: Box, + queue_lock: Arc, + pw_dev: pw::pyrowave_device, + pw_dec: pw::pyrowave_decoder, + ring: Vec, + next: usize, + cmd_pool: vk::CommandPool, + cmd: vk::CommandBuffer, + fence: vk::Fence, + width: u32, + height: u32, +} + +// SAFETY: used only from the single decode thread; the shared-queue accesses go through +// QueueLock, matching the FFmpeg-Vulkan backend's threading contract. +unsafe impl Send for PyroWaveDecoder {} + +impl PyroWaveDecoder { + pub fn new(vkd: &VulkanDecodeDevice, width: u32, height: u32) -> Result { + if !vkd.pyrowave_decode { + bail!("presenter device lacks the PyroWave compute feature set"); + } + if width % 2 != 0 || height % 2 != 0 { + bail!("pyrowave 4:2:0 needs even dimensions (got {width}x{height})"); + } + // SAFETY: the handles in `vkd` are the presenter's live instance/device (it + // outlives the decoder — same contract the FFmpeg Vulkan backend relies on); + // `Hold` pins the reconstructed create-infos for the pyrowave device's lifetime. + unsafe { Self::new_inner(vkd, width, height) } + } + + unsafe fn new_inner( + vkd: &VulkanDecodeDevice, + width: u32, + height: u32, + ) -> Result { + let static_fn = ash::StaticFn { + get_instance_proc_addr: std::mem::transmute::( + vkd.get_instance_proc_addr, + ), + }; + let instance_h = vk::Instance::from_raw(vkd.instance as u64); + let device_h = vk::Device::from_raw(vkd.device as u64); + let entry = ash::Entry::from_static_fn(static_fn.clone()); + let instance = ash::Instance::load(&static_fn, instance_h); + let device = ash::Device::load(instance.fp_v1_0(), device_h); + let queue = device.get_device_queue(vkd.graphics_qf, 0); + let _ = &entry; + + let hold = Box::new(Hold::build(vkd)); + let queue_lock = vkd.queue_lock.clone(); + let mut queue_info = pw::pyrowave_device_create_queue_info { + queue: queue.as_raw() as usize as pw::VkQueue, + familyIndex: vkd.graphics_qf, + index: 0, + }; + let create = pw::pyrowave_device_create_info { + // SAFETY(cast): re-labels the loader entry point between ash's and bindgen's + // identical C function-pointer types. + GetInstanceProcAddr: Some(std::mem::transmute::< + vk::PFN_vkGetInstanceProcAddr, + unsafe extern "C" fn(pw::VkInstance, *const c_char) -> pw::PFN_vkVoidFunction, + >(static_fn.get_instance_proc_addr)), + instance: vkd.instance as pw::VkInstance, + physical_device: vkd.physical_device as pw::VkPhysicalDevice, + device: vkd.device as pw::VkDevice, + instance_create_info: &*hold.instance_ci as *const vk::InstanceCreateInfo + as *const pw::VkInstanceCreateInfo, + device_create_info: &*hold.device_ci as *const vk::DeviceCreateInfo + as *const pw::VkDeviceCreateInfo, + queue_info: &mut queue_info, + queue_info_count: 1, + // The presenter/Skia/FFmpeg all serialize on this same lock. + queue_lock_callback: Some(queue_lock_cb), + queue_unlock_callback: Some(queue_unlock_cb), + userdata: Arc::as_ptr(&queue_lock) as *mut c_void, + }; + let mut pw_dev: pw::pyrowave_device = std::ptr::null_mut(); + pw_check( + pw::pyrowave_create_device(&create, &mut pw_dev), + "create_device (shared presenter device)", + )?; + let _ = + pw::pyrowave_device_set_queue_type(pw_dev, pw::VkQueueFlagBits_VK_QUEUE_COMPUTE_BIT); + + let dinfo = pw::pyrowave_decoder_create_info { + device: pw_dev, + width: width as i32, + height: height as i32, + chroma: pw::pyrowave_chroma_subsampling_PYROWAVE_CHROMA_SUBSAMPLING_420, + // The fragment-iDWT path is for Mali/Adreno-class mobile GPUs only. + fragment_path: false, + }; + let mut pw_dec: pw::pyrowave_decoder = std::ptr::null_mut(); + if let Err(e) = pw_check( + pw::pyrowave_decoder_create(&dinfo, &mut pw_dec), + "decoder_create", + ) { + pw::pyrowave_device_destroy(pw_dev); + return Err(e); + } + + // Plane-set ring: 3 × R8, storage (decode writes) + sampled (presenter CSC). + let mem_props = instance.get_physical_device_memory_properties( + vk::PhysicalDevice::from_raw(vkd.physical_device as u64), + ); + let make_plane = |w: u32, h: u32| -> Result<(vk::Image, vk::DeviceMemory, vk::ImageView)> { + let img = device.create_image( + &vk::ImageCreateInfo::default() + .image_type(vk::ImageType::TYPE_2D) + .format(vk::Format::R8_UNORM) + .extent(vk::Extent3D { + width: w, + height: h, + depth: 1, + }) + .mip_levels(1) + .array_layers(1) + .samples(vk::SampleCountFlags::TYPE_1) + .tiling(vk::ImageTiling::OPTIMAL) + .usage(vk::ImageUsageFlags::STORAGE | vk::ImageUsageFlags::SAMPLED) + .initial_layout(vk::ImageLayout::UNDEFINED), + None, + )?; + let req = device.get_image_memory_requirements(img); + let ti = (0..mem_props.memory_type_count) + .find(|&i| { + (req.memory_type_bits & (1 << i)) != 0 + && mem_props.memory_types[i as usize] + .property_flags + .contains(vk::MemoryPropertyFlags::DEVICE_LOCAL) + }) + .unwrap_or(0); + let mem = device.allocate_memory( + &vk::MemoryAllocateInfo::default() + .allocation_size(req.size) + .memory_type_index(ti), + None, + )?; + device.bind_image_memory(img, mem, 0)?; + let view = device.create_image_view( + &vk::ImageViewCreateInfo::default() + .image(img) + .view_type(vk::ImageViewType::TYPE_2D) + .format(vk::Format::R8_UNORM) + .subresource_range(vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_mip_level: 0, + level_count: 1, + base_array_layer: 0, + layer_count: 1, + }), + None, + )?; + Ok((img, mem, view)) + }; + let mut ring = Vec::with_capacity(RING); + for _ in 0..RING { + let (y, ym, yv) = make_plane(width, height)?; + let (cb, cbm, cbv) = make_plane(width / 2, height / 2)?; + let (cr, crm, crv) = make_plane(width / 2, height / 2)?; + ring.push(PlaneSet { + imgs: [y, cb, cr], + mems: [ym, cbm, crm], + views: [yv, cbv, crv], + initialized: false, + }); + } + + let cmd_pool = device.create_command_pool( + &vk::CommandPoolCreateInfo::default() + .queue_family_index(vkd.graphics_qf) + .flags(vk::CommandPoolCreateFlags::RESET_COMMAND_BUFFER), + None, + )?; + let cmd = device.allocate_command_buffers( + &vk::CommandBufferAllocateInfo::default() + .command_pool(cmd_pool) + .level(vk::CommandBufferLevel::PRIMARY) + .command_buffer_count(1), + )?[0]; + let fence = device.create_fence(&vk::FenceCreateInfo::default(), None)?; + + tracing::info!( + mode = %format!("{width}x{height}"), + "PyroWave decoder open on the presenter's device (compute iDWT, BT.709 limited)" + ); + Ok(PyroWaveDecoder { + device, + queue, + _hold: hold, + queue_lock, + pw_dev, + pw_dec, + ring, + next: 0, + cmd_pool, + cmd, + fence, + width, + height, + }) + } + + /// One AU in → one frame out (the AU is a complete pyrowave frame: one packet). + pub fn decode(&mut self, au: &[u8]) -> Result> { + // SAFETY: single decode thread; all handles owned/pinned by `self`; queue access + // serialized under the device-wide QueueLock; the fence bounds GPU completion + // before the frame is handed to the presenter. + unsafe { self.decode_inner(au) } + } + + unsafe fn decode_inner(&mut self, au: &[u8]) -> Result> { + pw_check( + pw::pyrowave_decoder_push_packet(self.pw_dec, au.as_ptr() as *const c_void, au.len()), + "push_packet", + )?; + // The reassembler delivers complete AUs only, so a frame is ready per push; a + // stale/duplicate packet (sequence rewind) simply isn't — skip, no error. + if !pw::pyrowave_decoder_decode_is_ready(self.pw_dec, false) { + return Ok(None); + } + + let slot = self.next; + self.next = (self.next + 1) % RING; + let dev = self.device.clone(); + dev.begin_command_buffer( + self.cmd, + &vk::CommandBufferBeginInfo::default() + .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), + )?; + let old_layout = if self.ring[slot].initialized { + vk::ImageLayout::GENERAL + } else { + vk::ImageLayout::UNDEFINED + }; + let range = vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_mip_level: 0, + level_count: 1, + base_array_layer: 0, + layer_count: 1, + }; + let to_write = |img| { + vk::ImageMemoryBarrier2::default() + // Order against the presenter's prior sampling of this slot (same queue). + .src_stage_mask(vk::PipelineStageFlags2::FRAGMENT_SHADER) + .src_access_mask(vk::AccessFlags2::NONE) + .dst_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER) + .dst_access_mask(vk::AccessFlags2::SHADER_STORAGE_WRITE) + .old_layout(old_layout) + .new_layout(vk::ImageLayout::GENERAL) + .image(img) + .subresource_range(range) + }; + let pre: Vec<_> = self.ring[slot].imgs.iter().map(|&i| to_write(i)).collect(); + dev.cmd_pipeline_barrier2( + self.cmd, + &vk::DependencyInfo::default().image_memory_barriers(&pre), + ); + + let plane = |img: vk::Image, w: u32, h: u32| pw::pyrowave_image_view { + image: img.as_raw() as usize as pw::VkImage, + width: w, + height: h, + image_format: pw::VkFormat_VK_FORMAT_R8_UNORM, + view_format: pw::VkFormat_VK_FORMAT_R8_UNORM, + mip_level: 0, + layer: 0, + aspect: pw::VkImageAspectFlagBits_VK_IMAGE_ASPECT_COLOR_BIT, + swizzle: pw::VkComponentSwizzle_VK_COMPONENT_SWIZZLE_IDENTITY, + layout: pw::VkImageLayout_VK_IMAGE_LAYOUT_GENERAL, + }; + let (w, h) = (self.width, self.height); + let buffers = pw::pyrowave_gpu_buffers { + planes: [ + plane(self.ring[slot].imgs[0], w, h), + plane(self.ring[slot].imgs[1], w / 2, h / 2), + plane(self.ring[slot].imgs[2], w / 2, h / 2), + ], + }; + pw::pyrowave_device_set_command_buffer( + self.pw_dev, + self.cmd.as_raw() as usize as pw::VkCommandBuffer, + ); + let dec_res = pw::pyrowave_decoder_decode_gpu_buffer( + self.pw_dec, + std::ptr::null(), + std::ptr::null(), + &buffers, + ); + pw::pyrowave_device_set_command_buffer(self.pw_dev, std::ptr::null_mut()); + pw_check(dec_res, "decode_gpu_buffer")?; + + // Decode's storage writes → the presenter's fragment sampling (layout stays + // GENERAL: that is what the planar CSC descriptors use for this path). + let to_read = |img| { + vk::ImageMemoryBarrier2::default() + .src_stage_mask(vk::PipelineStageFlags2::COMPUTE_SHADER) + .src_access_mask(vk::AccessFlags2::SHADER_STORAGE_WRITE) + .dst_stage_mask(vk::PipelineStageFlags2::FRAGMENT_SHADER) + .dst_access_mask(vk::AccessFlags2::SHADER_SAMPLED_READ) + .old_layout(vk::ImageLayout::GENERAL) + .new_layout(vk::ImageLayout::GENERAL) + .image(img) + .subresource_range(range) + }; + let post: Vec<_> = self.ring[slot].imgs.iter().map(|&i| to_read(i)).collect(); + dev.cmd_pipeline_barrier2( + self.cmd, + &vk::DependencyInfo::default().image_memory_barriers(&post), + ); + dev.end_command_buffer(self.cmd)?; + + dev.reset_fences(&[self.fence])?; + { + let _guard = self.queue_lock.guard(); + let cmds = [self.cmd]; + dev.queue_submit( + self.queue, + &[vk::SubmitInfo::default().command_buffers(&cmds)], + self.fence, + )?; + } + dev.wait_for_fences(&[self.fence], true, 5_000_000_000) + .context("pyrowave decode fence")?; + self.ring[slot].initialized = true; + + Ok(Some(PyroWavePlanarFrame { + views: [ + self.ring[slot].views[0].as_raw(), + self.ring[slot].views[1].as_raw(), + self.ring[slot].views[2].as_raw(), + ], + width: w, + height: h, + // No VUI in the bitstream: BT.709 limited is the fixed contract with the + // host's CSC (plan §4.7 CscRows note; sequence-header signaling is a + // follow-up once the C API exposes it). + color: ColorDesc { + primaries: 1, + transfer: 1, + matrix: 1, + full_range: false, + }, + keyframe: true, + })) + } +} + +impl Drop for PyroWaveDecoder { + fn drop(&mut self) { + // SAFETY: owned handles created by this struct on the presenter's device; the + // fence-synchronous decode means no work of OURS is in flight, and the presenter + // may still be sampling the last handed-over slot — idle the device's queue + // under the shared lock before destroying the plane images. + unsafe { + { + let _guard = self.queue_lock.guard(); + let _ = self.device.queue_wait_idle(self.queue); + } + pw::pyrowave_decoder_destroy(self.pw_dec); + pw::pyrowave_device_destroy(self.pw_dev); + for set in &self.ring { + for v in set.views { + self.device.destroy_image_view(v, None); + } + for i in set.imgs { + self.device.destroy_image(i, None); + } + for m in set.mems { + self.device.free_memory(m, None); + } + } + self.device.destroy_fence(self.fence, None); + self.device.destroy_command_pool(self.cmd_pool, None); + // `self.device`/instance are the PRESENTER's — never destroyed here. + } + } +} diff --git a/crates/pf-presenter/src/vk.rs b/crates/pf-presenter/src/vk.rs index 6b3aac89..38fa93fe 100644 --- a/crates/pf-presenter/src/vk.rs +++ b/crates/pf-presenter/src/vk.rs @@ -497,9 +497,23 @@ impl Presenter { .push_next(&mut have_f12) .push_next(&mut have_f13); unsafe { instance.get_physical_device_features2(pdev, &mut have_f2) }; + // Copy the one base-features fact out NOW: `have_f2` mutably borrows the 11/12/13 + // structs through its pNext chain, so any later use of it would pin those borrows. + let have_shader_int16 = have_f2.features.shader_int16; let features_ok = have_f11.sampler_ycbcr_conversion == vk::TRUE && have_f12.timeline_semaphore == vk::TRUE && have_f13.synchronization2 == vk::TRUE; + // PyroWave decode (the wired-LAN wavelet codec, design/pyrowave-codec-plan.md §4.5): + // plain Vulkan-1.3 compute on THIS device — no video extensions. Probed alongside so a + // capable device gets the features enabled below and advertises the codec; anything + // less simply never sets the CODEC_PYROWAVE bit. + let pyrowave_ok = dev_is_13 + && have_shader_int16 == vk::TRUE + && have_f12.storage_buffer8_bit_access == vk::TRUE + && have_f12.timeline_semaphore == vk::TRUE + && have_f13.subgroup_size_control == vk::TRUE + && have_f13.compute_full_subgroups == vk::TRUE + && have_f13.synchronization2 == vk::TRUE; // The decode queue family + which codec operations it can run. let decode_family: Option<(u32, vk::VideoCodecOperationFlagsKHR)> = { @@ -575,13 +589,18 @@ impl Presenter { let mut en_f11 = vk::PhysicalDeviceVulkan11Features::default() .sampler_ycbcr_conversion(have_f11.sampler_ycbcr_conversion == vk::TRUE); let mut en_f12 = vk::PhysicalDeviceVulkan12Features::default() - .timeline_semaphore(have_f12.timeline_semaphore == vk::TRUE); + .timeline_semaphore(have_f12.timeline_semaphore == vk::TRUE) + .storage_buffer8_bit_access(pyrowave_ok) + .shader_float16(pyrowave_ok && have_f12.shader_float16 == vk::TRUE); let mut en_f13 = vk::PhysicalDeviceVulkan13Features::default() - .synchronization2(have_f13.synchronization2 == vk::TRUE); + .synchronization2(have_f13.synchronization2 == vk::TRUE) + .subgroup_size_control(pyrowave_ok) + .compute_full_subgroups(pyrowave_ok); let mut en_f2 = vk::PhysicalDeviceFeatures2::default() .push_next(&mut en_f11) .push_next(&mut en_f12) .push_next(&mut en_f13); + en_f2.features.shader_int16 = if pyrowave_ok { vk::TRUE } else { vk::FALSE }; let priorities = [1.0f32]; let mut queue_info = vec![vk::DeviceQueueCreateInfo::default() @@ -632,9 +651,9 @@ impl Presenter { // all funnel their queue calls through it — see the `queue_lock` field docs). let queue_lock = std::sync::Arc::new(pf_client_core::video::QueueLock::new()); #[cfg(windows)] - let export_worthy = video_ok || win_capable; + let export_worthy = video_ok || win_capable || pyrowave_ok; #[cfg(not(windows))] - let export_worthy = video_ok; + let export_worthy = video_ok || pyrowave_ok; let video_export = if export_worthy { let qf_props = unsafe { instance.get_physical_device_queue_family_properties(pdev) }; let mut device_extensions: Vec = @@ -678,6 +697,14 @@ impl Presenter { f_sampler_ycbcr: have_f11.sampler_ycbcr_conversion == vk::TRUE, f_timeline_semaphore: have_f12.timeline_semaphore == vk::TRUE, f_synchronization2: have_f13.synchronization2 == vk::TRUE, + f_shader_int16: pyrowave_ok, + f_storage_buffer8: pyrowave_ok, + f_subgroup_size_control: pyrowave_ok, + f_compute_full_subgroups: pyrowave_ok, + f_shader_float16: pyrowave_ok && have_f12.shader_float16 == vk::TRUE, + api_version: dev_props.api_version, + queue_families: queue_info.iter().map(|q| q.queue_family_index).collect(), + pyrowave_decode: pyrowave_ok, video_decode: video_ok, #[cfg(windows)] d3d11_import: win_capable,