diff --git a/Cargo.lock b/Cargo.lock index ebf0a88b..aa18df8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -472,6 +472,8 @@ dependencies = [ "cexpr", "clang-sys", "itertools 0.13.0", + "log", + "prettyplease", "proc-macro2", "quote", "regex", @@ -1193,6 +1195,16 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -2759,6 +2771,18 @@ dependencies = [ "ureq", ] +[[package]] +name = "pf-console-ui" +version = "0.8.2" +dependencies = [ + "anyhow", + "ash", + "pf-presenter", + "sdl3", + "skia-safe", + "tracing", +] + [[package]] name = "pf-driver-proto" version = "0.0.1" @@ -2901,13 +2925,23 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", ] [[package]] @@ -2975,6 +3009,7 @@ version = "0.8.2" dependencies = [ "anyhow", "pf-client-core", + "pf-console-ui", "pf-presenter", "punktfunk-core", "tracing", @@ -3851,6 +3886,15 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -3936,6 +3980,35 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "skia-bindings" +version = "0.87.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704242769235d2ffe66a2a0a3002661262fc4af08d32807c362d7b0160ee703c" +dependencies = [ + "bindgen", + "cc", + "flate2", + "heck", + "lazy_static", + "pkg-config", + "regex", + "serde_json", + "tar", + "toml 0.8.23", +] + +[[package]] +name = "skia-safe" +version = "0.87.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f7d94f3e7537c71ad4cf132eb26e3be8c8a886ed3649c4525c089041fc312b2" +dependencies = [ + "bitflags", + "lazy_static", + "skia-bindings", +] + [[package]] name = "slab" version = "0.4.12" @@ -4071,6 +4144,17 @@ dependencies = [ "version-compare", ] +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.13.5" @@ -4256,6 +4340,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" @@ -4264,7 +4360,7 @@ checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ "indexmap", "serde_core", - "serde_spanned", + "serde_spanned 1.1.1", "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "toml_writer", @@ -4279,13 +4375,22 @@ checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ "indexmap", "serde_core", - "serde_spanned", + "serde_spanned 1.1.1", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", "winnow 1.0.3", ] +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "0.7.5+spec-1.1.0" @@ -4304,6 +4409,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_edit" version = "0.25.12+spec-1.1.0" @@ -4325,6 +4444,12 @@ dependencies = [ "winnow 1.0.3", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "toml_writer" version = "1.1.1+spec-1.1.0" @@ -5368,6 +5493,9 @@ name = "winnow" version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] [[package]] name = "winnow" @@ -5427,6 +5555,16 @@ dependencies = [ "time", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xkbcommon" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 7ba058b9..f90c7b7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/punktfunk-tray", "crates/pf-client-core", "crates/pf-presenter", + "crates/pf-console-ui", "crates/pf-driver-proto", "clients/probe", "clients/linux", diff --git a/clients/linux-session/Cargo.toml b/clients/linux-session/Cargo.toml index 63d524db..017e4a7e 100644 --- a/clients/linux-session/Cargo.toml +++ b/clients/linux-session/Cargo.toml @@ -12,9 +12,17 @@ repository.workspace = true name = "punktfunk-session" path = "src/main.rs" +[features] +default = ["ui"] +# The Skia console UI (stats OSD, capture HUD, later the gamepad library). Dropping it +# (`--no-default-features`) is the ~15 MB-smaller power-user build: same streaming, +# stats on stdout only. +ui = ["dep:pf-console-ui"] + # Same Linux gating as the rest of the client stack; elsewhere this is a stub binary. [target.'cfg(target_os = "linux")'.dependencies] pf-presenter = { path = "../../crates/pf-presenter" } +pf-console-ui = { path = "../../crates/pf-console-ui", optional = true } pf-client-core = { path = "../../crates/pf-client-core" } punktfunk-core = { path = "../../crates/punktfunk-core", features = ["quic"] } anyhow = "1" diff --git a/clients/linux-session/README.md b/clients/linux-session/README.md index 164cc31c..12a89673 100644 --- a/clients/linux-session/README.md +++ b/clients/linux-session/README.md @@ -23,6 +23,11 @@ In-stream keys match the desktop client: click captures input (Ctrl+Alt+Shift+Q releases), Ctrl+Alt+Shift+D disconnects, F11 toggles fullscreen; the controller escape chord (L1+R1+Start+Select, hold to disconnect) works the same. +The default build carries the Skia console UI (`ui` feature): the stats OSD and capture +hint render in-window (Ctrl+Alt+Shift+S toggles both the OSD and the stdout mirror). +`--no-default-features` is the ~5 MB power-user build — same streaming, stats on stdout +only, no Skia anywhere in the dependency tree. + Decode follows the Settings preference: VAAPI frames import zero-copy into Vulkan (per-plane dmabuf + the stream's CICP-driven CSC shader); boxes whose driver can't import (NVIDIA proprietary by design) fall back to software decode automatically. diff --git a/clients/linux-session/src/main.rs b/clients/linux-session/src/main.rs index 417313a8..be8e8980 100644 --- a/clients/linux-session/src/main.rs +++ b/clients/linux-session/src/main.rs @@ -152,6 +152,12 @@ mod session_main { // This host's card carries the accent bar in the desktop client now. trust::touch_last_used(&trust::hex(&fingerprint)); })), + // The Skia console UI (stats OSD, capture HUD) — compiled out of the + // power-user build (`--no-default-features` drops the `ui` feature). + #[cfg(feature = "ui")] + overlay: Some(Box::new(pf_console_ui::SkiaOverlay::new())), + #[cfg(not(feature = "ui"))] + overlay: None, }; let outcome = pf_presenter::run_session(opts, move |gamepad, native, force_software| { diff --git a/crates/pf-console-ui/Cargo.toml b/crates/pf-console-ui/Cargo.toml new file mode 100644 index 00000000..a1ba6da0 --- /dev/null +++ b/crates/pf-console-ui/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "pf-console-ui" +description = "The Skia console UI for the Vulkan session binary — stats OSD, capture HUD, and (next) the gamepad library; renders on the presenter's device via the Overlay contract" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[target.'cfg(target_os = "linux")'.dependencies] +pf-presenter = { path = "../pf-presenter" } + +# Skia on the presenter's VkDevice (`vulkan`); `textlayout` = skparagraph/harfbuzz for +# the typography the console library needs (~15 MB stripped, prebuilt binaries exist for +# this feature set on x86_64-unknown-linux-gnu — a source build is never triggered). +skia-safe = { version = "0.87", features = ["vulkan", "textlayout"] } +ash = { version = "0.38", features = ["loaded"] } +sdl3 = { version = "0.18", features = ["hidapi", "ash"] } + +anyhow = "1" +tracing = "0.1" diff --git a/crates/pf-console-ui/src/lib.rs b/crates/pf-console-ui/src/lib.rs new file mode 100644 index 00000000..e461ad29 --- /dev/null +++ b/crates/pf-console-ui/src/lib.rs @@ -0,0 +1,15 @@ +//! The Skia console UI (punktfunk-planning `linux-client-rearchitecture.md` §6): an +//! [`Overlay`] implementation rendering on the PRESENTER's Vulkan device into offscreen +//! RGBA images the presenter composites as one premultiplied quad. Skia never touches +//! the swapchain, and nothing here runs while the overlay has nothing to show — the +//! §6.1 invariants live or die in this crate. +//! +//! Milestone 1 (this file): the stats OSD panel + the capture-hint pill — small on +//! purpose, it proves the whole shared-device pipeline. The gamepad library moves in +//! next. + +#[cfg(target_os = "linux")] +mod skia_overlay; + +#[cfg(target_os = "linux")] +pub use skia_overlay::SkiaOverlay; diff --git a/crates/pf-console-ui/src/skia_overlay.rs b/crates/pf-console-ui/src/skia_overlay.rs new file mode 100644 index 00000000..5c9b4f29 --- /dev/null +++ b/crates/pf-console-ui/src/skia_overlay.rs @@ -0,0 +1,330 @@ +//! Skia on the presenter's device: `DirectContext` over the shared handles, a ring of +//! two offscreen render-target surfaces (the presenter runs one frame in flight and may +//! still be sampling the previous image while we render the next), damage-driven +//! redraws (content/size change only — an unchanged OSD costs zero GPU work per frame). + +use anyhow::{anyhow, Context as _, Result}; +use ash::vk as avk; +use ash::vk::Handle as _; +use pf_presenter::overlay::{FrameCtx, Overlay, OverlayFrame, SharedDevice}; +use skia_safe::gpu::vk as skvk; +use skia_safe::gpu::{self, DirectContext, SurfaceOrigin}; +use skia_safe::{Canvas, Color4f, Font, FontMgr, Paint, Point, RRect, Rect, Surface}; + +/// Skia's GPU resource budget — the OSD/HUD need a few MB; the console library will +/// revisit (the plan budgets 64 MB on Deck-class shared memory). +const RESOURCE_CACHE_BYTES: usize = 64 << 20; + +/// One offscreen target: the Skia surface + the raw Vulkan handles the presenter +/// samples. The image is Skia-owned (freed with the surface); the view is ours. +struct Slot { + surface: Surface, + image: avk::Image, + view: avk::ImageView, + width: u32, + height: u32, +} + +/// What the current ring slot has drawn — re-render only when this changes. +#[derive(PartialEq, Clone, Default)] +struct Drawn { + width: u32, + height: u32, + stats: Option, + hint: Option, +} + +pub struct SkiaOverlay { + /// Set by `init`; `None` until then (and after an init failure the run loop drops + /// the whole overlay, so mid-session these are always `Some`). + gpu: Option, + slots: [Option; 2], + /// Which slot the LAST returned frame lives in — the next render takes the other. + current: usize, + drawn: Drawn, + font: Option, +} + +struct Gpu { + device: ash::Device, + queue_family_index: u32, + context: DirectContext, + // Keep the loader library + instance dispatch alive as long as the DirectContext + // (its baked fn pointers live inside libvulkan). + _entry: ash::Entry, + _instance: ash::Instance, +} + +impl SkiaOverlay { + #[allow(clippy::new_without_default)] + pub fn new() -> SkiaOverlay { + SkiaOverlay { + gpu: None, + slots: [None, None], + current: 0, + drawn: Drawn::default(), + font: None, + } + } +} + +impl Drop for SkiaOverlay { + /// The run loop quiesces the queue before dropping us; releasing the views + Skia + /// surfaces (which free their VkImages) is then safe. Field order drops the slots + /// before the DirectContext. + fn drop(&mut self) { + if let Some(gpu) = &mut self.gpu { + for slot in self.slots.iter_mut().flat_map(Option::take) { + unsafe { gpu.device.destroy_image_view(slot.view, None) }; + drop(slot.surface); + } + gpu.context.flush_and_submit(); + } + } +} + +impl Overlay for SkiaOverlay { + fn init(&mut self, shared: &SharedDevice) -> Result<()> { + // Skia resolves its Vulkan entry points through us: instance-scoped names via + // the loader, device-scoped via the device — the exact same dispatch ash uses. + // Resolution completes inside `make_vulkan` (the DirectContext bakes its fn + // table); the closure and its clones end with `init`. + let entry = shared.entry.clone(); + let instance = shared.instance.clone(); + let get_proc = move |of: skvk::GetProcOf| -> *const std::ffi::c_void { + unsafe { + match of { + skvk::GetProcOf::Instance(raw_instance, name) => entry + .get_instance_proc_addr(avk::Instance::from_raw(raw_instance as _), name) + .map_or(std::ptr::null(), |f| f as *const std::ffi::c_void), + skvk::GetProcOf::Device(raw_device, name) => (instance.fp_v1_0() + .get_device_proc_addr)( + avk::Device::from_raw(raw_device as _), name + ) + .map_or(std::ptr::null(), |f| f as *const std::ffi::c_void), + } + } + }; + let backend = unsafe { + skvk::BackendContext::new( + shared.instance.handle().as_raw() as _, + shared.physical_device.as_raw() as _, + shared.device.handle().as_raw() as _, + ( + shared.queue.as_raw() as _, + shared.queue_family_index as usize, + ), + &get_proc, + ) + }; + let mut context = gpu::direct_contexts::make_vulkan(&backend, None) + .ok_or_else(|| anyhow!("Skia DirectContext over the shared device"))?; + context.set_resource_cache_limit(RESOURCE_CACHE_BYTES); + + let typeface = FontMgr::new() + .match_family_style("monospace", skia_safe::FontStyle::normal()) + .context("no monospace typeface via fontconfig")?; + self.font = Some(Font::new(typeface, 14.0)); + + self.gpu = Some(Gpu { + device: shared.device.clone(), + queue_family_index: shared.queue_family_index, + context, + _entry: shared.entry.clone(), + _instance: shared.instance.clone(), + }); + tracing::info!("Skia console UI on the presenter's device"); + Ok(()) + } + + fn handle_event(&mut self, _event: &sdl3::event::Event) -> bool { + false // the OSD/HUD consume nothing; the console library will + } + + fn frame(&mut self, ctx: &FrameCtx) -> Result> { + if ctx.stats.is_none() && ctx.hint.is_none() { + self.drawn = Drawn::default(); // forget content so re-show re-renders + return Ok(None); + } + let want = Drawn { + width: ctx.width, + height: ctx.height, + stats: ctx.stats.map(str::to_owned), + hint: ctx.hint.map(str::to_owned), + }; + if want == self.drawn { + // Unchanged — hand the presenter the already-rendered image. + return Ok(self + .slots[self.current] + .as_ref() + .map(|s| OverlayFrame { + image: s.image, + view: s.view, + width: s.width, + height: s.height, + })); + } + + // Render into the OTHER slot — the presenter may still be sampling the current + // one (one frame in flight; the ring of two is exactly deep enough). + let next = 1 - self.current; + self.ensure_slot(next, ctx.width, ctx.height)?; + let gpu = self.gpu.as_mut().expect("init ran"); + let slot = self.slots[next].as_mut().expect("just ensured"); + + let canvas = slot.surface.canvas(); + canvas.clear(Color4f::new(0.0, 0.0, 0.0, 0.0)); + let font = self.font.as_ref().expect("init ran"); + if let Some(stats) = &want.stats { + draw_osd_panel(canvas, font, stats, 12.0, 12.0); + } + if let Some(hint) = &want.hint { + draw_hint_pill(canvas, font, hint, ctx.width, ctx.height); + } + + // Flush on the shared queue, ending in SHADER_READ_ONLY on our family — the + // layout the presenter's composite samples (its own barrier covers visibility). + gpu.context.flush_surface_with_texture_state( + &mut slot.surface, + &gpu::FlushInfo::default(), + Some(&skvk::mutable_texture_states::new_vulkan( + skvk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + gpu.queue_family_index, + )), + ); + gpu.context.submit(None); + + self.current = next; + self.drawn = want; + Ok(Some(OverlayFrame { + image: slot.image, + view: slot.view, + width: slot.width, + height: slot.height, + })) + } +} + +impl SkiaOverlay { + /// Make `slots[i]` a render target of exactly `width`×`height` (rebuilt on resize). + fn ensure_slot(&mut self, i: usize, width: u32, height: u32) -> Result<()> { + if self.slots[i] + .as_ref() + .is_some_and(|s| s.width == width && s.height == height) + { + return Ok(()); + } + let gpu = self.gpu.as_mut().expect("init ran"); + if let Some(old) = self.slots[i].take() { + // Any in-flight sampling of THIS slot ended two presents ago (the ring + // alternates and the presenter waits its fence before each record). + unsafe { gpu.device.destroy_image_view(old.view, None) }; + } + let info = skia_safe::ImageInfo::new_n32_premul( + (width.max(1) as i32, height.max(1) as i32), + None, + ); + let mut surface = gpu::surfaces::render_target( + &mut gpu.context, + gpu::Budgeted::Yes, + &info, + None, + SurfaceOrigin::TopLeft, + None, + false, + None, + ) + .context("Skia render-target surface")?; + let texture = + gpu::surfaces::get_backend_texture( + &mut surface, + skia_safe::surface::BackendHandleAccess::FlushRead, + ) + .context("surface backend texture")?; + let image_info = texture + .vulkan_image_info() + .context("backend texture is not Vulkan")?; + let image = avk::Image::from_raw(*image_info.image() as u64); + let view = unsafe { + gpu.device.create_image_view( + &avk::ImageViewCreateInfo::default() + .image(image) + .view_type(avk::ImageViewType::TYPE_2D) + .format(avk::Format::from_raw(image_info.format as i32)) + .subresource_range( + avk::ImageSubresourceRange::default() + .aspect_mask(avk::ImageAspectFlags::COLOR) + .level_count(1) + .layer_count(1), + ), + None, + ) + } + .context("overlay image view")?; + self.slots[i] = Some(Slot { + surface, + image, + view, + width, + height, + }); + Ok(()) + } +} + +/// The stats OSD: a translucent rounded panel, one text line per `\n` (the GTK OSD's +/// look, minus the toolkit). +fn draw_osd_panel(canvas: &Canvas, font: &Font, text: &str, x: f32, y: f32) { + let (_, metrics) = font.metrics(); + let line_h = metrics.descent - metrics.ascent + metrics.leading; + let lines: Vec<&str> = text.lines().collect(); + let widest = lines + .iter() + .map(|l| font.measure_str(l, None).0) + .fold(0.0f32, f32::max); + let (pad_x, pad_y) = (10.0, 8.0); + let panel = Rect::from_xywh( + x, + y, + widest + 2.0 * pad_x, + line_h * lines.len() as f32 + 2.0 * pad_y, + ); + canvas.draw_rrect( + RRect::new_rect_xy(panel, 8.0, 8.0), + &Paint::new(Color4f::new(0.0, 0.0, 0.0, 0.62), None), + ); + let text_paint = Paint::new(Color4f::new(1.0, 1.0, 1.0, 0.92), None); + for (i, line) in lines.iter().enumerate() { + canvas.draw_str( + line, + Point::new( + x + pad_x, + y + pad_y - metrics.ascent + line_h * i as f32, + ), + font, + &text_paint, + ); + } +} + +/// The capture hint: a centered pill near the bottom edge (the GTK hint's position). +fn draw_hint_pill(canvas: &Canvas, font: &Font, text: &str, width: u32, height: u32) { + let (_, metrics) = font.metrics(); + let line_h = metrics.descent - metrics.ascent; + let text_w = font.measure_str(text, None).0; + let (pad_x, pad_y) = (14.0, 8.0); + let w = text_w + 2.0 * pad_x; + let h = line_h + 2.0 * pad_y; + let x = (width as f32 - w) / 2.0; + let y = height as f32 - h - 24.0; + canvas.draw_rrect( + RRect::new_rect_xy(Rect::from_xywh(x, y, w, h), h / 2.0, h / 2.0), + &Paint::new(Color4f::new(0.0, 0.0, 0.0, 0.62), None), + ); + canvas.draw_str( + text, + Point::new(x + pad_x, y + pad_y - metrics.ascent), + font, + &Paint::new(Color4f::new(1.0, 1.0, 1.0, 0.92), None), + ); +} diff --git a/crates/pf-presenter/shaders/overlay.frag b/crates/pf-presenter/shaders/overlay.frag new file mode 100644 index 00000000..84e84cec --- /dev/null +++ b/crates/pf-presenter/shaders/overlay.frag @@ -0,0 +1,15 @@ +// The overlay composite: one premultiplied-alpha RGBA texture (the console UI's +// offscreen image) sampled over the swapchain. Blending happens in fixed function +// (src ONE, dst ONE_MINUS_SRC_ALPHA — Skia surfaces are premultiplied). +// +// Regenerate: shaders/build.sh (committed .spv, no build-time toolchain). +#version 450 + +layout(location = 0) in vec2 v_uv; +layout(location = 0) out vec4 frag; + +layout(set = 0, binding = 0) uniform sampler2D u_tex; + +void main() { + frag = texture(u_tex, v_uv); +} diff --git a/crates/pf-presenter/shaders/overlay.frag.spv b/crates/pf-presenter/shaders/overlay.frag.spv new file mode 100644 index 00000000..e9034726 Binary files /dev/null and b/crates/pf-presenter/shaders/overlay.frag.spv differ diff --git a/crates/pf-presenter/src/csc.rs b/crates/pf-presenter/src/csc.rs index c21bd194..a5b7ec18 100644 --- a/crates/pf-presenter/src/csc.rs +++ b/crates/pf-presenter/src/csc.rs @@ -168,7 +168,13 @@ impl CscPass { ) }?[0]; - let pipeline = build_pipeline(device, render_pass, pipeline_layout)?; + let pipeline = build_fullscreen_pipeline( + device, + render_pass, + pipeline_layout, + include_bytes!("../shaders/nv12_csc.frag.spv"), + false, // opaque — the CSC output IS the video + )?; Ok(CscPass { render_pass, @@ -217,19 +223,23 @@ impl CscPass { } } -fn build_pipeline( +/// A bufferless fullscreen-triangle pipeline over `fullscreen.vert` + the given +/// fragment SPIR-V, dynamic viewport/scissor. `blend` = premultiplied-alpha over the +/// destination (the overlay composite); `false` = opaque write (the CSC pass). Shared +/// by both passes — the geometry and states are identical. +pub(crate) fn build_fullscreen_pipeline( device: &ash::Device, render_pass: vk::RenderPass, layout: vk::PipelineLayout, + frag_spv: &[u8], + blend: bool, ) -> Result { // Committed SPIR-V (shaders/build.sh) — include_bytes! alignment is unspecified, so // read_spv copies into aligned Vecs. let vert = ash::util::read_spv(&mut std::io::Cursor::new( &include_bytes!("../shaders/fullscreen.vert.spv")[..], ))?; - let frag = ash::util::read_spv(&mut std::io::Cursor::new( - &include_bytes!("../shaders/nv12_csc.frag.spv")[..], - ))?; + let frag = ash::util::read_spv(&mut std::io::Cursor::new(frag_spv))?; let vert_mod = unsafe { device.create_shader_module(&vk::ShaderModuleCreateInfo::default().code(&vert), None) }?; @@ -271,8 +281,21 @@ fn build_pipeline( .line_width(1.0); let multisample = vk::PipelineMultisampleStateCreateInfo::default() .rasterization_samples(vk::SampleCountFlags::TYPE_1); - let blend_attachment = [vk::PipelineColorBlendAttachmentState::default() - .color_write_mask(vk::ColorComponentFlags::RGBA)]; + let blend_attachment = [if blend { + // Premultiplied alpha over the destination (Skia surfaces are premultiplied). + vk::PipelineColorBlendAttachmentState::default() + .color_write_mask(vk::ColorComponentFlags::RGBA) + .blend_enable(true) + .src_color_blend_factor(vk::BlendFactor::ONE) + .dst_color_blend_factor(vk::BlendFactor::ONE_MINUS_SRC_ALPHA) + .color_blend_op(vk::BlendOp::ADD) + .src_alpha_blend_factor(vk::BlendFactor::ONE) + .dst_alpha_blend_factor(vk::BlendFactor::ONE_MINUS_SRC_ALPHA) + .alpha_blend_op(vk::BlendOp::ADD) + } else { + vk::PipelineColorBlendAttachmentState::default() + .color_write_mask(vk::ColorComponentFlags::RGBA) + }]; let blend = vk::PipelineColorBlendStateCreateInfo::default().attachments(&blend_attachment); let info = vk::GraphicsPipelineCreateInfo::default() diff --git a/crates/pf-presenter/src/lib.rs b/crates/pf-presenter/src/lib.rs index 9ee5749a..97ad448a 100644 --- a/crates/pf-presenter/src/lib.rs +++ b/crates/pf-presenter/src/lib.rs @@ -18,6 +18,8 @@ pub mod input; #[cfg(target_os = "linux")] pub mod keymap_sdl; #[cfg(target_os = "linux")] +pub mod overlay; +#[cfg(target_os = "linux")] mod run; #[cfg(target_os = "linux")] pub mod vk; diff --git a/crates/pf-presenter/src/overlay.rs b/crates/pf-presenter/src/overlay.rs new file mode 100644 index 00000000..83a6e349 --- /dev/null +++ b/crates/pf-presenter/src/overlay.rs @@ -0,0 +1,63 @@ +//! The presenter↔console-UI contract (punktfunk-planning +//! `linux-client-rearchitecture.md` §6.1): the presenter exposes its device and +//! composites at most ONE sampled RGBA quad per frame; the overlay implementation +//! (pf-console-ui, Skia) fills offscreen images on its own damage-driven schedule. No +//! Skia type crosses this line — everything here is ash — and a `frame()` returning +//! `None` costs the hot path nothing (the quad isn't even recorded). + +use ash::vk; + +/// The presenter's device, shared with the overlay so its renderer (Skia's +/// `DirectContext`) creates resources on the same VkDevice/queue. Handles stay valid for +/// the presenter's lifetime — the overlay must be dropped before it (the run loop owns +/// both and drops the overlay first). +pub struct SharedDevice { + pub entry: ash::Entry, + pub instance: ash::Instance, + pub physical_device: vk::PhysicalDevice, + pub device: ash::Device, + pub queue: vk::Queue, + pub queue_family_index: u32, +} + +/// What the overlay may draw this frame — composed by the run loop from session state. +/// Milestone 1 (OSD/HUD) is text-shaped; the console library replaces this with a +/// richer scene enum when it moves in. +pub struct FrameCtx<'a> { + /// Swapchain size in pixels — the overlay renders 1:1. + pub width: u32, + pub height: u32, + /// Multi-line stats OSD (top-left panel); `None` = hidden. + pub stats: Option<&'a str>, + /// The capture hint (bottom-center pill, "click to capture…"); `None` = hidden. + pub hint: Option<&'a str>, +} + +/// One overlay image ready to composite: RGBA, PREMULTIPLIED alpha, already in +/// `SHADER_READ_ONLY_OPTIMAL`, sized `width`×`height` (normally the `FrameCtx` size; a +/// stale size during a resize just stretches for a frame). +pub struct OverlayFrame { + pub image: vk::Image, + pub view: vk::ImageView, + pub width: u32, + pub height: u32, +} + +/// The console-UI side. Object-safe; the session binary passes +/// `Option>` (None = the Skia-free power-user build). +pub trait Overlay { + /// One-time setup on the presenter's device. + fn init(&mut self, shared: &SharedDevice) -> anyhow::Result<()>; + + /// Input routing, before capture sees the event. `true` = consumed (a menu is up) — + /// the event must not reach capture/forwarding. The OSD/HUD milestone consumes + /// nothing; the console library will. + fn handle_event(&mut self, event: &sdl3::event::Event) -> bool; + + /// Once per presenter iteration. Damage-driven: re-render (flush + transition to + /// SHADER_READ_ONLY) only when the content or size changed, else return the previous + /// image. `None` = nothing to composite. The returned image must stay untouched + /// until `frame()` runs again (the presenter runs one frame in flight and the + /// implementation keeps a ring of two, so alternating satisfies this). + fn frame(&mut self, ctx: &FrameCtx) -> anyhow::Result>; +} diff --git a/crates/pf-presenter/src/run.rs b/crates/pf-presenter/src/run.rs index ba7f4d93..c8537ea8 100644 --- a/crates/pf-presenter/src/run.rs +++ b/crates/pf-presenter/src/run.rs @@ -7,6 +7,7 @@ //! (Ctrl+Alt+Shift+S toggles). Logs go to stderr (the binary configures tracing so). use crate::input::Capture; +use crate::overlay::{FrameCtx, Overlay, OverlayFrame}; use crate::vk::{FrameInput, Presenter}; use anyhow::{Context as _, Result}; use pf_client_core::gamepad::GamepadService; @@ -31,6 +32,10 @@ pub struct SessionOpts { /// Called once on `Connected` with the host's fingerprint (trust persistence is the /// binary's business — this loop stays store-agnostic). pub on_connected: Option>, + /// The console-UI overlay (§6.1) — `None` is the Skia-free power-user build (stats + /// stay stdout-only). An overlay whose `init` fails degrades to `None` with a + /// warning rather than killing the session. + pub overlay: Option>, } pub enum Outcome { @@ -68,7 +73,16 @@ where .map_err(|e| anyhow::anyhow!("vulkan instance extensions: {e}"))?; let mut presenter = Presenter::new(&window, &instance_exts).context("vulkan presenter")?; // A valid black frame immediately — the window is honest while the connect runs. - presenter.present(&window, FrameInput::Redraw)?; + presenter.present(&window, FrameInput::Redraw, None)?; + + let mut overlay = opts.overlay.take(); + if let Some(o) = overlay.as_mut() { + if let Err(e) = o.init(&presenter.shared_device()) { + tracing::warn!(error = %format!("{e:#}"), + "console-UI overlay init failed — continuing without it"); + overlay = None; + } + } let gamepad_subsystem = sdl.gamepad().context("SDL gamepad")?; let (gamepad, mut pump) = GamepadService::pumped(gamepad_subsystem); @@ -119,6 +133,11 @@ where // demotes the decoder to software via the shared flag — once per session. let mut dmabuf_demoted = false; let mut hw_fails = 0u32; + // The OSD's text (multi-line; rebuilt each Stats window) and the console-UI frame + // currently composited. The frame persists across presents within an iteration — + // the overlay only re-renders when `frame()` runs again (ring of two, see §6.1). + let mut osd_text = String::new(); + let mut overlay_frame: Option = None; let outcome = 'main: loop { // --- SDL events (input, window, gamepads) --------------------------------------- @@ -134,6 +153,13 @@ where queued.push(e); } for event in queued { + // The console UI sees input first: a consumed event (its menu is up) never + // reaches capture/forwarding. The OSD/HUD milestone consumes nothing. + if let Some(o) = overlay.as_mut() { + if o.handle_event(&event) { + continue; + } + } match event { Event::Quit { .. } => { // Window close / SIGINT: deliberate user exit — QUIT_CLOSE_CODE so @@ -155,10 +181,10 @@ where } WindowEvent::PixelSizeChanged(..) | WindowEvent::Resized(..) => { presenter.recreate_swapchain(&window)?; - presenter.present(&window, FrameInput::Redraw)?; + presenter.present(&window, FrameInput::Redraw, overlay_frame.as_ref())?; } WindowEvent::Exposed => { - presenter.present(&window, FrameInput::Redraw)?; + presenter.present(&window, FrameInput::Redraw, overlay_frame.as_ref())?; } _ => {} }, @@ -302,8 +328,9 @@ where } } SessionEvent::Stats(s) => { + osd_text = stats_text(&mode_line, &s, &presented, hdr); if print_stats { - print_stats_line(&mode_line, &s, &presented, hdr); + println!("stats: {}", osd_text.replace('\n', " | ")); } } SessionEvent::Failed { msg, trust_rejected } => { @@ -320,6 +347,35 @@ where } } + // --- Console UI: damage-driven overlay re-render for this iteration -------------- + if let Some(o) = overlay.as_mut() { + let (pw, ph) = window.size_in_pixels(); + let hint = match &capture { + Some(cap) if !cap.captured() => Some(if gamepad.active().is_some() { + HINT_WITH_PAD + } else { + HINT_KEYBOARD + }), + _ => None, + }; + let ctx = FrameCtx { + width: pw, + height: ph, + stats: (print_stats && connector.is_some() && !osd_text.is_empty()) + .then_some(osd_text.as_str()), + hint, + }; + match o.frame(&ctx) { + Ok(f) => overlay_frame = f, + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), + "overlay frame failed — disabling the console UI"); + overlay = None; + overlay_frame = None; + } + } + } + // --- Frames: drain to the newest, upload + present ------------------------------- let mut newest: Option = None; while let Ok(f) = handle.frames.try_recv() { @@ -334,11 +390,11 @@ where let did_present = match image { DecodedImage::Cpu(c) => { hdr = c.color.is_pq(); - presenter.present(&window, FrameInput::Cpu(&c))? + presenter.present(&window, FrameInput::Cpu(&c), overlay_frame.as_ref())? } DecodedImage::Dmabuf(d) if presenter.supports_dmabuf() && !dmabuf_demoted => { hdr = d.color.is_pq(); - match presenter.present(&window, FrameInput::Dmabuf(d)) { + match presenter.present(&window, FrameInput::Dmabuf(d), overlay_frame.as_ref()) { Ok(p) => { hw_fails = 0; p @@ -406,6 +462,10 @@ where }; handle.stop.store(true, Ordering::SeqCst); + // Overlay resources live on the presenter's device: quiesce the queue first, drop + // the overlay (its Drop destroys the Skia surfaces), THEN the presenter tears down. + presenter.wait_idle(); + drop(overlay); Ok(outcome) } @@ -417,32 +477,37 @@ struct PresentedWindow { display_ms: f32, } -/// One `stats:` line per 1 s window — the OSD's numbers (design/stats-unification.md) in -/// terminal form: headline end-to-end capture→displayed, then the per-stage p50s. -fn print_stats_line(mode_line: &str, s: &Stats, p: &PresentedWindow, hdr: bool) { - let mut line = format!( - "stats: {mode_line} · {:.0} fps · {:.1} Mb/s · {}{} · e2e {:.1}/{:.1} ms (p50/p95)", +/// The capture hints (`ui_stream` parity — the words the user reads while released). +const HINT_KEYBOARD: &str = "Click the stream to capture input · Ctrl+Alt+Shift+Q releases · \ + Ctrl+Alt+Shift+D disconnects · Ctrl+Alt+Shift+S stats"; +const HINT_WITH_PAD: &str = "Click the stream to capture input · Ctrl+Alt+Shift+Q releases · \ + Ctrl+Alt+Shift+D disconnects · hold L1 + R1 + Start + Select to leave"; + +/// The unified stats window (design/stats-unification.md) as OSD text — multi-line for +/// the console-UI panel; the stdout `stats:` line joins it with `|`. +fn stats_text(mode_line: &str, s: &Stats, p: &PresentedWindow, hdr: bool) -> String { + let mut text = format!( + "{mode_line} · {:.0} fps · {:.1} Mb/s · {}{}", s.fps, s.mbps, if s.decoder.is_empty() { "-" } else { s.decoder }, if hdr { " · HDR" } else { "" }, - p.e2e_p50_ms, - p.e2e_p95_ms, ); + text.push_str(&format!( + "\ne2e {:.1}/{:.1} ms (p50/p95)", + p.e2e_p50_ms, p.e2e_p95_ms + )); if s.split { - line.push_str(&format!( - " · host {:.1} · net {:.1}", - s.host_ms, s.net_ms - )); + text.push_str(&format!(" · host {:.1} · net {:.1}", s.host_ms, s.net_ms)); } else { - line.push_str(&format!(" · host+net {:.1}", s.host_net_ms)); + text.push_str(&format!(" · host+net {:.1}", s.host_net_ms)); } - line.push_str(&format!( + text.push_str(&format!( " · decode {:.1} · display {:.1} ms", s.decode_ms, p.display_ms )); if s.lost > 0 { - line.push_str(&format!(" · lost {} ({:.1}%)", s.lost, s.lost_pct)); + text.push_str(&format!("\nlost {} ({:.1}%)", s.lost, s.lost_pct)); } - println!("{line}"); + text } diff --git a/crates/pf-presenter/src/vk.rs b/crates/pf-presenter/src/vk.rs index 1e1b8f27..2e1f659d 100644 --- a/crates/pf-presenter/src/vk.rs +++ b/crates/pf-presenter/src/vk.rs @@ -14,8 +14,9 @@ //! arrival-paced by the caller: a frame input on each decoded frame, //! `FrameInput::Redraw` re-blits the retained video image (expose/resize redraws). -use crate::csc::{csc_rows, CscPass}; +use crate::csc::{build_fullscreen_pipeline, csc_rows, CscPass}; use crate::dmabuf::{self, HwFrame}; +use crate::overlay::{OverlayFrame, SharedDevice}; use anyhow::{anyhow, bail, Context as _, Result}; use ash::vk; use pf_client_core::video::{CpuFrame, DmabufFrame}; @@ -35,6 +36,191 @@ struct HwCtx { csc: CscPass, } +/// The overlay composite: one premultiplied-alpha quad blended over the swapchain image +/// after the video blit (the §6.1 contract's presenter half). Always built — it has no +/// Skia dependency and costs nothing while no overlay frame arrives (the render pass +/// isn't even recorded). +struct OverlayPipe { + render_pass: vk::RenderPass, + set_layout: vk::DescriptorSetLayout, + pipeline_layout: vk::PipelineLayout, + pipeline: vk::Pipeline, + desc_pool: vk::DescriptorPool, + desc_set: vk::DescriptorSet, + sampler: vk::Sampler, + /// Per-swapchain-image render targets, rebuilt with the swapchain. + views: Vec, + framebuffers: Vec, +} + +impl OverlayPipe { + fn new(device: &ash::Device, format: vk::Format) -> Result { + // LOAD the blitted video, blend the overlay, end PRESENT-ready — this pass owns + // the swapchain image's final transition on overlay frames. + let attachment = [vk::AttachmentDescription::default() + .format(format) + .samples(vk::SampleCountFlags::TYPE_1) + .load_op(vk::AttachmentLoadOp::LOAD) + .store_op(vk::AttachmentStoreOp::STORE) + .initial_layout(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL) + .final_layout(vk::ImageLayout::PRESENT_SRC_KHR)]; + let color_ref = [vk::AttachmentReference::default() + .attachment(0) + .layout(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL)]; + let subpass = [vk::SubpassDescription::default() + .pipeline_bind_point(vk::PipelineBindPoint::GRAPHICS) + .color_attachments(&color_ref)]; + let deps = [vk::SubpassDependency::default() + .src_subpass(vk::SUBPASS_EXTERNAL) + .dst_subpass(0) + .src_stage_mask(vk::PipelineStageFlags::ALL_COMMANDS) + .src_access_mask(vk::AccessFlags::MEMORY_WRITE) + .dst_stage_mask(vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT) + .dst_access_mask( + vk::AccessFlags::COLOR_ATTACHMENT_READ | vk::AccessFlags::COLOR_ATTACHMENT_WRITE, + )]; + let render_pass = unsafe { + device.create_render_pass( + &vk::RenderPassCreateInfo::default() + .attachments(&attachment) + .subpasses(&subpass) + .dependencies(&deps), + None, + ) + } + .context("overlay render pass")?; + + let sampler = unsafe { + device.create_sampler( + &vk::SamplerCreateInfo::default() + .mag_filter(vk::Filter::LINEAR) + .min_filter(vk::Filter::LINEAR) + .address_mode_u(vk::SamplerAddressMode::CLAMP_TO_EDGE) + .address_mode_v(vk::SamplerAddressMode::CLAMP_TO_EDGE) + .address_mode_w(vk::SamplerAddressMode::CLAMP_TO_EDGE), + None, + ) + }?; + let samplers = [sampler]; + let bindings = [vk::DescriptorSetLayoutBinding::default() + .binding(0) + .descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) + .descriptor_count(1) + .stage_flags(vk::ShaderStageFlags::FRAGMENT) + .immutable_samplers(&samplers)]; + let set_layout = unsafe { + device.create_descriptor_set_layout( + &vk::DescriptorSetLayoutCreateInfo::default().bindings(&bindings), + None, + ) + }?; + let set_layouts = [set_layout]; + let pipeline_layout = unsafe { + device.create_pipeline_layout( + &vk::PipelineLayoutCreateInfo::default().set_layouts(&set_layouts), + None, + ) + }?; + let pool_sizes = [vk::DescriptorPoolSize::default() + .ty(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) + .descriptor_count(1)]; + let desc_pool = unsafe { + device.create_descriptor_pool( + &vk::DescriptorPoolCreateInfo::default() + .max_sets(1) + .pool_sizes(&pool_sizes), + None, + ) + }?; + let desc_set = unsafe { + device.allocate_descriptor_sets( + &vk::DescriptorSetAllocateInfo::default() + .descriptor_pool(desc_pool) + .set_layouts(&set_layouts), + ) + }?[0]; + let pipeline = build_fullscreen_pipeline( + device, + render_pass, + pipeline_layout, + include_bytes!("../shaders/overlay.frag.spv"), + true, // premultiplied blend over the video + )?; + Ok(OverlayPipe { + render_pass, + set_layout, + pipeline_layout, + pipeline, + desc_pool, + desc_set, + sampler, + views: Vec::new(), + framebuffers: Vec::new(), + }) + } + + /// Rebuild the per-swapchain-image views + framebuffers (swapchain recreation). + fn rebuild_targets( + &mut self, + device: &ash::Device, + images: &[vk::Image], + format: vk::Format, + extent: vk::Extent2D, + ) -> Result<()> { + self.destroy_targets(device); + for &image in images { + let view = unsafe { + device.create_image_view( + &vk::ImageViewCreateInfo::default() + .image(image) + .view_type(vk::ImageViewType::TYPE_2D) + .format(format) + .subresource_range(subresource_range()), + None, + ) + }?; + self.views.push(view); + let attachments = [view]; + let fb = unsafe { + device.create_framebuffer( + &vk::FramebufferCreateInfo::default() + .render_pass(self.render_pass) + .attachments(&attachments) + .width(extent.width) + .height(extent.height) + .layers(1), + None, + ) + }?; + self.framebuffers.push(fb); + } + Ok(()) + } + + fn destroy_targets(&mut self, device: &ash::Device) { + unsafe { + for fb in self.framebuffers.drain(..) { + device.destroy_framebuffer(fb, None); + } + for v in self.views.drain(..) { + device.destroy_image_view(v, None); + } + } + } + + fn destroy(&mut self, device: &ash::Device) { + self.destroy_targets(device); + unsafe { + device.destroy_pipeline(self.pipeline, None); + device.destroy_pipeline_layout(self.pipeline_layout, None); + device.destroy_descriptor_pool(self.desc_pool, None); + device.destroy_descriptor_set_layout(self.set_layout, None); + device.destroy_sampler(self.sampler, None); + device.destroy_render_pass(self.render_pass, None); + } + } +} + /// The one video image (device-local RGBA the size of the decoded stream) + its staging. /// `view`/`framebuffer` exist only on hw-capable devices (the CSC pass renders into it). struct VideoImage { @@ -67,6 +253,8 @@ pub struct Presenter { qfi: u32, /// Dmabuf import + CSC — `None` when the device lacks the import extensions. hw: Option, + /// The console-UI composite quad (§6.1's presenter half). + overlay_pipe: OverlayPipe, /// The submitted hw frame (plane images + decoder-surface guard): its GPU reads end /// with the in-flight fence, so it's destroyed right after the next fence wait. retired_hw: Option, @@ -175,6 +363,7 @@ impl Presenter { let format = pick_format(&surface_i, pdev, surface)?; let present_mode = pick_present_mode(&surface_i, pdev, surface)?; tracing::info!(?format, ?present_mode, "swapchain config"); + let overlay_pipe = OverlayPipe::new(&device, format.format)?; let cmd_pool = unsafe { device.create_command_pool( @@ -213,6 +402,7 @@ impl Presenter { queue, qfi, hw, + overlay_pipe, retired_hw: None, format, present_mode, @@ -285,6 +475,8 @@ impl Presenter { self.swapchain = swapchain; self.images = unsafe { self.swap_d.get_swapchain_images(swapchain) }?; self.extent = extent; + self.overlay_pipe + .rebuild_targets(&self.device, &self.images, self.format.format, extent)?; for s in self.render_sems.drain(..) { unsafe { self.device.destroy_semaphore(s, None) }; @@ -310,11 +502,36 @@ impl Presenter { self.hw.is_some() } + /// Quiesce the queue — the run loop calls this before dropping the overlay so + /// nothing in flight still references its images. + pub fn wait_idle(&self) { + unsafe { self.device.device_wait_idle() }.ok(); + } + + /// The device handles the console-UI overlay renders on (§6.1). Valid for the + /// presenter's lifetime; the run loop drops the overlay first. + pub fn shared_device(&self) -> SharedDevice { + SharedDevice { + entry: self.entry.clone(), + instance: self.instance.clone(), + physical_device: self.pdev, + device: self.device.clone(), + queue: self.queue, + queue_family_index: self.qfi, + } + } + /// Present one frame: route `input` into the video image (staging upload or dmabuf /// import + CSC pass; `Redraw` re-blits what's retained), clear, letterbox-blit, - /// present. Returns false when the swapchain was out of date — the caller recreates - /// (with current window state) and may retry. - pub fn present(&mut self, window: &sdl3::video::Window, input: FrameInput) -> Result { + /// blend the console-UI `overlay` quad if one arrived, present. Returns false when + /// the swapchain was out of date — the caller recreates (with current window state) + /// and may retry. + pub fn present( + &mut self, + window: &sdl3::video::Window, + input: FrameInput, + overlay: Option<&OverlayFrame>, + ) -> Result { if self.extent.width == 0 || self.extent.height == 0 { return Ok(true); // minimized — nothing to do } @@ -363,6 +580,18 @@ impl Presenter { let hw = self.hw.as_ref().unwrap(); hw.csc.bind_planes(&self.device, f.luma_view, f.chroma_view); } + if let Some(o) = overlay { + // Point the composite at this overlay image (same fence-wait safety). + let infos = [vk::DescriptorImageInfo::default() + .image_view(o.view) + .image_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL)]; + let writes = [vk::WriteDescriptorSet::default() + .dst_set(self.overlay_pipe.desc_set) + .dst_binding(0) + .descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) + .image_info(&infos)]; + unsafe { self.device.update_descriptor_sets(&writes, &[]) }; + } let (index, _suboptimal) = match unsafe { self.swap_d.acquire_next_image( @@ -535,13 +764,79 @@ impl Presenter { vk::Filter::LINEAR, ); } - barrier( - &self.device, - self.cmd_buf, - swap_image, - vk::ImageLayout::TRANSFER_DST_OPTIMAL, - vk::ImageLayout::PRESENT_SRC_KHR, - ); + if let Some(o) = overlay { + // Cross-submit visibility for the overlay image (Skia flushed it on this + // queue): same-layout barrier = execution + memory dependency only. + barrier( + &self.device, + self.cmd_buf, + o.image, + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + ); + barrier( + &self.device, + self.cmd_buf, + swap_image, + vk::ImageLayout::TRANSFER_DST_OPTIMAL, + vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL, + ); + // The composite pass blends the quad and ends the image PRESENT-ready. + self.device.cmd_begin_render_pass( + self.cmd_buf, + &vk::RenderPassBeginInfo::default() + .render_pass(self.overlay_pipe.render_pass) + .framebuffer(self.overlay_pipe.framebuffers[index as usize]) + .render_area(vk::Rect2D { + offset: vk::Offset2D { x: 0, y: 0 }, + extent: self.extent, + }), + vk::SubpassContents::INLINE, + ); + self.device.cmd_bind_pipeline( + self.cmd_buf, + vk::PipelineBindPoint::GRAPHICS, + self.overlay_pipe.pipeline, + ); + self.device.cmd_set_viewport( + self.cmd_buf, + 0, + &[vk::Viewport { + x: 0.0, + y: 0.0, + width: self.extent.width as f32, + height: self.extent.height as f32, + min_depth: 0.0, + max_depth: 1.0, + }], + ); + self.device.cmd_set_scissor( + self.cmd_buf, + 0, + &[vk::Rect2D { + offset: vk::Offset2D { x: 0, y: 0 }, + extent: self.extent, + }], + ); + self.device.cmd_bind_descriptor_sets( + self.cmd_buf, + vk::PipelineBindPoint::GRAPHICS, + self.overlay_pipe.pipeline_layout, + 0, + &[self.overlay_pipe.desc_set], + &[], + ); + self.device.cmd_draw(self.cmd_buf, 3, 1, 0, 0); + self.device.cmd_end_render_pass(self.cmd_buf); + } else { + barrier( + &self.device, + self.cmd_buf, + swap_image, + vk::ImageLayout::TRANSFER_DST_OPTIMAL, + vk::ImageLayout::PRESENT_SRC_KHR, + ); + } self.device.end_command_buffer(self.cmd_buf)?; let render_sem = self.render_sems[index as usize]; @@ -779,6 +1074,7 @@ impl Drop for Presenter { if let Some(hw) = self.hw.take() { hw.csc.destroy(&self.device); } + self.overlay_pipe.destroy(&self.device); for s in self.render_sems.drain(..) { self.device.destroy_semaphore(s, None); }