From 1efb8aef74c9bd2b403a3ba5bba88ea4ab63deed Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Wed, 22 Jul 2026 01:09:51 +0200 Subject: [PATCH] =?UTF-8?q?feat(core+host+client):=20cursor=20channel=20?= =?UTF-8?q?=E2=80=94=20remote-desktop=20sweep=20M2a+M2b?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host cursor stops riding the video and becomes a real OS cursor on the client (the Parsec/RDP model): pointer feel no longer pays the capture→encode→network→decode→present round trip. Wire (M2a): - Hello grows a client_caps trailing byte (CLIENT_CAP_CURSOR) after the fixed display_hdr block — presence disambiguated by remaining length, which caps the post-HDR tail at 27 bytes (documented); Welcome answers HOST_CAP_CURSOR (capable-and-asked, the 444/clipboard precedent). - CursorShape (0x50, control stream): serial + dims + hotspot + straight RGBA, ≤120px/side so the u16 frame always fits (128² would overshoot); client caches by serial — re-showing a known shape costs 14 bytes, not a bitmap (RDP pointer-cache for free). - CursorState (0xD0 datagram): serial + visible/relative_hint flags + position, sent once per encode-loop tick — latest-wins, self-healing under loss, no refresh timer. relative_hint is reserved for M3. - Client core: two new planes (control-task + datagram-task arms) → next_cursor_shape/next_cursor_state; connect() grows client_caps (C ABI passes 0 until the v11 cursor poll fns exist). Host (M2b, Linux portal only): - handshake::cursor_forward is THE predicate (client asked ∧ Linux ∧ compositor ≠ gamescope) — Welcome bit and session wiring both read it. - SessionPlan.cursor_blend goes false for a forwarding session; the encode loop ticks a CursorForwarder every iteration: shape-serial diff → control-task bridge (mirrors probe_result), state datagram → conn. - CursorOverlay/capture CursorState carry the hotspot through (nearest-neighbor downscale backstop for XL cursors, unit-tested). Presenter: - CursorChannel drains both planes per loop iteration; shapes become SDL color cursors (from_surface + hotspot), applied while the desktop mouse model is engaged; visibility follows the host; capture/released hands back the system cursor. Sessions advertise the cap when they START in desktop mode. Verified on .21: fmt + clippy -D warnings (7 crates) + tests green (core 218 incl. new wire roundtrips, host 245 incl. e2e + forwarder downscale tests). Co-Authored-By: Claude Fable 5 --- clients/session/src/main.rs | 5 + crates/pf-capture/src/linux/mod.rs | 8 + crates/pf-client-core/src/session.rs | 12 ++ crates/pf-frame/src/lib.rs | 5 + crates/pf-presenter/src/cursor.rs | 119 +++++++++++++++ crates/pf-presenter/src/lib.rs | 2 + crates/pf-presenter/src/run.rs | 16 ++ crates/punktfunk-core/src/abi.rs | 4 + crates/punktfunk-core/src/client/mod.rs | 51 ++++++- crates/punktfunk-core/src/client/planes.rs | 10 ++ crates/punktfunk-core/src/client/pump.rs | 4 + .../src/client/pump/control_task.rs | 8 + .../src/client/pump/datagram_task.rs | 9 ++ .../src/client/pump/handshake.rs | 4 + crates/punktfunk-core/src/client/worker.rs | 3 + crates/punktfunk-core/src/quic/caps.rs | 18 +++ crates/punktfunk-core/src/quic/control.rs | 115 ++++++++++++++ crates/punktfunk-core/src/quic/datagram.rs | 97 ++++++++++++ crates/punktfunk-core/src/quic/handshake.rs | 126 +++++++++++++-- crates/punktfunk-host/src/native.rs | 18 +++ crates/punktfunk-host/src/native/control.rs | 10 ++ .../punktfunk-host/src/native/cursor_fwd.rs | 144 ++++++++++++++++++ crates/punktfunk-host/src/native/handshake.rs | 24 +++ crates/punktfunk-host/src/native/stream.rs | 28 +++- include/punktfunk_core.h | 58 +++++++ 25 files changed, 885 insertions(+), 13 deletions(-) create mode 100644 crates/pf-presenter/src/cursor.rs create mode 100644 crates/punktfunk-host/src/native/cursor_fwd.rs diff --git a/clients/session/src/main.rs b/clients/session/src/main.rs index 3205e47a..043f1787 100644 --- a/clients/session/src/main.rs +++ b/clients/session/src/main.rs @@ -172,6 +172,11 @@ mod session_main { // defaults for Linux clients; `PUNKTFUNK_CLIENT_PEAK_NITS` (read in the session // pump) pins one manually. display_hdr: None, + // The presenter renders the host cursor locally in desktop mouse mode (M2 cursor + // channel); capture-mode sessions keep the composited cursor, so only advertise + // when the session STARTS in desktop mode. The host gates further (Linux portal + // compositors only). + cursor_forward: settings.mouse_mode() == trust::MouseMode::Desktop, mic_enabled: settings.mic_enabled, clipboard, // The Settings preference (auto → VAAPI where it exists; the presenter diff --git a/crates/pf-capture/src/linux/mod.rs b/crates/pf-capture/src/linux/mod.rs index fe161fba..25bf882b 100644 --- a/crates/pf-capture/src/linux/mod.rs +++ b/crates/pf-capture/src/linux/mod.rs @@ -868,6 +868,10 @@ mod pipewire { /// Bumps whenever the bitmap (`rgba`/`bw`/`bh`) changes — stable across position-only moves, /// so the GPU encoder re-uploads its cursor texture only on change. serial: u64, + /// The compositor-reported hotspot — carried on the overlay for the cursor-forward + /// channel (the blend path uses the pre-adjusted `x`/`y` and never reads it). + hot_x: i32, + hot_y: i32, } impl CursorState { @@ -884,6 +888,8 @@ mod pipewire { h: self.bh, rgba: self.rgba.clone(), serial: self.serial, + hot_x: self.hot_x.max(0) as u32, + hot_y: self.hot_y.max(0) as u32, }) } } @@ -1380,6 +1386,8 @@ mod pipewire { cursor.visible = true; cursor.x = pos_x - hot_x; cursor.y = pos_y - hot_y; + cursor.hot_x = hot_x; + cursor.hot_y = hot_y; if bmp_off == 0 { // Position-only update — keep the cached bitmap. return; diff --git a/crates/pf-client-core/src/session.rs b/crates/pf-client-core/src/session.rs index 5b07a794..48d3e684 100644 --- a/crates/pf-client-core/src/session.rs +++ b/crates/pf-client-core/src/session.rs @@ -44,6 +44,13 @@ pub struct SessionParams { /// Share the clipboard with this host (the per-host `KnownHost::clipboard_sync`). The /// bridge additionally needs the host to advertise `HOST_CAP_CLIPBOARD`. pub clipboard: bool, + /// Advertise `quic::CLIENT_CAP_CURSOR`: this embedder renders the host cursor locally + /// (the presenter's cursor channel, design/remote-desktop-sweep.md M2), so the host may + /// stop compositing the pointer into the video. Only set when the embedder actually + /// draws it (the SDL presenter in desktop mouse mode) — a session that advertises it + /// without rendering streams with NO visible cursor. The host answers `HOST_CAP_CURSOR` + /// when its capture can forward (Linux portal, not gamescope/Windows). + pub cursor_forward: bool, /// Video decoder preference (Settings; `PUNKTFUNK_DECODER` overrides — see /// `video::Decoder::new`). pub decoder: String, @@ -255,6 +262,11 @@ fn pump( // This display's HDR volume → the host's virtual-display EDID. The env hatch wins so an // A/B run can pin an exact peak (PUNKTFUNK_CLIENT_PEAK_NITS=600). punktfunk_core::client::display_hdr_env_override().or(params.display_hdr), + if params.cursor_forward { + punktfunk_core::quic::CLIENT_CAP_CURSOR + } else { + 0 + }, params.launch.clone(), params.pin, Some(params.identity), diff --git a/crates/pf-frame/src/lib.rs b/crates/pf-frame/src/lib.rs index 682caa27..5f82fa11 100644 --- a/crates/pf-frame/src/lib.rs +++ b/crates/pf-frame/src/lib.rs @@ -196,6 +196,11 @@ pub struct CursorOverlay { pub rgba: std::sync::Arc>, /// Bumps whenever `rgba`/`w`/`h` change; stable across position-only moves. pub serial: u64, + /// Hotspot (the pixel that IS the pointer position) within `w`×`h`. The blend paths ignore + /// it (`x`/`y` are already hotspot-adjusted); the cursor-forward channel ships it to the + /// client so a locally-drawn OS cursor points with the right pixel. + pub hot_x: u32, + pub hot_y: u32, } /// A captured frame. [`format`](Self::format)/dimensions describe the pixels regardless of diff --git a/crates/pf-presenter/src/cursor.rs b/crates/pf-presenter/src/cursor.rs new file mode 100644 index 00000000..a2414e6d --- /dev/null +++ b/crates/pf-presenter/src/cursor.rs @@ -0,0 +1,119 @@ +//! Client-side cursor rendering (design/remote-desktop-sweep.md M2): the host forwards the +//! pointer's SHAPE (reliable control stream, cached by serial) and per-frame STATE (lossy +//! `0xD0` — position/visibility), and WE draw it as a real OS cursor — pointer feel stops +//! paying the video round-trip (the Parsec/RDP model). Active only when the session +//! negotiated it (`HOST_CAP_CURSOR` in the Welcome — the host stopped compositing then) and +//! only applied while the DESKTOP mouse model is engaged: under capture the pointer is +//! relative-locked (SDL hides it) and games draw their own cursor in-frame. + +use punktfunk_core::client::NativeClient; +use punktfunk_core::quic::{CursorState, HOST_CAP_CURSOR}; +use sdl3::mouse::{Cursor, MouseUtil, SystemCursor}; +use std::collections::HashMap; +use std::time::Duration; + +/// Shape serials cached at most — cursors cycle through a handful of shapes (arrow, I-beam, +/// resize…); a runaway host can't grow the map past this (the cache resets, shapes re-arrive +/// on the reliable stream via the serial-miss path). +const SHAPE_CACHE_MAX: usize = 64; + +pub struct CursorChannel { + /// The Welcome carried `HOST_CAP_CURSOR` — the host forwards instead of compositing. + negotiated: bool, + /// Serial → built OS cursor. An SDL `Cursor` must outlive its `set()`, so the cache owns + /// every shape ever applied this session (bounded by [`SHAPE_CACHE_MAX`]). + shapes: HashMap, + /// The serial currently installed via `Cursor::set` (`None` = default/system cursor). + installed: Option, + /// Latest `0xD0` state (latest-wins across a drained batch). + state: Option, +} + +impl CursorChannel { + pub fn new(connector: &NativeClient) -> CursorChannel { + let negotiated = connector.host_caps() & HOST_CAP_CURSOR != 0; + if negotiated { + tracing::info!("cursor channel negotiated — host cursor renders locally"); + } + CursorChannel { + negotiated, + shapes: HashMap::new(), + installed: None, + state: None, + } + } + + /// Whether the host forwards the cursor this session (it no longer composites one). + pub fn negotiated(&self) -> bool { + self.negotiated + } + + /// Drain the two planes and apply the newest state — once per run-loop iteration. + /// `desktop_active` = the desktop mouse model is engaged (captured + desktop): only then + /// do we own the local cursor's shape/visibility; under capture SDL's relative mode owns + /// it, and released the system cursor must look normal. + pub fn pump(&mut self, connector: &NativeClient, mouse: &MouseUtil, desktop_active: bool) { + if !self.negotiated { + return; + } + while let Ok(shape) = connector.next_cursor_shape(Duration::ZERO) { + if self.shapes.len() >= SHAPE_CACHE_MAX { + // Degenerate host: reset — live shapes re-install via the serial-miss path. + self.shapes.clear(); + self.installed = None; + } + let mut data = shape.rgba; + let built = sdl3::surface::Surface::from_data( + &mut data, + shape.w as u32, + shape.h as u32, + shape.w as u32 * 4, + sdl3::pixels::PixelFormat::RGBA32, + ) + .map_err(|e| e.to_string()) + .and_then(|surf| { + Cursor::from_surface(&surf, shape.hot_x as i32, shape.hot_y as i32) + .map_err(|e| e.to_string()) + }); + match built { + Ok(cursor) => { + // A re-sent serial replaces its entry; force re-install if it's current. + if self.installed == Some(shape.serial) { + self.installed = None; + } + self.shapes.insert(shape.serial, cursor); + } + Err(e) => tracing::warn!(error = %e, w = shape.w, h = shape.h, + "cursor shape rejected by SDL — keeping the previous cursor"), + } + } + while let Ok(st) = connector.next_cursor_state(Duration::ZERO) { + self.state = Some(st); // latest wins + } + + if !desktop_active { + // Capture mode / released: hand the cursor back to the system default so a + // released pointer over the window doesn't wear the host's shape. + if self.installed.take().is_some() { + Cursor::from_system(SystemCursor::Arrow) + .map(|c| c.set()) + .ok(); + } + return; + } + let Some(st) = self.state else { return }; + if st.visible() && self.installed != Some(st.serial) { + if let Some(cursor) = self.shapes.get(&st.serial) { + cursor.set(); + self.installed = Some(st.serial); + } + // Serial miss: the (reliable) shape hasn't landed yet — keep the previous + // cursor for the RTT rather than flashing default. + } + // Visibility follows the host (a host app hid its pointer ⇒ ours hides too). Queried, + // not shadowed, so apply_capture's own show/hide calls can never desync us. + if mouse.is_cursor_showing() != st.visible() { + mouse.show_cursor(st.visible()); + } + } +} diff --git a/crates/pf-presenter/src/lib.rs b/crates/pf-presenter/src/lib.rs index 386bbb10..6737de42 100644 --- a/crates/pf-presenter/src/lib.rs +++ b/crates/pf-presenter/src/lib.rs @@ -16,6 +16,8 @@ #[cfg(any(target_os = "linux", windows))] pub mod csc; +#[cfg(any(target_os = "linux", windows))] +pub mod cursor; #[cfg(windows)] pub mod d3d11; #[cfg(target_os = "linux")] diff --git a/crates/pf-presenter/src/run.rs b/crates/pf-presenter/src/run.rs index 9f197618..d49cec78 100644 --- a/crates/pf-presenter/src/run.rs +++ b/crates/pf-presenter/src/run.rs @@ -233,6 +233,9 @@ struct StreamState { /// window-normalized position must be re-based onto the content rect). `None` until /// the first frame; touches before then have nothing to map onto and are dropped. last_video: Option<(u32, u32)>, + /// Client-side cursor rendering (M2 cursor channel) — created with the connector; inert + /// when the host didn't negotiate the channel. + cursor_chan: Option, } impl StreamState { @@ -263,6 +266,7 @@ impl StreamState { frames: wake_rx, connector: None, capture: None, + cursor_chan: None, force_software, canceled: false, ready_announced: false, @@ -744,6 +748,17 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result if let Some(cap) = stream.as_mut().and_then(|s| s.capture.as_mut()) { cap.flush_motion(); } + // Cursor channel (M2): drain forwarded shape/state and drive the local OS cursor — + // only meaningful in the desktop mouse model (capture's relative lock hides it). + if let Some(st) = stream.as_mut() { + if let (Some(chan), Some(c)) = (st.cursor_chan.as_mut(), st.connector.as_ref()) { + let desktop_active = st + .capture + .as_ref() + .is_some_and(|cap| cap.captured() && cap.desktop()); + chan.pump(c, &mouse, desktop_active); + } + } // Text input follows the overlay's editing state (edge-triggered). let want_text = overlay.as_ref().is_some_and(|o| o.text_input_active()); @@ -888,6 +903,7 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result cap.engage(); // capture engages when the stream starts (ui_stream parity) apply_capture(&mut window, &mouse, true, cap.desktop()); st.capture = Some(cap); + st.cursor_chan = Some(crate::cursor::CursorChannel::new(&c)); st.connector = Some(c); if let Some(f) = opts.on_connected.as_mut() { f(fingerprint); diff --git a/crates/punktfunk-core/src/abi.rs b/crates/punktfunk-core/src/abi.rs index 7744a5ad..8cd685c1 100644 --- a/crates/punktfunk-core/src/abi.rs +++ b/crates/punktfunk-core/src/abi.rs @@ -1574,6 +1574,10 @@ unsafe fn connect_ex_impl( // themselves (EDR / MediaCodec), so the host's EDID defaults are fine there. An `ex8` // variant can carry it if a passthrough embedder ever needs it. None, + // No client_caps in the C ABI yet either: cursor-channel opt-in for Apple/Android + // arrives with the ABI v11 cursor poll fns — until an embedder can RENDER the + // forwarded cursor it must not ask the host to stop compositing it. + 0, launch, pin, identity, diff --git a/crates/punktfunk-core/src/client/mod.rs b/crates/punktfunk-core/src/client/mod.rs index 98798c22..579146cf 100644 --- a/crates/punktfunk-core/src/client/mod.rs +++ b/crates/punktfunk-core/src/client/mod.rs @@ -42,8 +42,8 @@ pub use self::rumble::{ActuatorQuirks, RumbleCommand}; use self::control::{CtrlRequest, Negotiated}; use self::frame_channel::{DecodeLatAcc, FrameChannel, FramePop}; use self::planes::{ - RumbleUpdate, AUDIO_QUEUE, CLIP_EVENT_QUEUE, HDR_META_QUEUE, HIDOUT_QUEUE, HOST_TIMING_QUEUE, - RUMBLE_QUEUE, + RumbleUpdate, AUDIO_QUEUE, CLIP_EVENT_QUEUE, CURSOR_SHAPE_QUEUE, CURSOR_STATE_QUEUE, + HDR_META_QUEUE, HIDOUT_QUEUE, HOST_TIMING_QUEUE, RUMBLE_QUEUE, }; use self::probe::ProbeState; use self::pump::run_pump; @@ -93,6 +93,12 @@ pub struct NativeClient { /// Inbound per-AU host capture→send timings — 0xCF datagrams (the client always advertises /// [`quic::VIDEO_CAP_HOST_TIMING`]; an older host simply never sends any). host_timing: Mutex>, + /// Inbound cursor shapes (control-stream [`crate::quic::CursorShape`]) — only a session + /// that advertised [`quic::CLIENT_CAP_CURSOR`] against a [`quic::HOST_CAP_CURSOR`] host + /// ever receives any. + cursor_shape: Mutex>, + /// Inbound per-frame cursor state — `0xD0` datagrams (same negotiation gate as shapes). + cursor_state: Mutex>, input_tx: tokio::sync::mpsc::UnboundedSender, /// Outbound mic frames `(seq, pts_ns, opus)` → encoded as 0xCB datagrams by the worker. /// Bounded ([`MIC_QUEUE`]): a wedged worker drops fresh frames (logged) instead of queueing @@ -316,6 +322,12 @@ impl NativeClient { // display's EDID so host apps tone-map to the client's real panel; `None` = unknown/SDR // (the host keeps its built-in EDID defaults). See [`crate::quic::Hello::display_hdr`]. display_hdr: Option, + // Non-video client capabilities ([`crate::quic::Hello::client_caps`]) — set + // [`crate::quic::CLIENT_CAP_CURSOR`] ONLY if this embedder actually renders the host + // cursor locally (shape + state planes): the host stops compositing the pointer into + // the video for a session that advertises it, so a non-rendering embedder that sets it + // streams with NO visible cursor at all. `0` = today's composited behavior. + client_caps: u8, launch: Option, pin: Option<[u8; 32]>, identity: Option<(String, String)>, @@ -337,6 +349,10 @@ impl NativeClient { let (clip_event_tx, clip_event_rx) = std::sync::mpsc::sync_channel::(CLIP_EVENT_QUEUE); let (clip_cmd_tx, clip_cmd_rx) = tokio::sync::mpsc::unbounded_channel::(); + let (cursor_shape_tx, cursor_shape_rx) = + std::sync::mpsc::sync_channel::(CURSOR_SHAPE_QUEUE); + let (cursor_state_tx, cursor_state_rx) = + std::sync::mpsc::sync_channel::(CURSOR_STATE_QUEUE); let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); let shutdown = Arc::new(AtomicBool::new(false)); let quit = Arc::new(AtomicBool::new(false)); @@ -390,6 +406,7 @@ impl NativeClient { video_codecs, preferred_codec, display_hdr, + client_caps, launch, pin, identity, @@ -401,6 +418,8 @@ impl NativeClient { hidout_tx, hdr_meta_tx, host_timing_tx, + cursor_shape_tx, + cursor_state_tx, input_rx, mic_rx, rich_input_rx, @@ -445,6 +464,8 @@ impl NativeClient { hidout: Mutex::new(hidout_rx), hdr_meta: Mutex::new(hdr_meta_rx), host_timing: Mutex::new(host_timing_rx), + cursor_shape: Mutex::new(cursor_shape_rx), + cursor_state: Mutex::new(cursor_state_rx), input_tx, mic_tx, rich_input_tx, @@ -892,6 +913,32 @@ impl NativeClient { } } + /// Pull the next host cursor shape (design/remote-desktop-sweep.md M2): RGBA bitmap + + /// hotspot, sent on pointer-bitmap change over the reliable control stream. The embedder + /// caches by `serial` and builds an OS cursor from it; [`NativeClient::next_cursor_state`] + /// references shapes by serial. Only a session that advertised + /// [`crate::quic::CLIENT_CAP_CURSOR`] against a capable host receives any. Same + /// timeout/closed semantics as [`NativeClient::next_hidout`]. + pub fn next_cursor_shape(&self, timeout: Duration) -> Result { + match self.cursor_shape.lock().unwrap().recv_timeout(timeout) { + Ok(s) => Ok(s), + Err(RecvTimeoutError::Timeout) => Err(PunktfunkError::NoFrame), + Err(RecvTimeoutError::Disconnected) => Err(PunktfunkError::Closed), + } + } + + /// Pull the next per-frame cursor state (`0xD0`): position, visibility and the M3 + /// relative-mode hint, referencing a shape by serial. Latest-wins — an embedder should + /// drain the queue and apply only the newest. Same negotiation gate and timeout/closed + /// semantics as [`NativeClient::next_cursor_shape`]. + pub fn next_cursor_state(&self, timeout: Duration) -> Result { + match self.cursor_state.lock().unwrap().recv_timeout(timeout) { + Ok(s) => Ok(s), + Err(RecvTimeoutError::Timeout) => Err(PunktfunkError::NoFrame), + Err(RecvTimeoutError::Disconnected) => Err(PunktfunkError::Closed), + } + } + /// Pull the next per-AU host timing (0xCF): the host's capture→sent duration for one access /// unit, correlated to the AU by `pts_ns`. Feeds the unified stats HUD's `host` / `network` /// split (`network = (received + clock_offset − pts) − host_us`); a stats consumer should diff --git a/crates/punktfunk-core/src/client/planes.rs b/crates/punktfunk-core/src/client/planes.rs index 7c0b6e6e..0adccbbc 100644 --- a/crates/punktfunk-core/src/client/planes.rs +++ b/crates/punktfunk-core/src/client/planes.rs @@ -35,6 +35,16 @@ pub(crate) const HOST_TIMING_QUEUE: usize = 512; /// a dropped fetch-request makes the serving stream time out and reset cleanly. pub(crate) const CLIP_EVENT_QUEUE: usize = 32; +/// Cursor-shape plane depth (control-stream [`crate::quic::CursorShape`], one per pointer-bitmap +/// change — human-paced). Overflow drops the newest (try_send); the next shape change or a +/// serial mismatch against `0xD0` state heals it visually within a shape-change period. +pub(crate) const CURSOR_SHAPE_QUEUE: usize = 8; + +/// Cursor-state plane depth (`0xD0`, one datagram per captured frame). Latest-wins state — the +/// embedder drains per present; a tiny ring only bridges scheduling jitter. Overflow drops the +/// newest (try_send), healed by the very next frame's datagram. +pub(crate) const CURSOR_STATE_QUEUE: usize = 8; + /// One Opus packet from the host's audio datagram stream (48 kHz stereo, 5 ms frames). #[derive(Clone, Debug)] pub struct AudioPacket { diff --git a/crates/punktfunk-core/src/client/pump.rs b/crates/punktfunk-core/src/client/pump.rs index f379dfbf..a84f2bd9 100644 --- a/crates/punktfunk-core/src/client/pump.rs +++ b/crates/punktfunk-core/src/client/pump.rs @@ -51,6 +51,8 @@ pub(super) async fn run_pump(args: WorkerArgs) { hidout_tx, hdr_meta_tx, host_timing_tx, + cursor_shape_tx, + cursor_state_tx, input_rx, mut mic_rx, mut rich_input_rx, @@ -123,6 +125,7 @@ pub(super) async fn run_pump(args: WorkerArgs) { clock_offset: clock_offset.clone(), clock_gen: clock_gen.clone(), clip_event_tx: clip_event_tx.clone(), + cursor_shape_tx, } .run(), ); @@ -136,6 +139,7 @@ pub(super) async fn run_pump(args: WorkerArgs) { hidout_tx, hdr_meta_tx, host_timing_tx, + cursor_state_tx, )); // Clipboard task: the fetch-stream accept loop (host pulls what we offered) + outbound fetches diff --git a/crates/punktfunk-core/src/client/pump/control_task.rs b/crates/punktfunk-core/src/client/pump/control_task.rs index 9b217d0a..929c90a1 100644 --- a/crates/punktfunk-core/src/client/pump/control_task.rs +++ b/crates/punktfunk-core/src/client/pump/control_task.rs @@ -21,6 +21,9 @@ pub(super) struct ControlTask { /// Clipboard metadata events (ClipState/ClipOffer) feed the same event plane the /// clipboard task uses for fetch data. pub(super) clip_event_tx: std::sync::mpsc::SyncSender, + /// Host cursor shapes ([`CursorShape`], sent on pointer-bitmap change) → the embedder's + /// shape plane ([`NativeClient::next_cursor_shape`]). + pub(super) cursor_shape_tx: std::sync::mpsc::SyncSender, } impl ControlTask { @@ -36,6 +39,7 @@ impl ControlTask { clock_offset, clock_gen, clip_event_tx, + cursor_shape_tx, } = self; // Mid-stream clock re-sync (see [`ClockResync`]): a batch runs every // CLOCK_RESYNC_INTERVAL and whenever the pump asks (CtrlRequest::ClockResync after @@ -167,6 +171,10 @@ impl ControlTask { seq: offer.seq, kinds: offer.kinds, }); + } else if let Ok(shape) = crate::quic::CursorShape::decode(&msg) { + // Pointer bitmap changed (cursor channel, only when negotiated). try_send: + // an overflowing ring drops the newest shape — the next change resends. + let _ = cursor_shape_tx.try_send(shape); } else { tracing::warn!( tag = ?msg.first(), diff --git a/crates/punktfunk-core/src/client/pump/datagram_task.rs b/crates/punktfunk-core/src/client/pump/datagram_task.rs index fafeca33..904f2144 100644 --- a/crates/punktfunk-core/src/client/pump/datagram_task.rs +++ b/crates/punktfunk-core/src/client/pump/datagram_task.rs @@ -3,6 +3,9 @@ use super::*; +// One parameter per demuxed plane — grouping them into a struct would just move the field +// list one hop away from the single call site. +#[allow(clippy::too_many_arguments)] pub(super) async fn run( conn: quinn::Connection, audio_tx: std::sync::mpsc::SyncSender, @@ -11,6 +14,7 @@ pub(super) async fn run( hidout_tx: std::sync::mpsc::SyncSender, hdr_meta_tx: std::sync::mpsc::SyncSender, host_timing_tx: std::sync::mpsc::SyncSender, + cursor_state_tx: std::sync::mpsc::SyncSender, ) { // Per-pad reorder gate for v2 rumble envelopes (the seq analog of the host's gamepad-state // gate): a datagram the network reordered must not roll a stopped motor back on. Legacy v1 @@ -73,6 +77,11 @@ pub(super) async fn run( let _ = host_timing_tx.try_send(t); } } + Some(&crate::quic::CURSOR_STATE_MAGIC) => { + if let Some(s) = crate::quic::decode_cursor_state_datagram(&d) { + let _ = cursor_state_tx.try_send(s); + } + } _ => {} // unknown tag — a newer host; ignore } } diff --git a/crates/punktfunk-core/src/client/pump/handshake.rs b/crates/punktfunk-core/src/client/pump/handshake.rs index b3d7a6e7..88ab8d70 100644 --- a/crates/punktfunk-core/src/client/pump/handshake.rs +++ b/crates/punktfunk-core/src/client/pump/handshake.rs @@ -143,6 +143,10 @@ pub(super) async fn connect_and_handshake(args: &WorkerArgs) -> Result, + pub(crate) client_caps: u8, pub(crate) launch: Option, pub(crate) pin: Option<[u8; 32]>, pub(crate) identity: Option<(String, String)>, @@ -38,6 +39,8 @@ pub(crate) struct WorkerArgs { pub(crate) hidout_tx: SyncSender, pub(crate) hdr_meta_tx: SyncSender, pub(crate) host_timing_tx: SyncSender, + pub(crate) cursor_shape_tx: SyncSender, + pub(crate) cursor_state_tx: SyncSender, pub(crate) input_rx: tokio::sync::mpsc::UnboundedReceiver, pub(crate) mic_rx: tokio::sync::mpsc::Receiver<(u32, u64, Vec)>, pub(crate) rich_input_rx: tokio::sync::mpsc::UnboundedReceiver, diff --git a/crates/punktfunk-core/src/quic/caps.rs b/crates/punktfunk-core/src/quic/caps.rs index 034ba335..c8f6e64f 100644 --- a/crates/punktfunk-core/src/quic/caps.rs +++ b/crates/punktfunk-core/src/quic/caps.rs @@ -71,6 +71,24 @@ pub const HOST_CAP_GAMEPAD_STATE: u8 = 0x01; /// trailing `host_caps` byte — no wire-layout change. pub const HOST_CAP_CLIPBOARD: u8 = 0x02; +/// [`Hello::client_caps`] bit: the client renders the host cursor LOCALLY +/// (design/remote-desktop-sweep.md M2). It consumes [`CursorShape`](super::control::CursorShape) +/// control messages (RGBA bitmap + hotspot, cached by serial) and per-frame +/// [`CursorState`](super::datagram::CursorState) `0xD0` datagrams (position/visibility), and +/// draws the pointer itself — so the host must STOP compositing the cursor into the video +/// (`SessionPlan.cursor_blend = false`) or the user sees it twice. Active only when the host +/// answers with [`HOST_CAP_CURSOR`] (capable-and-agreed, the 444/clipboard precedent); toward +/// an older or incapable host nothing changes. +pub const CLIENT_CAP_CURSOR: u8 = 0x01; + +/// [`Welcome::host_caps`] bit: the host CAN forward the cursor out-of-band (it captures cursor +/// metadata separately from the frame — the Linux portal `SPA_META_Cursor` path; NOT gamescope, +/// whose capture carries no cursor, and NOT Windows yet, where DWM composites into the IDD +/// frame). Set only when the client asked via [`CLIENT_CAP_CURSOR`]; when both bits agree the +/// host stops blending and ships [`CursorShape`](super::control::CursorShape) + +/// [`CursorState`](super::datagram::CursorState) instead. +pub const HOST_CAP_CURSOR: u8 = 0x04; + /// [`Hello::video_codecs`] bit: the client can decode H.264 / AVC. The GPU-less **software** /// encode path (openh264) emits H.264, so a client that wants to stream from a software host MUST /// advertise this. diff --git a/crates/punktfunk-core/src/quic/control.rs b/crates/punktfunk-core/src/quic/control.rs index 9e32b42a..47b44b51 100644 --- a/crates/punktfunk-core/src/quic/control.rs +++ b/crates/punktfunk-core/src/quic/control.rs @@ -783,6 +783,83 @@ impl ClipFetchHdr { } } +// --- Cursor channel (design/remote-desktop-sweep.md M2) -------------------------------------- +// The host cursor, forwarded out-of-band so the CLIENT draws it as a real OS cursor (the +// Parsec/RDP model) instead of paying the video round-trip. Shape (rare, needs reliability) +// rides here on the control stream; per-frame position/visibility rides the lossy `0xD0` +// datagram plane ([`super::datagram::CursorState`]). Active only when the client's +// [`CLIENT_CAP_CURSOR`](super::caps::CLIENT_CAP_CURSOR) met the host's +// [`HOST_CAP_CURSOR`](super::caps::HOST_CAP_CURSOR) — the host stops compositing then. +// --------------------------------------------------------------------------------------------- + +/// Type byte of [`CursorShape`] (host → client): the pointer's bitmap + hotspot changed. +pub const MSG_CURSOR_SHAPE: u8 = 0x50; + +/// Per-side pixel cap for a forwarded cursor bitmap. The control-stream frame is length-prefixed +/// with a `u16`, so a whole message must fit 65535 bytes — 128×128 RGBA (65536 B) already +/// overshoots before the 17-byte header. 120² (57.6 KiB + header) fits with headroom and covers +/// real cursors (typically ≤ 64 px, ≤ 96 px at HiDPI scale); the HOST downscales anything +/// larger before forwarding, so the cap is invisible to clients. +pub const CURSOR_SHAPE_MAX_SIDE: u16 = 120; + +/// `host → client` ([`MSG_CURSOR_SHAPE`]): one cursor shape, sent when the pointer's bitmap +/// changes (never per-frame — [`super::datagram::CursorState`] carries the motion). The client +/// caches shapes by `serial` and re-installs a cached one without any bitmap crossing again +/// (the RDP pointer-cache idea for free: re-showing a known serial is a 14-byte +/// [`super::datagram::CursorState`], not a resend). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CursorShape { + /// Bitmap identity — bumped by the host's capture layer only on shape change; position + /// moves keep the serial stable. [`super::datagram::CursorState::serial`] references it. + pub serial: u32, + /// Bitmap dimensions in pixels, `1..=`[`CURSOR_SHAPE_MAX_SIDE`] each. + pub w: u16, + pub h: u16, + /// Hotspot (the pixel that IS the pointer position), within `w`×`h`. + pub hot_x: u16, + pub hot_y: u16, + /// Straight-alpha RGBA8, exactly `w * h * 4` bytes, no padding. + pub rgba: Vec, +} + +impl CursorShape { + pub fn encode(&self) -> Vec { + // magic[0..4] type[4] serial[5..9] w[9..11] h[11..13] hot_x[13..15] hot_y[15..17] rgba… + let mut b = Vec::with_capacity(17 + self.rgba.len()); + b.extend_from_slice(CTL_MAGIC); + b.push(MSG_CURSOR_SHAPE); + b.extend_from_slice(&self.serial.to_le_bytes()); + b.extend_from_slice(&self.w.to_le_bytes()); + b.extend_from_slice(&self.h.to_le_bytes()); + b.extend_from_slice(&self.hot_x.to_le_bytes()); + b.extend_from_slice(&self.hot_y.to_le_bytes()); + b.extend_from_slice(&self.rgba); + b + } + + pub fn decode(b: &[u8]) -> Result { + if b.len() < 17 || &b[0..4] != CTL_MAGIC || b[4] != MSG_CURSOR_SHAPE { + return Err(PunktfunkError::InvalidArg("bad CursorShape")); + } + let u16at = |o: usize| u16::from_le_bytes([b[o], b[o + 1]]); + let (w, h) = (u16at(9), u16at(11)); + if w == 0 || h == 0 || w > CURSOR_SHAPE_MAX_SIDE || h > CURSOR_SHAPE_MAX_SIDE { + return Err(PunktfunkError::InvalidArg("bad CursorShape dims")); + } + if b.len() != 17 + (w as usize) * (h as usize) * 4 { + return Err(PunktfunkError::InvalidArg("bad CursorShape len")); + } + Ok(CursorShape { + serial: u32::from_le_bytes(b[5..9].try_into().unwrap()), + w, + h, + hot_x: u16at(13), + hot_y: u16at(15), + rgba: b[17..].to_vec(), + }) + } +} + #[cfg(test)] mod tests { use crate::config::Mode; @@ -1147,4 +1224,42 @@ mod tests { assert!(ClipFetchHdr::decode(&[bytes.as_slice(), &[0]].concat()).is_err()); assert!(ClipFetchHdr::decode(&bytes[..bytes.len() - 1]).is_err()); } + #[test] + fn cursor_shape_roundtrip() { + let s = CursorShape { + serial: 7, + w: 2, + h: 3, + hot_x: 1, + hot_y: 2, + rgba: (0..2 * 3 * 4).map(|i| i as u8).collect(), + }; + assert_eq!(CursorShape::decode(&s.encode()).unwrap(), s); + // Max-side shape still fits the u16 control frame with headroom. + let side = CURSOR_SHAPE_MAX_SIDE; + let big = CursorShape { + serial: u32::MAX, + w: side, + h: side, + hot_x: side - 1, + hot_y: 0, + rgba: vec![0xAB; side as usize * side as usize * 4], + }; + let bytes = big.encode(); + assert!(bytes.len() <= u16::MAX as usize, "must fit a control frame"); + assert_eq!(CursorShape::decode(&bytes).unwrap(), big); + // Rejections: zero / oversize dims, and a length that disagrees with them. + let mut zero = s.encode(); + zero[9] = 0; + zero[10] = 0; + assert!(CursorShape::decode(&zero).is_err()); + let mut oversize = s.encode(); + oversize[9..11].copy_from_slice(&(CURSOR_SHAPE_MAX_SIDE + 1).to_le_bytes()); + assert!(CursorShape::decode(&oversize).is_err()); + let mut short = s.encode(); + short.pop(); + assert!(CursorShape::decode(&short).is_err()); + // Distinct from the neighboring vocabulary. + assert!(ClipState::decode(&s.encode()).is_err()); + } } diff --git a/crates/punktfunk-core/src/quic/datagram.rs b/crates/punktfunk-core/src/quic/datagram.rs index 0f2ed4ff..b8cd283e 100644 --- a/crates/punktfunk-core/src/quic/datagram.rs +++ b/crates/punktfunk-core/src/quic/datagram.rs @@ -606,6 +606,75 @@ pub fn decode_host_timing_datagram(b: &[u8]) -> Option { }) } +/// Cursor-state datagram tag, host → client (design/remote-desktop-sweep.md M2). Next tag after +/// [`HOST_TIMING_MAGIC`]. Sent once per captured frame while the cursor channel is negotiated +/// ([`CLIENT_CAP_CURSOR`](super::caps::CLIENT_CAP_CURSOR) ∧ +/// [`HOST_CAP_CURSOR`](super::caps::HOST_CAP_CURSOR)) — per-frame resend makes the plane +/// self-healing under loss (latest-wins, no refresh timer). The bitmap itself rides the +/// reliable control stream ([`CursorShape`](super::control::CursorShape)); this 14-byte +/// datagram only moves/hides the pointer. +pub const CURSOR_STATE_MAGIC: u8 = 0xD0; + +/// [`CursorState::flags`] bit: the host cursor is visible. +pub const CURSOR_VISIBLE: u8 = 0x01; +/// [`CursorState::flags`] bit: a host app captured/hid the pointer — the client SHOULD run +/// relative/captured (M3 auto-flip; advisory, user override always wins). +pub const CURSOR_RELATIVE_HINT: u8 = 0x02; + +/// Per-frame host-cursor state (position, visibility, mode hint). `x`/`y` are the pointer +/// position (hotspot point, not bitmap top-left) in the host OUTPUT's pixel space — the same +/// space the video mode describes, so the client maps through its letterbox exactly like it +/// maps touches, in reverse. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CursorState { + /// The [`CursorShape`](super::control::CursorShape) serial this state refers to. A client + /// that has no cached shape for it keeps its previous cursor until the (reliable) shape + /// message lands — at worst one control-stream RTT of stale shape, never a wrong position. + pub serial: u32, + /// Bitfield of [`CURSOR_VISIBLE`] / [`CURSOR_RELATIVE_HINT`]. + pub flags: u8, + pub x: i32, + pub y: i32, +} + +impl CursorState { + pub fn visible(&self) -> bool { + self.flags & CURSOR_VISIBLE != 0 + } + pub fn relative_hint(&self) -> bool { + self.flags & CURSOR_RELATIVE_HINT != 0 + } +} + +/// Wire length of a [`CURSOR_STATE_MAGIC`] datagram: tag + u32 serial + flags + 2 × i32 = 14. +const CURSOR_STATE_LEN: usize = 1 + 4 + 1 + 8; + +/// Encode a [`CursorState`] into a [`CURSOR_STATE_MAGIC`] datagram. +pub fn encode_cursor_state_datagram(s: &CursorState) -> Vec { + let mut b = Vec::with_capacity(CURSOR_STATE_LEN); + b.push(CURSOR_STATE_MAGIC); + b.extend_from_slice(&s.serial.to_le_bytes()); + b.push(s.flags); + b.extend_from_slice(&s.x.to_le_bytes()); + b.extend_from_slice(&s.y.to_le_bytes()); + b +} + +/// Parse a [`CURSOR_STATE_MAGIC`] datagram → [`CursorState`]. `None` on bad tag or a short +/// buffer (the fixed length bounds every read before it happens; a longer buffer is tolerated +/// for append-extension, like 0xCF). +pub fn decode_cursor_state_datagram(b: &[u8]) -> Option { + if b.len() < CURSOR_STATE_LEN || b[0] != CURSOR_STATE_MAGIC { + return None; + } + Some(CursorState { + serial: u32::from_le_bytes(b[1..5].try_into().unwrap()), + flags: b[5], + x: i32::from_le_bytes(b[6..10].try_into().unwrap()), + y: i32::from_le_bytes(b[10..14].try_into().unwrap()), + }) +} + #[cfg(test)] mod tests { use crate::quic::*; @@ -922,4 +991,32 @@ mod tests { ) .is_none()); } + #[test] + fn cursor_state_roundtrip() { + for (flags, x, y) in [ + (CURSOR_VISIBLE, 0i32, 0i32), + (CURSOR_VISIBLE | CURSOR_RELATIVE_HINT, -5, 2160), + (0, i32::MIN, i32::MAX), + ] { + let s = CursorState { + serial: 42, + flags, + x, + y, + }; + let d = encode_cursor_state_datagram(&s); + assert_eq!(decode_cursor_state_datagram(&d), Some(s)); + assert_eq!(s.visible(), flags & CURSOR_VISIBLE != 0); + assert_eq!(s.relative_hint(), flags & CURSOR_RELATIVE_HINT != 0); + // Append-extensible like 0xCF: a longer buffer still parses the known prefix. + let mut ext = d.clone(); + ext.push(0xFF); + assert_eq!(decode_cursor_state_datagram(&ext), Some(s)); + // Short / wrong tag are rejected before any read. + assert_eq!(decode_cursor_state_datagram(&d[..d.len() - 1]), None); + let mut bad = d.clone(); + bad[0] = HOST_TIMING_MAGIC; + assert_eq!(decode_cursor_state_datagram(&bad), None); + } + } } diff --git a/crates/punktfunk-core/src/quic/handshake.rs b/crates/punktfunk-core/src/quic/handshake.rs index 5f04256c..0f25aa54 100644 --- a/crates/punktfunk-core/src/quic/handshake.rs +++ b/crates/punktfunk-core/src/quic/handshake.rs @@ -83,6 +83,15 @@ pub struct Hello { /// forcing the earlier placeholders. Omitted by older clients / when the client has no HDR /// display (decodes to `None` — the host keeps its built-in EDID defaults). pub display_hdr: Option, + /// Non-video client capabilities — a bitfield of [`CLIENT_CAP_CURSOR`] (the client renders + /// the host cursor locally; the host stops compositing it and forwards shape + state + /// instead). Appended as a single byte AFTER `display_hdr`; because that block is a fixed + /// [`super::datagram::HDR_META_BODY_LEN`]-byte optional with no placeholder form, presence is + /// disambiguated by REMAINING LENGTH at decode: fewer than `HDR_META_BODY_LEN` bytes after + /// `preferred_codec` ⇒ no HDR block, the tail bytes are the post-HDR fields directly. This + /// caps everything after `display_hdr` at `HDR_META_BODY_LEN − 1` bytes total — document any + /// future field here and mind the budget. Omitted when zero and by older clients (→ `0`). + pub client_caps: u8, } /// QUIC application error code a punktfunk/1 client closes the control connection with on a @@ -244,8 +253,13 @@ impl Hello { let vcodecs_present = self.video_codecs != 0; let pref_present = self.preferred_codec != 0; let hdr_present = self.display_hdr.is_some(); - let need_placeholders = - self.video_caps != 0 || ac_present || vcodecs_present || pref_present || hdr_present; + let ccaps_present = self.client_caps != 0; + let need_placeholders = self.video_caps != 0 + || ac_present + || vcodecs_present + || pref_present + || hdr_present + || ccaps_present; match (&self.name, &self.launch) { (None, None) if !need_placeholders => {} (name, _) => { @@ -266,21 +280,27 @@ impl Hello { b.push(self.video_caps); } // audio_channels: emitted when non-stereo OR a later field follows. - if ac_present || vcodecs_present || pref_present || hdr_present { + if ac_present || vcodecs_present || pref_present || hdr_present || ccaps_present { b.push(self.audio_channels); } // video_codecs: emitted when non-zero OR a later field follows. - if vcodecs_present || pref_present || hdr_present { + if vcodecs_present || pref_present || hdr_present || ccaps_present { b.push(self.video_codecs); } - // preferred_codec: emitted when non-zero OR display_hdr follows. - if pref_present || hdr_present { + // preferred_codec: emitted when non-zero OR a later field follows. + if pref_present || hdr_present || ccaps_present { b.push(self.preferred_codec); } - // display_hdr: fixed HDR_META_BODY_LEN-byte HdrMeta body. Last field; omitted when `None`. + // display_hdr: fixed HDR_META_BODY_LEN-byte HdrMeta body; omitted when `None` even if + // later fields follow (no placeholder form — the decoder disambiguates by remaining + // length, which caps the post-HDR tail at HDR_META_BODY_LEN − 1 bytes). if let Some(m) = &self.display_hdr { super::datagram::write_hdr_meta_body(m, &mut b); } + // client_caps: single byte after the (optional) HDR block. Emitted when non-zero. + if ccaps_present { + b.push(self.client_caps); + } b } @@ -346,9 +366,26 @@ impl Hello { preferred_codec: b.get(tail + 3).copied().unwrap_or(0), // Optional trailing HdrMeta body (fixed length) — absent on an older client / a // client without an HDR display → `None` (the host keeps its EDID defaults). - display_hdr: b - .get(tail + 4..tail + 4 + super::datagram::HDR_META_BODY_LEN) - .map(super::datagram::read_hdr_meta_body), + // Presence is decided by REMAINING LENGTH (there is no placeholder form for the + // fixed block): ≥ HDR_META_BODY_LEN bytes after `preferred_codec` ⇒ the block is + // there and post-HDR fields follow it; fewer ⇒ no block, the bytes ARE the post-HDR + // fields. Sound as long as the post-HDR tail stays under HDR_META_BODY_LEN bytes. + display_hdr: (b.len().saturating_sub(tail + 4) >= super::datagram::HDR_META_BODY_LEN) + .then(|| { + b.get(tail + 4..tail + 4 + super::datagram::HDR_META_BODY_LEN) + .map(super::datagram::read_hdr_meta_body) + }) + .flatten(), + // client_caps: the byte after the HDR block when present, else directly at tail+4. + client_caps: { + let off = if b.len().saturating_sub(tail + 4) >= super::datagram::HDR_META_BODY_LEN + { + tail + 4 + super::datagram::HDR_META_BODY_LEN + } else { + tail + 4 + }; + b.get(off).copied().unwrap_or(0) + }, }) } } @@ -829,6 +866,7 @@ mod tests { video_codecs: CODEC_H264 | CODEC_HEVC, preferred_codec: CODEC_H264, display_hdr: None, + client_caps: 0, }; let enc = h.encode(); let dec = Hello::decode(&enc).unwrap(); @@ -905,6 +943,7 @@ mod tests { video_codecs: CODEC_H264 | CODEC_HEVC, // exercise the codec bitfield roundtrip preferred_codec: CODEC_HEVC, display_hdr: None, + client_caps: 0, }; assert_eq!(Hello::decode(&h.encode()).unwrap(), h); let s = Start { @@ -935,6 +974,7 @@ mod tests { video_codecs: 0, preferred_codec: 0, display_hdr: None, + client_caps: 0, }; let enc = h.encode(); assert_eq!(enc.len(), 26); @@ -1052,6 +1092,7 @@ mod tests { video_codecs: 0, preferred_codec: 0, display_hdr: None, + client_caps: 0, }; let enc = base.encode(); assert_eq!( @@ -1103,6 +1144,7 @@ mod tests { video_codecs: 0, preferred_codec: 0, display_hdr: None, + client_caps: 0, }; // launch alone (no name): a zero-length name placeholder keeps the offset deterministic. let with_launch = Hello { @@ -1162,6 +1204,7 @@ mod tests { video_codecs: 0, preferred_codec: 0, display_hdr: None, + client_caps: 0, }; // A real client-panel volume (P3 primaries, 800-nit peak, 0.05-nit floor, 400-nit FALL). let vol = HdrMeta { @@ -1229,6 +1272,7 @@ mod tests { video_codecs: 0, preferred_codec: 0, display_hdr: None, + client_caps: 0, } .encode(); assert!(PairRequest::decode(&h).is_err(), "abi {abi} parsed as pair"); @@ -1242,4 +1286,66 @@ mod tests { .encode(); assert!(Hello::decode(&pr).is_err()); } + #[test] + fn hello_client_caps_roundtrip_and_back_compat() { + let base = Hello { + abi_version: 2, + mode: Mode { + width: 1920, + height: 1080, + refresh_hz: 60, + }, + compositor: CompositorPref::Auto, + gamepad: GamepadPref::Auto, + bitrate_kbps: 0, + name: None, + launch: None, + video_caps: 0, + audio_channels: 2, + video_codecs: 0, + preferred_codec: 0, + display_hdr: None, + client_caps: 0, + }; + let vol = HdrMeta { + display_primaries: [[13250, 34500], [7500, 3000], [34000, 16000]], + white_point: [15635, 16450], + max_display_mastering_luminance: 8_000_000, + min_display_mastering_luminance: 500, + max_cll: 0, + max_fall: 400, + }; + // caps WITHOUT an HDR block: the single byte after preferred_codec (remaining < the + // fixed block length, so the decoder must NOT read it as a truncated HdrMeta). + let caps_only = Hello { + client_caps: CLIENT_CAP_CURSOR, + ..base.clone() + }; + assert_eq!(Hello::decode(&caps_only.encode()).unwrap(), caps_only); + // caps AND the HDR block: caps lands after the fixed block. + let both = Hello { + display_hdr: Some(vol), + client_caps: CLIENT_CAP_CURSOR, + ..base.clone() + }; + assert_eq!(Hello::decode(&both.encode()).unwrap(), both); + // HDR without caps stays byte-identical to the pre-caps wire form and decodes caps 0. + let hdr_only = Hello { + display_hdr: Some(vol), + ..base.clone() + }; + assert_eq!(Hello::decode(&hdr_only.encode()).unwrap(), hdr_only); + // An older client (no trailing byte at all) decodes to 0. + assert_eq!(Hello::decode(&base.encode()).unwrap().client_caps, 0); + // An older HOST reading a caps-bearing Hello: its decode simply never looks past the + // fields it knows — nothing before the caps byte moved. + let enc = both.encode(); + assert_eq!( + Hello::decode(&enc[..enc.len() - 1]).unwrap(), + Hello { + client_caps: 0, + ..both.clone() + } + ); + } } diff --git a/crates/punktfunk-host/src/native.rs b/crates/punktfunk-host/src/native.rs index d5ec9c02..a38c3f8e 100644 --- a/crates/punktfunk-host/src/native.rs +++ b/crates/punktfunk-host/src/native.rs @@ -76,6 +76,8 @@ mod handshake; /// The mid-stream control task (plan §W1); `serve_session` spawns `control::run` after the /// handshake to multiplex renegotiation / speed-test control messages onto the data-plane channels. mod control; +/// Cursor-forward channel (M2): the encode loop's shape/state emission. +mod cursor_fwd; /// The capture→encode→send data plane (plan §W1); `serve_session` dispatches the synthetic or /// virtual source here (`synthetic_stream` / `virtual_stream`) and hands the latter a @@ -943,6 +945,15 @@ async fn serve_session( // accepted ack as "the active mode is now X" and fixes itself; old clients just log it. let (reconfig_result_tx, reconfig_result_rx) = tokio::sync::mpsc::unbounded_channel::(); + // Cursor-forward bridge (M2): the encode loop diffs each frame's cursor serial and hands + // changed SHAPES here; the control task (the control stream's sole writer) sends them. + // Same shape as `probe_result_tx`. Wired even when the channel wasn't negotiated — it + // just never fires then. + let (cursor_shape_tx, cursor_shape_rx) = + tokio::sync::mpsc::unbounded_channel::(); + // Negotiated cursor forwarding: MUST match the HOST_CAP_CURSOR bit the Welcome advertised + // (handshake::cursor_forward is the single predicate both read). + let cursor_forward = handshake::cursor_forward(hello.client_caps, compositor); // Adaptive FEC: the control task maps each client LossReport to a recovery percent and publishes // it here; the data-plane send loop reads + applies it per frame. Disabled (pinned) when // PUNKTFUNK_FEC_PCT is set. Seeded with the session's starting FEC so it's a no-op until a report. @@ -978,6 +989,7 @@ async fn serve_session( probe_tx, probe_result_rx, reconfig_result_rx, + cursor_shape_rx, clip_enabled, clip, )); @@ -1363,6 +1375,8 @@ async fn serve_session( fec_target: fec_target_dp, conn: conn_stream, timing_conn, + cursor_forward, + cursor_shape_tx, probe_seq, streamed_au, stats: stats_dp, @@ -2029,6 +2043,7 @@ mod tests { 0, // video_codecs (HEVC-only) 0, // preferred_codec None, // display_hdr + 0, // client_caps None, // launch None, // pin (TOFU) None, // identity (host doesn't require pairing) @@ -2199,6 +2214,7 @@ mod tests { 0, // video_codecs (0 → HEVC-only) 0, // preferred_codec (auto) None, // display_hdr + 0, // client_caps None, // launch None, // pin: TOFU — the operator's approval (not a PIN) authorizes this client Some((cert, key)), @@ -2266,6 +2282,7 @@ mod tests { 0, // video_codecs 0, // preferred_codec None, // display_hdr + 0, // client_caps None, // launch None, None, @@ -2295,6 +2312,7 @@ mod tests { 0, // video_codecs 0, // preferred_codec None, // display_hdr + 0, // client_caps None, // launch Some(host_fp), Some((cert.clone(), key.clone())), diff --git a/crates/punktfunk-host/src/native/control.rs b/crates/punktfunk-host/src/native/control.rs index 59aac14c..41dea4ba 100644 --- a/crates/punktfunk-host/src/native/control.rs +++ b/crates/punktfunk-host/src/native/control.rs @@ -30,6 +30,7 @@ pub(super) async fn run( probe_tx: std::sync::mpsc::Sender, mut probe_result_rx: tokio::sync::mpsc::UnboundedReceiver, mut reconfig_result_rx: tokio::sync::mpsc::UnboundedReceiver, + mut cursor_shape_rx: tokio::sync::mpsc::UnboundedReceiver, clip_enabled: Arc, clip: pf_clipboard::ClipCoord, ) { @@ -262,6 +263,15 @@ pub(super) async fn run( break; } } + shape = cursor_shape_rx.recv() => { + // Cursor-forward bridge (M2): the encode loop diffed a new pointer bitmap. + // Rare (shape changes are human-paced); ≤ ~58 KiB fits the u16 frame by + // construction (cursor_fwd downscales). + let Some(shape) = shape else { break }; // data plane gone + if io::write_msg(&mut ctrl_send, &shape.encode()).await.is_err() { + break; + } + } offer = clip_offer_rx.recv(), if !clip_offer_closed => { // Host copied → the coordinator minted a `ClipOffer`; forward it to the client // (only while sync is on — a race with a just-received disable would otherwise diff --git a/crates/punktfunk-host/src/native/cursor_fwd.rs b/crates/punktfunk-host/src/native/cursor_fwd.rs new file mode 100644 index 00000000..923c003b --- /dev/null +++ b/crates/punktfunk-host/src/native/cursor_fwd.rs @@ -0,0 +1,144 @@ +//! Cursor-forward channel, host side (design/remote-desktop-sweep.md M2). +//! +//! When the session negotiated the cursor channel (client `CLIENT_CAP_CURSOR` met our +//! `HOST_CAP_CURSOR`), the encoder stops blending the pointer into the video +//! (`SessionPlan::cursor_blend = false`) and the encode loop forwards it out-of-band instead: +//! the SHAPE (bitmap + hotspot, rare) rides the reliable control stream via the control-task +//! bridge, per-tick STATE (position/visibility, 14 B) rides a lossy `0xD0` datagram — resent +//! every iteration so loss self-heals with no refresh timer. + +use punktfunk_core::quic::{ + encode_cursor_state_datagram, CursorShape, CursorState, CURSOR_SHAPE_MAX_SIDE, CURSOR_VISIBLE, +}; + +/// Per-session forward state, owned by the encode loop (the thread that binds frames). +pub(super) struct CursorForwarder { + /// Serial of the last shape handed to the control-task bridge (`None` = none yet). + sent_serial: Option, + /// Last visible pointer position (hotspot point, frame px) — held across hidden spans so + /// a hide still states WHERE the pointer was (the M3 reappear position). + last_pos: (i32, i32), +} + +impl CursorForwarder { + pub(super) fn new() -> CursorForwarder { + CursorForwarder { + sent_serial: None, + last_pos: (0, 0), + } + } + + /// Called once per encode-loop iteration with the bound frame's overlay (also on repeat + /// iterations — the state datagram is the plane's loss heal, so it goes out every tick). + /// `None` overlay = hidden pointer (or no bitmap yet): state only, `visible` clear. + pub(super) fn tick( + &mut self, + cursor: Option<&pf_frame::CursorOverlay>, + conn: &quinn::Connection, + shape_tx: &tokio::sync::mpsc::UnboundedSender, + ) { + let flags = match cursor { + Some(ov) => { + if self.sent_serial != Some(ov.serial) { + if let Some(shape) = shape_from_overlay(ov) { + // Bridge full ⇒ control task gone ⇒ session is tearing down anyway. + let _ = shape_tx.send(shape); + self.sent_serial = Some(ov.serial); + } + } + self.last_pos = (ov.x + ov.hot_x as i32, ov.y + ov.hot_y as i32); + CURSOR_VISIBLE + } + None => 0, + }; + let state = CursorState { + serial: self.sent_serial.unwrap_or(0) as u32, + flags, + x: self.last_pos.0, + y: self.last_pos.1, + }; + let _ = conn.send_datagram(encode_cursor_state_datagram(&state).into()); + } +} + +/// Build the wire shape from a capture overlay, integer-downscaling (nearest-neighbor) anything +/// over [`CURSOR_SHAPE_MAX_SIDE`] so the message always fits the u16-length control frame. +/// Real cursors are far under the cap — the scale path is a correctness backstop for XL +/// accessibility cursors, not a quality path. `None` on a malformed overlay (short buffer). +fn shape_from_overlay(ov: &pf_frame::CursorOverlay) -> Option { + let px = (ov.w as usize).checked_mul(ov.h as usize)?.checked_mul(4)?; + if ov.w == 0 || ov.h == 0 || ov.rgba.len() < px { + return None; + } + let max = CURSOR_SHAPE_MAX_SIDE as u32; + let f = ov.w.max(ov.h).div_ceil(max).max(1); + let (w, h) = (ov.w.div_ceil(f), ov.h.div_ceil(f)); + let rgba = if f == 1 { + ov.rgba.as_ref().clone() + } else { + let mut out = Vec::with_capacity((w * h * 4) as usize); + for y in 0..h { + for x in 0..w { + let (sx, sy) = ((x * f).min(ov.w - 1), (y * f).min(ov.h - 1)); + let o = ((sy * ov.w + sx) * 4) as usize; + out.extend_from_slice(&ov.rgba[o..o + 4]); + } + } + out + }; + Some(CursorShape { + serial: ov.serial as u32, + w: w as u16, + h: h as u16, + hot_x: (ov.hot_x / f).min(w - 1) as u16, + hot_y: (ov.hot_y / f).min(h - 1) as u16, + rgba, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + fn overlay(w: u32, h: u32, hot: (u32, u32)) -> pf_frame::CursorOverlay { + pf_frame::CursorOverlay { + x: 10, + y: 20, + w, + h, + rgba: Arc::new((0..w * h * 4).map(|i| i as u8).collect()), + serial: 3, + hot_x: hot.0, + hot_y: hot.1, + } + } + + #[test] + fn small_shape_passes_through() { + let s = shape_from_overlay(&overlay(32, 32, (4, 5))).unwrap(); + assert_eq!((s.w, s.h, s.hot_x, s.hot_y, s.serial), (32, 32, 4, 5, 3)); + assert_eq!(s.rgba.len(), 32 * 32 * 4); + // Encodes within the u16 control-frame cap. + assert!(s.encode().len() <= u16::MAX as usize); + } + + #[test] + fn oversize_shape_downscales_with_hotspot() { + // 256² → f = ceil(256/120) = 3 → 86² (256.div_ceil(3)), hotspot scales with it. + let s = shape_from_overlay(&overlay(256, 256, (255, 0))).unwrap(); + assert!(s.w <= CURSOR_SHAPE_MAX_SIDE && s.h <= CURSOR_SHAPE_MAX_SIDE); + assert_eq!(s.rgba.len(), s.w as usize * s.h as usize * 4); + assert!(s.hot_x < s.w && s.hot_y < s.h); + assert!(s.encode().len() <= u16::MAX as usize); + // The scaled message must decode (dims within the cap). + assert_eq!(CursorShape::decode(&s.encode()).unwrap(), s); + } + + #[test] + fn short_buffer_rejected() { + let mut ov = overlay(8, 8, (0, 0)); + ov.rgba = Arc::new(vec![0; 8]); + assert!(shape_from_overlay(&ov).is_none()); + } +} diff --git a/crates/punktfunk-host/src/native/handshake.rs b/crates/punktfunk-host/src/native/handshake.rs index f9d41128..ade4ee6d 100644 --- a/crates/punktfunk-host/src/native/handshake.rs +++ b/crates/punktfunk-host/src/native/handshake.rs @@ -8,6 +8,22 @@ use super::*; +/// Whether this session forwards the cursor out-of-band (design/remote-desktop-sweep.md M2): +/// the client asked ([`CLIENT_CAP_CURSOR`](punktfunk_core::quic::CLIENT_CAP_CURSOR)) AND the +/// capture path can deliver cursor metadata separately from the frame — today that is the +/// Linux portal `SPA_META_Cursor` path only: not gamescope (its capture paints no cursor at +/// all), not Windows (DWM composites into the IDD frame — M2c). THE single predicate: the +/// Welcome's `HOST_CAP_CURSOR` bit and the session's forwarding/blend-off wiring both read it, +/// so they can never disagree. +pub(super) fn cursor_forward( + client_caps: u8, + compositor: Option, +) -> bool { + cfg!(target_os = "linux") + && client_caps & punktfunk_core::quic::CLIENT_CAP_CURSOR != 0 + && compositor.is_some_and(|c| c != crate::vdisplay::Compositor::Gamescope) +} + /// Run the Hello→Welcome→Start negotiation. Borrows the control streams (the caller keeps them for /// mid-stream renegotiation afterwards). `first` is the already-read first control message. #[allow(clippy::type_complexity, clippy::too_many_arguments)] @@ -444,6 +460,14 @@ pub(super) async fn negotiate( punktfunk_core::quic::HOST_CAP_CLIPBOARD } else { 0 + } + // Cursor channel granted (client asked + this capture path can deliver cursor + // metadata out of the frame) — the client turns its local renderer on ONLY when + // it sees this bit, and serve_session wires forwarding from the same predicate. + | if cursor_forward(hello.client_caps, compositor) { + punktfunk_core::quic::HOST_CAP_CURSOR + } else { + 0 }, // The negotiated session AEAD (resolved above) + its 32-byte key toward a ChaCha // client; toward everyone else cipher 0 keeps the Welcome byte-identical to the diff --git a/crates/punktfunk-host/src/native/stream.rs b/crates/punktfunk-host/src/native/stream.rs index 753063e7..2ee69a34 100644 --- a/crates/punktfunk-host/src/native/stream.rs +++ b/crates/punktfunk-host/src/native/stream.rs @@ -938,6 +938,14 @@ pub(super) struct SessionContext { /// thread emits one 0xCF datagram per AU (capture→sent µs) on it, so the client can split its /// `host+network` latency stage. `None` = older client, no emission. pub(super) timing_conn: Option, + /// The session negotiated the cursor channel (design/remote-desktop-sweep.md M2 — + /// `handshake::cursor_forward`): the encoder does NOT blend the pointer into the video; + /// the encode loop forwards shape (via `cursor_shape_tx`) + per-tick `0xD0` state instead. + pub(super) cursor_forward: bool, + /// SHAPE bridge to the control task (the control stream's sole writer) — mirrors + /// `probe_result_tx`. Inert when `cursor_forward` is false. + pub(super) cursor_shape_tx: + tokio::sync::mpsc::UnboundedSender, /// The client advertised [`punktfunk_core::quic::VIDEO_CAP_PROBE_SEQ`]: speed-test bursts may /// run mid-session in the probe index space (its reassembler keeps a separate probe window). /// `false` = older client whose single-window reassembler would drop probe-space frames as @@ -987,7 +995,10 @@ pub(super) fn virtual_stream(ctx: SessionContext, prepared: Option= std::time::Duration::from_secs(2) { let secs = diag_at.elapsed().as_secs_f64(); tracing::info!( diff --git a/include/punktfunk_core.h b/include/punktfunk_core.h index 1c55ceca..469f7682 100644 --- a/include/punktfunk_core.h +++ b/include/punktfunk_core.h @@ -498,6 +498,28 @@ #define HOST_CAP_CLIPBOARD 2 #endif +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`Hello::client_caps`] bit: the client renders the host cursor LOCALLY +// (design/remote-desktop-sweep.md M2). It consumes [`CursorShape`](super::control::CursorShape) +// control messages (RGBA bitmap + hotspot, cached by serial) and per-frame +// [`CursorState`](super::datagram::CursorState) `0xD0` datagrams (position/visibility), and +// draws the pointer itself — so the host must STOP compositing the cursor into the video +// (`SessionPlan.cursor_blend = false`) or the user sees it twice. Active only when the host +// answers with [`HOST_CAP_CURSOR`] (capable-and-agreed, the 444/clipboard precedent); toward +// an older or incapable host nothing changes. +#define CLIENT_CAP_CURSOR 1 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`Welcome::host_caps`] bit: the host CAN forward the cursor out-of-band (it captures cursor +// metadata separately from the frame — the Linux portal `SPA_META_Cursor` path; NOT gamescope, +// whose capture carries no cursor, and NOT Windows yet, where DWM composites into the IDD +// frame). Set only when the client asked via [`CLIENT_CAP_CURSOR`]; when both bits agree the +// host stops blending and ships [`CursorShape`](super::control::CursorShape) + +// [`CursorState`](super::datagram::CursorState) instead. +#define HOST_CAP_CURSOR 4 +#endif + #if defined(PUNKTFUNK_FEATURE_QUIC) // [`Hello::video_codecs`] bit: the client can decode H.264 / AVC. The GPU-less **software** // encode path (openh264) emits H.264, so a client that wants to stream from a software host MUST @@ -750,6 +772,20 @@ #define CLIP_FILE_INDEX_NONE UINT32_MAX #endif +#if defined(PUNKTFUNK_FEATURE_QUIC) +// Type byte of [`CursorShape`] (host → client): the pointer's bitmap + hotspot changed. +#define MSG_CURSOR_SHAPE 80 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// Per-side pixel cap for a forwarded cursor bitmap. The control-stream frame is length-prefixed +// with a `u16`, so a whole message must fit 65535 bytes — 128×128 RGBA (65536 B) already +// overshoots before the 17-byte header. 120² (57.6 KiB + header) fits with headroom and covers +// real cursors (typically ≤ 64 px, ≤ 96 px at HiDPI scale); the HOST downscales anything +// larger before forwarding, so the cap is invisible to clients. +#define CURSOR_SHAPE_MAX_SIDE 120 +#endif + #if defined(PUNKTFUNK_FEATURE_QUIC) // Datagram wire tags. Video rides UDP; everything low-rate rides QUIC datagrams, // demultiplexed by the first byte: input = [`crate::input::INPUT_MAGIC`] (0xC8, client→host), @@ -838,6 +874,28 @@ #define HOST_TIMING_MAGIC 207 #endif +#if defined(PUNKTFUNK_FEATURE_QUIC) +// Cursor-state datagram tag, host → client (design/remote-desktop-sweep.md M2). Next tag after +// [`HOST_TIMING_MAGIC`]. Sent once per captured frame while the cursor channel is negotiated +// ([`CLIENT_CAP_CURSOR`](super::caps::CLIENT_CAP_CURSOR) ∧ +// [`HOST_CAP_CURSOR`](super::caps::HOST_CAP_CURSOR)) — per-frame resend makes the plane +// self-healing under loss (latest-wins, no refresh timer). The bitmap itself rides the +// reliable control stream ([`CursorShape`](super::control::CursorShape)); this 14-byte +// datagram only moves/hides the pointer. +#define CURSOR_STATE_MAGIC 208 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`CursorState::flags`] bit: the host cursor is visible. +#define CURSOR_VISIBLE 1 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`CursorState::flags`] bit: a host app captured/hid the pointer — the client SHOULD run +// relative/captured (M3 auto-flip; advisory, user override always wins). +#define CURSOR_RELATIVE_HINT 2 +#endif + #if defined(PUNKTFUNK_FEATURE_QUIC) // QUIC application error code a punktfunk/1 client closes the control connection with on a // **deliberate quit** (a user "stop", not a network drop). The host reads it off the connection's