From c3cbffe662f8a5429e06a95533519e73b9f73c36 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Wed, 22 Jul 2026 08:05:35 +0200 Subject: [PATCH] =?UTF-8?q?feat(host+client):=20host-driven=20mouse-model?= =?UTF-8?q?=20flip=20=E2=80=94=20remote-desktop=20sweep=20M3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The full Parsec model: launching a game from a desktop session flips the client into captured relative automatically, and back — no chords. - Capture overlay now distinguishes 'hidden' from 'no cursor yet': CursorOverlay grows a visible flag, overlay() returns Some whenever a bitmap is known (the encode loop strips invisible overlays after forwarding so no blend path ever draws one; the CPU composite guarded on visibility already). - Host forwarder maps it onto the reserved wire bit: visible ⇒ VISIBLE, hidden-but-known ⇒ RELATIVE_HINT (an app grabbed/hid the pointer), never any hint before the first bitmap — a cold start can't flip a desktop session into capture. - Presenter: edge-triggered auto-flip on hint changes via the new Capture::set_desktop (chord semantics untouched); a manual ⌃⌥⇧M sets an override latch that holds until the HOST's intent next changes, so the hint never fights the user. Leaving relative warps the local cursor to the host's last pointer position through the new content_to_window inverse-letterbox mapping (unit-tested against finger_to_content) — the hand-back is seamless. Verified on .21: fmt + clippy -D warnings + tests (presenter 20 incl. the inverse-mapping roundtrip, host 245). Co-Authored-By: Claude Fable 5 --- crates/pf-capture/src/linux/mod.rs | 11 +- crates/pf-frame/src/lib.rs | 5 + crates/pf-presenter/src/cursor.rs | 6 + crates/pf-presenter/src/input.rs | 14 ++ crates/pf-presenter/src/run.rs | 127 ++++++++++++++++-- .../punktfunk-host/src/native/cursor_fwd.rs | 12 +- crates/punktfunk-host/src/native/stream.rs | 8 +- 7 files changed, 166 insertions(+), 17 deletions(-) diff --git a/crates/pf-capture/src/linux/mod.rs b/crates/pf-capture/src/linux/mod.rs index 25bf882b..04d8a215 100644 --- a/crates/pf-capture/src/linux/mod.rs +++ b/crates/pf-capture/src/linux/mod.rs @@ -875,10 +875,14 @@ mod pipewire { } impl CursorState { - /// A shareable overlay for the GPU encode paths (blended at encode time), or `None` when - /// there is nothing to draw. Cheap: clones an `Arc` + a few scalars. + /// A shareable overlay for the encode/forward paths, or `None` before the first bitmap + /// arrived. A HIDDEN pointer still yields `Some` (with `visible: false`): the + /// cursor-forward channel needs "known but hidden" — an app grabbed the pointer, the + /// client's relative-mode hint (M3) — which is a different fact from "no cursor yet". + /// The encode loop strips invisible overlays before any blend path sees the frame. + /// Cheap: clones an `Arc` + a few scalars. fn overlay(&self) -> Option { - if !self.visible || self.rgba.is_empty() { + if self.rgba.is_empty() { return None; } Some(pf_frame::CursorOverlay { @@ -890,6 +894,7 @@ mod pipewire { serial: self.serial, hot_x: self.hot_x.max(0) as u32, hot_y: self.hot_y.max(0) as u32, + visible: self.visible, }) } } diff --git a/crates/pf-frame/src/lib.rs b/crates/pf-frame/src/lib.rs index 5f82fa11..06140dc4 100644 --- a/crates/pf-frame/src/lib.rs +++ b/crates/pf-frame/src/lib.rs @@ -201,6 +201,11 @@ pub struct CursorOverlay { /// client so a locally-drawn OS cursor points with the right pixel. pub hot_x: u32, pub hot_y: u32, + /// Compositor-reported pointer visibility. `false` = an app on the host grabbed/hid the + /// pointer — the cursor-forward channel turns that into the client's relative-mode hint + /// (remote-desktop-sweep M3). The encode loop STRIPS invisible overlays before the frame + /// reaches any blend path, so encoders may keep treating `Some` as "draw it". + pub visible: bool, } /// A captured frame. [`format`](Self::format)/dimensions describe the pixels regardless of diff --git a/crates/pf-presenter/src/cursor.rs b/crates/pf-presenter/src/cursor.rs index a2414e6d..ce9bfa39 100644 --- a/crates/pf-presenter/src/cursor.rs +++ b/crates/pf-presenter/src/cursor.rs @@ -48,6 +48,12 @@ impl CursorChannel { self.negotiated } + /// The latest drained `0xD0` state — the run loop reads `relative_hint` off it for the + /// M3 host-driven mode flip (and `x`/`y` as the reappear position when leaving relative). + pub fn state(&self) -> Option { + self.state + } + /// Drain the two planes and apply the newest state — once per run-loop iteration. /// `desktop_active` = the desktop mouse model is engaged (captured + desktop): only then /// do we own the local cursor's shape/visibility; under capture SDL's relative mode owns diff --git a/crates/pf-presenter/src/input.rs b/crates/pf-presenter/src/input.rs index ef82eefb..ca713619 100644 --- a/crates/pf-presenter/src/input.rs +++ b/crates/pf-presenter/src/input.rs @@ -133,6 +133,20 @@ impl Capture { Some(self.desktop) } + /// Set the mouse model directly (the M3 host-driven flip — `relative_hint` says a host + /// app grabbed/hid the pointer, so run relative; hint clear = back to absolute). Same + /// gating and motion hygiene as [`toggle_desktop`](Self::toggle_desktop); returns whether + /// the model actually changed. + pub fn set_desktop(&mut self, on: bool) -> bool { + if !self.abs_ok || self.desktop == on { + return false; + } + self.desktop = on; + self.pending_rel = (0, 0); + self.pending_abs = None; + true + } + /// Whether a regained focus should re-engage: yes unless the user released /// deliberately (the chord keeps its meaning across an Alt-Tab). pub fn should_reengage(&self) -> bool { diff --git a/crates/pf-presenter/src/run.rs b/crates/pf-presenter/src/run.rs index d49cec78..311d9ecf 100644 --- a/crates/pf-presenter/src/run.rs +++ b/crates/pf-presenter/src/run.rs @@ -236,6 +236,12 @@ struct StreamState { /// Client-side cursor rendering (M2 cursor channel) — created with the connector; inert /// when the host didn't negotiate the channel. cursor_chan: Option, + /// Last observed `relative_hint` (M3): the auto-flip fires on CHANGES only, so it never + /// fights a user who chorded away from the hinted model. + last_hint: Option, + /// The user flipped the model manually (⌃⌥⇧M) — the standing hint stops driving until + /// the HOST's intent next changes (a fresh hint edge clears this and applies). + hint_override: bool, } impl StreamState { @@ -267,6 +273,8 @@ impl StreamState { connector: None, capture: None, cursor_chan: None, + last_hint: None, + hint_override: false, force_software, canceled: false, ready_announced: false, @@ -558,18 +566,27 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result // Mouse model flip (capture ⇄ desktop) — applies immediately when // engaged; a released stream just changes what the next engage does. if chord && sc == Scancode::M { - if let Some(cap) = stream.as_mut().and_then(|s| s.capture.as_mut()) { - match cap.toggle_desktop() { - Some(desktop) => { - if cap.captured() { - apply_capture(&mut window, &mouse, true, desktop); + if let Some(st) = stream.as_mut() { + let mut flipped = false; + if let Some(cap) = st.capture.as_mut() { + match cap.toggle_desktop() { + Some(desktop) => { + if cap.captured() { + apply_capture(&mut window, &mouse, true, desktop); + } + flipped = true; + tracing::info!(desktop, "chord: mouse mode"); } - tracing::info!(desktop, "chord: mouse mode"); + None => tracing::info!( + "chord: mouse mode — host has no absolute pointer \ + (gamescope), staying captured" + ), } - None => tracing::info!( - "chord: mouse mode — host has no absolute pointer \ - (gamescope), staying captured" - ), + } + // A manual flip outranks the standing hint until the host's + // intent next CHANGES (M3 — the hint edge clears this). + if flipped { + st.hint_override = true; } } continue; @@ -758,6 +775,46 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result .is_some_and(|cap| cap.captured() && cap.desktop()); chan.pump(c, &mouse, desktop_active); } + // M3 — host-driven mode flip: `relative_hint` set = a host app grabbed/hid the + // pointer (run captured relative, like a game expects); clear = the desktop is + // back (return to absolute, local cursor reappearing at the host's position). + // Edge-triggered so a user's manual chord isn't fought: the override latch + // holds until the HOST's intent next changes. + let hint_state = st.cursor_chan.as_ref().and_then(|ch| ch.state()); + if let Some(hs) = hint_state { + let hint = hs.relative_hint(); + if st.last_hint != Some(hint) { + st.last_hint = Some(hint); + st.hint_override = false; + } + if !st.hint_override { + let video = st.last_video; + if let Some(cap) = st.capture.as_mut() { + // Desired model: hint ⇒ capture (desktop off); clear ⇒ desktop on. + if cap.captured() && cap.set_desktop(!hint) { + apply_capture(&mut window, &mouse, true, cap.desktop()); + if cap.desktop() { + // Reappear where the host last had the pointer, so the + // hand-back is seamless (Parsec's positionX/Y idea). + if let Some(video) = video { + let (wx, wy) = content_to_window( + window.size(), + window.size_in_pixels(), + video, + hs.x, + hs.y, + ); + mouse.warp_mouse_in_window(&window, wx, wy); + } + } + tracing::info!( + desktop = cap.desktop(), + "host cursor hint: mouse model flipped" + ); + } + } + } + } } // Text input follows the overlay's editing state (edge-triggered). @@ -1682,6 +1739,32 @@ fn finger_to_content( (cx.round() as i32, cy.round() as i32, dw as u32, dh as u32) } +/// The inverse direction of [`finger_to_content`] for the M3 reappear warp: a HOST-frame pixel +/// (`video` space — what `CursorState` carries) → LOGICAL window coordinates (what +/// `warp_mouse_in_window` takes). Maps through the aspect-fit letterbox (physical), then +/// physical → logical via the window's pixel density; out-of-range host coords clamp into the +/// content rect so the warp always lands on the video. +fn content_to_window( + logical: (u32, u32), + surface: (u32, u32), + video: (u32, u32), + x: i32, + y: i32, +) -> (f32, f32) { + let (pw, ph) = (f64::from(surface.0), f64::from(surface.1)); + let (vw, vh) = (f64::from(video.0.max(1)), f64::from(video.1.max(1))); + let scale = (pw / vw).min(ph / vh); + let (dw, dh) = ((vw * scale).max(1.0), (vh * scale).max(1.0)); + let ox = (pw - dw) / 2.0; + let oy = (ph - dh) / 2.0; + let px = ox + (f64::from(x).clamp(0.0, vw - 1.0)) * scale; + let py = oy + (f64::from(y).clamp(0.0, vh - 1.0)) * scale; + // Physical → logical (HiDPI): the ratio of the window's logical size to its pixel size. + let lx = px * f64::from(logical.0) / pw.max(1.0); + let ly = py * f64::from(logical.1) / ph.max(1.0); + (lx as f32, ly as f32) +} + /// The presenter's share of the unified stats window — folded into each printed line. #[derive(Default)] struct PresentedWindow { @@ -1779,6 +1862,30 @@ fn stats_text( mod tests { use super::*; + #[test] + fn content_to_window_inverts_the_letterbox() { + // 1920×1080 video letterboxed in a 1600×1200 (4:3) window at 2× HiDPI: pillarless + // top/bottom bars — scale = 1600/1920, dh = 900, oy = 150 (physical). + let logical = (800u32, 600u32); + let surface = (1600u32, 1200u32); + let video = (1920u32, 1080u32); + // The host-frame center must land at the logical window center. + let (wx, wy) = content_to_window(logical, surface, video, 960, 540); + assert!((wx - 400.0).abs() < 1.0, "wx = {wx}"); + assert!((wy - 300.0).abs() < 1.0, "wy = {wy}"); + // Roundtrip through the forward mapping: normalized window pos → the same host + // content-rect pixel (finger_to_content returns content-RECT coords, i.e. the + // host pixel scaled by the letterbox factor). + let (nx, ny) = (wx / logical.0 as f32, wy / logical.1 as f32); + let (cx, cy, dw, dh) = finger_to_content(surface, video, nx, ny); + assert_eq!((dw, dh), (1600, 900)); + assert!((cx - 800).abs() <= 1, "cx = {cx}"); // 960 * (1600/1920) + assert!((cy - 450).abs() <= 1, "cy = {cy}"); // 540 * ( 900/1080) + // Out-of-range host coords clamp into the video, never the bars. + let (_, wy_clamped) = content_to_window(logical, surface, video, 0, 10_000); + assert!(wy_clamped <= 300.0 + 225.0 + 1.0, "wy = {wy_clamped}"); // ≤ bottom of content + } + #[test] fn resize_decision_follows_the_d2_discipline() { let t0 = Instant::now(); diff --git a/crates/punktfunk-host/src/native/cursor_fwd.rs b/crates/punktfunk-host/src/native/cursor_fwd.rs index 923c003b..74dc8a8b 100644 --- a/crates/punktfunk-host/src/native/cursor_fwd.rs +++ b/crates/punktfunk-host/src/native/cursor_fwd.rs @@ -8,7 +8,8 @@ //! every iteration so loss self-heals with no refresh timer. use punktfunk_core::quic::{ - encode_cursor_state_datagram, CursorShape, CursorState, CURSOR_SHAPE_MAX_SIDE, CURSOR_VISIBLE, + encode_cursor_state_datagram, CursorShape, CursorState, CURSOR_RELATIVE_HINT, + CURSOR_SHAPE_MAX_SIDE, CURSOR_VISIBLE, }; /// Per-session forward state, owned by the encode loop (the thread that binds frames). @@ -30,7 +31,10 @@ impl CursorForwarder { /// Called once per encode-loop iteration with the bound frame's overlay (also on repeat /// iterations — the state datagram is the plane's loss heal, so it goes out every tick). - /// `None` overlay = hidden pointer (or no bitmap yet): state only, `visible` clear. + /// Flag mapping (M3): a visible overlay states VISIBLE; a hidden-but-known overlay + /// (an app grabbed/hid the pointer) states RELATIVE_HINT — the client should flip to + /// captured relative; `None` (no bitmap has EVER arrived) states neither — never hint + /// off a cold start, only off an observed hide. pub(super) fn tick( &mut self, cursor: Option<&pf_frame::CursorOverlay>, @@ -38,7 +42,7 @@ impl CursorForwarder { shape_tx: &tokio::sync::mpsc::UnboundedSender, ) { let flags = match cursor { - Some(ov) => { + Some(ov) if ov.visible => { if self.sent_serial != Some(ov.serial) { if let Some(shape) = shape_from_overlay(ov) { // Bridge full ⇒ control task gone ⇒ session is tearing down anyway. @@ -49,6 +53,7 @@ impl CursorForwarder { self.last_pos = (ov.x + ov.hot_x as i32, ov.y + ov.hot_y as i32); CURSOR_VISIBLE } + Some(_) => CURSOR_RELATIVE_HINT, None => 0, }; let state = CursorState { @@ -111,6 +116,7 @@ mod tests { serial: 3, hot_x: hot.0, hot_y: hot.1, + visible: true, } } diff --git a/crates/punktfunk-host/src/native/stream.rs b/crates/punktfunk-host/src/native/stream.rs index 2ee69a34..8fbc0b84 100644 --- a/crates/punktfunk-host/src/native/stream.rs +++ b/crates/punktfunk-host/src/native/stream.rs @@ -2007,10 +2007,16 @@ pub(super) fn virtual_stream(ctx: SessionContext, prepared: Option