From 087e7d040571c5d97b7d289062ffcc783bf133fa Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 24 Jul 2026 17:38:36 +0200 Subject: [PATCH] fix(inject/windows): map absolute input over the streamed output's rect, not the whole desktop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pen, touch, and absolute mouse arrive normalized to the STREAMED display's frame, but pointer_windows::to_screen and sendinput's MouseMoveAbs mapped them over the whole virtual desktop — correct only when the virtual display is the sole active display (Exclusive topology). In Extend — a physical monitor kept on beside the virtual output, or an Exclusive isolate degraded to the 0x57 keep-physicals fallback — the streamed output sits at a non-zero origin, so every sample landed shifted and mis-scaled. The pen exposed it first (field report 2026-07-24): a stylus is strictly absolute, with no closed-loop correction onto the target like a cursor. New pf-inject::stream_target (Windows): the host publishes the streamed output's CCD target id at capture bring-up (one central site — capture_virtual_output covers the native and GameStream planes); mapping sites resolve its current desktop rect through pf-win-display's source_desktop_rect — the same resolver the cursor-readback poller uses, so inject and readback always agree — TTL-cached (250 ms) because a group-layout re-arrange moves a live output's origin mid-session. No target set / never resolved falls back to the whole virtual desktop (the historical mapping, still right for Exclusive topology and devtest). One change in to_screen covers pen + touch; MouseMoveAbs converts the same desktop pixel into the 0..65535 MOUSEEVENTF_VIRTUALDESK coordinate, closing the identical latent absolute-mouse bug (apollo-comparison open item #14/#30). Co-Authored-By: Claude Fable 5 --- Cargo.lock | 1 + crates/pf-inject/Cargo.toml | 3 + .../src/inject/windows/pointer_windows.rs | 27 +-- .../pf-inject/src/inject/windows/sendinput.rs | 36 ++-- .../src/inject/windows/stream_target.rs | 181 ++++++++++++++++++ crates/pf-inject/src/lib.rs | 10 + crates/punktfunk-host/src/capture.rs | 5 + 7 files changed, 220 insertions(+), 43 deletions(-) create mode 100644 crates/pf-inject/src/inject/windows/stream_target.rs diff --git a/Cargo.lock b/Cargo.lock index f7d20c89..a22f6744 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2981,6 +2981,7 @@ dependencies = [ "pf-driver-proto", "pf-host-config", "pf-paths", + "pf-win-display", "punktfunk-core", "reis", "tokio", diff --git a/crates/pf-inject/Cargo.toml b/crates/pf-inject/Cargo.toml index 2da67618..81150f51 100644 --- a/crates/pf-inject/Cargo.toml +++ b/crates/pf-inject/Cargo.toml @@ -48,6 +48,9 @@ xkbcommon = "0.8" usbip-sim = { path = "../punktfunk-host/vendor/usbip-sim" } [target.'cfg(target_os = "windows")'.dependencies] +# The streamed-output desktop rect (CCD source rect) absolute input maps into — the same +# resolver the cursor-readback poller uses, so inject and readback always agree. +pf-win-display = { path = "../pf-win-display" } windows = { version = "0.62", features = [ "Win32_Foundation", "Win32_Security", diff --git a/crates/pf-inject/src/inject/windows/pointer_windows.rs b/crates/pf-inject/src/inject/windows/pointer_windows.rs index 5133d588..e94e9777 100644 --- a/crates/pf-inject/src/inject/windows/pointer_windows.rs +++ b/crates/pf-inject/src/inject/windows/pointer_windows.rs @@ -41,9 +41,8 @@ use windows::Win32::UI::Input::Pointer::{ POINTER_FLAG_UPDATE, POINTER_INFO, POINTER_PEN_INFO, POINTER_TOUCH_INFO, }; use windows::Win32::UI::WindowsAndMessaging::{ - GetSystemMetrics, PEN_FLAG_BARREL, PEN_FLAG_ERASER, PEN_FLAG_INVERTED, PEN_FLAG_NONE, - PEN_MASK_PRESSURE, PEN_MASK_ROTATION, PEN_MASK_TILT_X, PEN_MASK_TILT_Y, PT_PEN, PT_TOUCH, - SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, TOUCH_FLAG_NONE, + PEN_FLAG_BARREL, PEN_FLAG_ERASER, PEN_FLAG_INVERTED, PEN_FLAG_NONE, PEN_MASK_PRESSURE, + PEN_MASK_ROTATION, PEN_MASK_TILT_X, PEN_MASK_TILT_Y, PT_PEN, PT_TOUCH, TOUCH_FLAG_NONE, TOUCH_MASK_NONE, }; @@ -149,22 +148,14 @@ unsafe fn inject_following_desktop( /// Windows pen pressure is 0..1024 (vs the wire's full-scale u16). const WIN_PEN_PRESSURE_MAX: u32 = 1024; -/// Map a normalized [0,1] coordinate pair onto virtual-desktop pixels — the same surface the -/// SendInput absolute mouse targets, so pen, touch, and pointer all land identically. +/// Map a normalized [0,1] coordinate pair onto desktop pixels over the STREAMED output's rect +/// ([`crate::stream_target`]) — the surface the SendInput absolute mouse targets too, so pen, +/// touch, and pointer all land identically. The wire normalizes over the streamed display's +/// frame, not the desktop: mapping over the whole virtual desktop is only right when they +/// coincide (Exclusive topology — the fallback when no stream target is live). fn to_screen(x: f32, y: f32) -> POINT { - // SAFETY: `GetSystemMetrics` takes a constant index and reads global metrics; no memory in. - let (vx, vy, vw, vh) = unsafe { - ( - GetSystemMetrics(SM_XVIRTUALSCREEN), - GetSystemMetrics(SM_YVIRTUALSCREEN), - GetSystemMetrics(SM_CXVIRTUALSCREEN).max(1), - GetSystemMetrics(SM_CYVIRTUALSCREEN).max(1), - ) - }; - POINT { - x: vx + (x.clamp(0.0, 1.0) * (vw - 1) as f32) as i32, - y: vy + (y.clamp(0.0, 1.0) * (vh - 1) as f32) as i32, - } + let (px, py) = crate::stream_target::map_normalized(x as f64, y as f64); + POINT { x: px, y: py } } /// An owned `HSYNTHETICPOINTERDEVICE` (destroyed exactly once on drop). diff --git a/crates/pf-inject/src/inject/windows/sendinput.rs b/crates/pf-inject/src/inject/windows/sendinput.rs index 1c1a699a..4f1baab9 100644 --- a/crates/pf-inject/src/inject/windows/sendinput.rs +++ b/crates/pf-inject/src/inject/windows/sendinput.rs @@ -1,6 +1,6 @@ //! Windows input injection via `SendInput` (Win32 KeyboardAndMouse) — the Windows analogue of -//! [`super::wlr`]: absolute mouse normalized to the virtual desktop, relative mouse for games, -//! scancode keyboard, scroll, buttons. Survives UAC/lock desktop switches with Sunshine's +//! [`super::wlr`]: absolute mouse mapped over the streamed output's rect +//! ([`crate::stream_target`]), relative mouse for games, scancode keyboard, scroll, buttons. Survives UAC/lock desktop switches with Sunshine's //! retry-on-failure model: the thread stays bound to its desktop and only reattaches //! (`OpenInputDesktop`/`SetThreadDesktop`) when `SendInput` reports a short write (the input //! desktop switched) — no per-event reattach overhead. @@ -32,14 +32,10 @@ use windows::Win32::UI::Input::KeyboardAndMouse::{ MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK, MOUSEEVENTF_WHEEL, MOUSEEVENTF_XDOWN, MOUSEEVENTF_XUP, MOUSEINPUT, VIRTUAL_KEY, }; -use windows::Win32::UI::WindowsAndMessaging::{ - GetForegroundWindow, GetSystemMetrics, GetWindowThreadProcessId, SM_CXVIRTUALSCREEN, - SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, -}; +use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId}; use super::InputInjector; -const ABS_MAX: f64 = 65535.0; // SendInput absolute coords are 0..65535 over the chosen surface. const GENERIC_ALL: u32 = 0x1000_0000; const XBUTTON1: u32 = 0x0001; const XBUTTON2: u32 = 0x0002; @@ -165,13 +161,16 @@ impl InputInjector for SendInputInjector { if w == 0 || h == 0 { return Ok(()); // contract: drop zero extent } - let (_vx, _vy, vw, vh) = virtual_desktop_rect(); - // One virtual output spanning the virtual desktop: map client (0..w,0..h) -> 0..65535. + // Client (0..w,0..h) → the STREAMED output's desktop rect + // ([`crate::stream_target`]; the whole virtual desktop only as fallback) → + // 0..65535 over the virtual desktop for MOUSEEVENTF_VIRTUALDESK. Mapping over + // the desktop alone is the Extend-topology offset bug the pen plane exposed + // (design/pen-tablet-input.md): correct only when the streamed display IS the + // whole desktop. let cx = (event.x.clamp(0, w as i32)) as f64 / w as f64; let cy = (event.y.clamp(0, h as i32)) as f64 / h as f64; - let ax = (cx * ABS_MAX).round() as i32; - let ay = (cy * ABS_MAX).round() as i32; - let _ = (vw, vh); // virtual-desktop rect reserved for multi-output mapping + let px = crate::stream_target::map_normalized(cx, cy); + let (ax, ay) = crate::stream_target::desktop_px_to_virtualdesk(px); let mi = MOUSEINPUT { dx: ax, dy: ay, @@ -378,19 +377,6 @@ fn key(ki: KEYBDINPUT) -> INPUT { } } -fn virtual_desktop_rect() -> (i32, i32, i32, i32) { - // SAFETY: each `GetSystemMetrics` takes a single by-value `SYSTEM_METRICS_INDEX` constant and - // returns an `i32`; it dereferences no pointer and has no side effects — FFI-`unsafe` only. - unsafe { - ( - GetSystemMetrics(SM_XVIRTUALSCREEN), - GetSystemMetrics(SM_YVIRTUALSCREEN), - GetSystemMetrics(SM_CXVIRTUALSCREEN), - GetSystemMetrics(SM_CYVIRTUALSCREEN), - ) - } -} - // VKs Windows wants flagged extended even when the scancode high bits aren't set: the editing // cluster (Ins/Del/Home/End/PgUp/PgDn = 0x21..0x28, 0x2D, 0x2E), the Win keys (0x5B/0x5C/0x5D), // RCtrl (0xA3), RAlt (0xA5), Pause (0x90). MAPVK_VK_TO_VSC_EX already encodes E0 for most; this is a diff --git a/crates/pf-inject/src/inject/windows/stream_target.rs b/crates/pf-inject/src/inject/windows/stream_target.rs new file mode 100644 index 00000000..4a119ad2 --- /dev/null +++ b/crates/pf-inject/src/inject/windows/stream_target.rs @@ -0,0 +1,181 @@ +//! The streamed display every absolute coordinate maps into (design/pen-tablet-input.md field +//! fix). Pen, touch, and absolute-mouse positions arrive normalized to the STREAMED output's +//! frame, but the injectors historically mapped them over the whole virtual desktop — correct +//! only when the virtual display is the sole active display (Exclusive topology, normalized to +//! origin). In Extend — a physical monitor kept on beside the virtual output, or an Exclusive +//! isolate degraded to the keep-physicals fallback — the streamed output sits at a non-zero +//! origin, so every sample landed shifted and mis-scaled (the pen exposed it first: a stylus is +//! strictly absolute, with no closed-loop correction onto the target like a cursor). +//! +//! The host publishes the streamed output's CCD target id at capture bring-up +//! ([`set_stream_target`]); the mapping sites resolve its CURRENT desktop rect through +//! [`pf_win_display::win_display::source_desktop_rect`] — the same resolver the cursor-readback +//! poller maps frames with, so the two directions always agree — TTL-cached because a +//! group-layout re-arrange moves a live output's origin mid-session. With no target set, or none +//! resolved yet, mapping falls back to the whole virtual desktop: the historical behavior, still +//! correct for Exclusive topology and the client-less devtest paths. + +use std::sync::Mutex; +use std::time::{Duration, Instant}; +use windows::Win32::UI::WindowsAndMessaging::{ + GetSystemMetrics, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, +}; + +/// `(x, y, w, h)` in desktop coordinates, physical pixels (`source_desktop_rect` order). +type Rect = (i32, i32, i32, i32); + +/// How long a resolved rect stays fresh: long enough that the CCD query cost vanishes at input +/// rates (pen samples + the 40 ms refresh threads), short enough that a mid-session layout move +/// (a parallel session joining the auto-row) is picked up within a blink. +const RECT_TTL: Duration = Duration::from_millis(250); + +struct State { + target_id: Option, + rect: Option, + queried: Option, +} + +static STATE: Mutex = Mutex::new(State { + target_id: None, + rect: None, + queried: None, +}); + +/// Publish the streamed output (its CCD target id) that absolute input maps into. The host calls +/// this at capture bring-up; it is never cleared at teardown — a deactivated target simply stops +/// resolving (the last-known rect is kept, and nothing injects between sessions), and the next +/// session's bring-up re-targets. One slot per process: with parallel sessions the LAST bring-up +/// wins for every session's absolute input — per-session routing needs source-tagged input +/// events (parallel-displays plan) and the single slot is never worse than the historical +/// whole-desktop mapping. +pub fn set_stream_target(target_id: Option) { + let mut st = STATE.lock().unwrap(); + if st.target_id != target_id { + tracing::info!(?target_id, "absolute-input stream target set"); + st.target_id = target_id; + st.rect = None; + st.queried = None; + } +} + +/// The streamed output's current desktop rect, TTL-cached. `None` = no target set / never +/// resolved (callers fall back to the whole virtual desktop). +fn stream_rect() -> Option { + let mut st = STATE.lock().unwrap(); + let target_id = st.target_id?; + let fresh = st.queried.is_some_and(|at| at.elapsed() < RECT_TTL); + if !fresh { + st.queried = Some(Instant::now()); + // SAFETY: read-only QueryDisplayConfig over owned locals (`source_desktop_rect`'s + // contract — the same call the cursor-readback poller makes at spawn). + match unsafe { pf_win_display::win_display::source_desktop_rect(target_id) } { + Some(r) => { + if st.rect != Some(r) { + tracing::info!(target_id, rect = ?r, "stream-target desktop rect resolved"); + } + st.rect = Some(r); + } + // Not an active path right now (teardown, or a topology commit in flight): keep the + // last-known rect — snapping mid-stroke to the whole-desktop mapping would visibly + // jump, and after teardown nothing injects until the next session re-targets. + None => { + if st.rect.is_some() { + tracing::debug!( + target_id, + "stream target not an active path — keeping last rect" + ); + } + } + } + } + st.rect +} + +/// Desktop-space pixel for a normalized `[0,1]²` coordinate over the streamed output's rect, +/// falling back to the whole virtual desktop when no stream target is live. +pub(crate) fn map_normalized(nx: f64, ny: f64) -> (i32, i32) { + map_into(stream_rect().unwrap_or_else(virtual_desktop_rect), nx, ny) +} + +/// Pure mapping: `[0,1]²` over `(x, y, w, h)`, inclusive edges (1.0 lands on the last pixel). +fn map_into((x, y, w, h): Rect, nx: f64, ny: f64) -> (i32, i32) { + ( + x + (nx.clamp(0.0, 1.0) * (w - 1).max(0) as f64).round() as i32, + y + (ny.clamp(0.0, 1.0) * (h - 1).max(0) as f64).round() as i32, + ) +} + +/// The virtual-desktop bounds `(x, y, w, h)` — the mapping fallback, and the surface +/// `MOUSEEVENTF_VIRTUALDESK` absolute coordinates normalize over. +pub(crate) fn virtual_desktop_rect() -> Rect { + // SAFETY: each `GetSystemMetrics` takes a single by-value `SYSTEM_METRICS_INDEX` constant and + // returns an `i32`; it dereferences no pointer and has no side effects — FFI-`unsafe` only. + unsafe { + ( + GetSystemMetrics(SM_XVIRTUALSCREEN), + GetSystemMetrics(SM_YVIRTUALSCREEN), + GetSystemMetrics(SM_CXVIRTUALSCREEN).max(1), + GetSystemMetrics(SM_CYVIRTUALSCREEN).max(1), + ) + } +} + +/// A desktop-space pixel as the `MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK` 0..65535 +/// coordinate pair `SendInput` wants. +pub(crate) fn desktop_px_to_virtualdesk(px: (i32, i32)) -> (i32, i32) { + px_to_abs(virtual_desktop_rect(), px) +} + +/// SendInput absolute coordinates span 0..65535 over the chosen surface. +const ABS_MAX: f64 = 65535.0; + +/// Pure normalization: a desktop pixel inside `(x, y, w, h)` → 0..65535 over that surface. +fn px_to_abs((vx, vy, vw, vh): Rect, (px, py): (i32, i32)) -> (i32, i32) { + ( + ((px - vx) as f64 * ABS_MAX / (vw - 1).max(1) as f64).round() as i32, + ((py - vy) as f64 * ABS_MAX / (vh - 1).max(1) as f64).round() as i32, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The Extend-topology field bug: physical 1920x1080 at (0,0), streamed virtual 2560x1440 + /// beside it at (1920,0) — samples must land inside the VIRTUAL output, not at the desktop + /// origin. + #[test] + fn maps_over_the_streamed_rect_not_the_desktop() { + let r = (1920, 0, 2560, 1440); + assert_eq!(map_into(r, 0.0, 0.0), (1920, 0)); + assert_eq!(map_into(r, 1.0, 1.0), (1920 + 2559, 1439)); + assert_eq!(map_into(r, 0.5, 0.5), (1920 + 1280, 720)); + } + + #[test] + fn clamps_out_of_range_and_handles_negative_origins() { + // An output placed LEFT of / ABOVE the primary has a negative desktop origin. + let r = (-2560, -100, 2560, 1440); + assert_eq!(map_into(r, 0.0, 0.0), (-2560, -100)); + assert_eq!(map_into(r, 2.0, -1.0), (-2560 + 2559, -100)); + } + + #[test] + fn degenerate_rect_pins_to_its_origin() { + assert_eq!(map_into((10, 20, 0, 0), 0.7, 0.7), (10, 20)); + } + + /// The VIRTUALDESK round trip: win32k maps an absolute coordinate back to a pixel roughly as + /// `px = ax * vw / 65536` (floor) — edge pixels and the streamed output's origin must survive. + #[test] + fn virtualdesk_normalization_round_trips() { + let v = (0, 0, 4480, 1080); + assert_eq!(px_to_abs(v, (0, 0)), (0, 0)); + assert_eq!(px_to_abs(v, (4479, 1079)), (65535, 65535)); + let (ax, _) = px_to_abs(v, (1920, 0)); + assert_eq!((ax as i64 * 4480 / 65536) as i32, 1920); + // Negative-origin desktops (a monitor left of the primary) still normalize from 0. + let v = (-2560, 0, 4480, 1440); + assert_eq!(px_to_abs(v, (-2560, 0)), (0, 0)); + } +} diff --git a/crates/pf-inject/src/lib.rs b/crates/pf-inject/src/lib.rs index 7394c9d7..3b2deb42 100644 --- a/crates/pf-inject/src/lib.rs +++ b/crates/pf-inject/src/lib.rs @@ -417,6 +417,16 @@ pub mod pen; #[cfg(target_os = "windows")] #[path = "inject/windows/pointer_windows.rs"] pub mod pen; +/// Windows: the streamed output's desktop rect that every absolute coordinate (pen, touch, +/// absolute mouse) maps into — published by the host at capture bring-up, resolved through the +/// CCD source rect (the cursor-readback poller's resolver, so both directions agree). Mapping +/// over the whole virtual desktop instead is the Extend-topology offset bug the pen exposed +/// (design/pen-tablet-input.md). +#[cfg(target_os = "windows")] +#[path = "inject/windows/stream_target.rs"] +pub mod stream_target; +#[cfg(target_os = "windows")] +pub use stream_target::set_stream_target; /// Stub — pen injection needs the Linux uinput tablet or Windows synthetic pointers; /// `pen_supported()` is false here, so no host advertises the cap and no batches arrive. #[cfg(not(any(target_os = "linux", target_os = "windows")))] diff --git a/crates/punktfunk-host/src/capture.rs b/crates/punktfunk-host/src/capture.rs index 04205be2..11f884f2 100644 --- a/crates/punktfunk-host/src/capture.rs +++ b/crates/punktfunk-host/src/capture.rs @@ -124,6 +124,11 @@ pub fn capture_virtual_output( virtual-display warnings above)" ) })?; + // Aim the injectors' absolute mapping (pen/touch/abs-mouse) at THIS display: the wire + // normalizes over the streamed frame, and mapping it over the whole virtual desktop is wrong + // the moment a physical monitor shares the desktop (Extend topology, or an Exclusive isolate + // degraded to the keep-physicals fallback) — the pen-offset field bug. + crate::inject::set_stream_target(Some(target.target_id)); let pref = vout.preferred_mode; let keep = vout.keepalive; // The sealed-channel delivery seam: resolve the pf-vdisplay control device ONCE (it is