From dc375668ee50c5e5afab2afde3f6853e2562370f Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Wed, 10 Jun 2026 22:38:44 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20touch=20input=20=E2=80=94=20TouchDown/M?= =?UTF-8?q?ove/Up=20+=20host=20libei=20ei=5Ftouchscreen=20injection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Roadmap #5 (touch, ahead of the XL UHID DualSense work). Touch fits the existing 18-byte InputEvent: code = touch id, x/y = client pixels, flags = (w<<16)|h — the same absolute mapping as MouseMoveAbs. - core: InputKind::{TouchDown=9, TouchMove=10, TouchUp=11} + from_u8 + roundtrip test. - host inject/libei.rs: request the RemoteDesktop Touchscreen device type, bind the Touch capability, and inject ei_touchscreen down/motion/up (one event = one frame, per the protocol rule), mapping coordinates into the device region like the abs pointer. wlroots has no virtual-touch protocol wired — no-ops there. - client-rs --touch-test: drags a synthetic finger (touch id 0) in a circle. Validated live on headless KWin: the portal GRANTS the Touchscreen device type (Keyboard|Pointer|Touchscreen), proving the request path — but KWin's EIS server creates no touchscreen *device*, so touch currently no-ops on this KWin (now logged once, not silent). The injection code is correct and will land on a backend that exposes ei_touchscreen (gamescope / a newer compositor / the real touch-client path). Workspace green, clippy/fmt clean, +1 unit test. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/punktfunk-client-rs/src/main.rs | 51 ++++++++++++++++++++- crates/punktfunk-core/src/input.rs | 33 ++++++++++++++ crates/punktfunk-host/src/inject/libei.rs | 54 +++++++++++++++++++++-- crates/punktfunk-host/src/inject/wlr.rs | 2 + docs/roadmap.md | 11 +++-- include/punktfunk_core.h | 8 ++++ 6 files changed, 152 insertions(+), 7 deletions(-) diff --git a/crates/punktfunk-client-rs/src/main.rs b/crates/punktfunk-client-rs/src/main.rs index cbc2822..d71f14e 100644 --- a/crates/punktfunk-client-rs/src/main.rs +++ b/crates/punktfunk-client-rs/src/main.rs @@ -9,7 +9,8 @@ //! `--input-test` exercises the input plane: scripted mouse/keyboard datagrams during the //! stream (watch them land in the host session, e.g. xev inside gamescope). `--mic-test` //! exercises the mic uplink: a synthetic 440 Hz tone streamed as Opus (0xCB) → the host's -//! virtual microphone source (record it host-side to hear the tone). +//! virtual microphone source (record it host-side to hear the tone). `--touch-test` drags a +//! synthetic finger in a circle → host libei `ei_touchscreen` injection. //! //! `--pin <64-hex>` pins the host's certificate fingerprint (the host logs it at startup); //! without it the client trusts on first use and prints the observed fingerprint to pin. @@ -41,6 +42,8 @@ struct Args { input_test: bool, /// `--mic-test` — stream a synthetic 440 Hz tone as the mic uplink (proves the mic path). mic_test: bool, + /// `--touch-test` — drag a synthetic finger in a circle (proves the touch path). + touch_test: bool, pin: Option<[u8; 32]>, /// `--remode WxHxFPS:SECS` — request this mode SECS seconds into the stream. remode: Option<(Mode, u32)>, @@ -142,6 +145,7 @@ fn parse_args() -> Args { out: get("--out").map(String::from), input_test: argv.iter().any(|a| a == "--input-test"), mic_test: argv.iter().any(|a| a == "--mic-test"), + touch_test: argv.iter().any(|a| a == "--touch-test"), pin, remode, pair: get("--pair").map(String::from), @@ -396,6 +400,51 @@ async fn session(args: Args) -> Result<()> { }); } + // Touch plane: drag a synthetic finger (touch id 0) in a circle on the client surface, so + // the host injects it via libei ei_touchscreen — proves the touch path end to end. `flags` + // packs the surface w/h; x/y are pixels (the host maps them into the device region). + if args.touch_test { + let conn2 = conn.clone(); + let (w, h) = (args.mode.width, args.mode.height); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + let flags = (w << 16) | (h & 0xffff); + let (cx, cy, r) = (w as f32 / 2.0, h as f32 / 2.0, h as f32 / 4.0); + let touch = |kind, x: f32, y: f32| InputEvent { + kind, + _pad: [0; 3], + code: 0, // touch id 0 + x: x as i32, + y: y as i32, + flags, + }; + tracing::info!("touch-test: dragging a finger in a circle for ~6s"); + for loop_i in 0..3u32 { + let _ = conn2.send_datagram( + touch(InputKind::TouchDown, cx + r, cy) + .encode() + .to_vec() + .into(), + ); + for i in 0..60u32 { + let a = std::f32::consts::TAU * i as f32 / 60.0; + let mv = touch(InputKind::TouchMove, cx + r * a.cos(), cy + r * a.sin()); + let _ = conn2.send_datagram(mv.encode().to_vec().into()); + tokio::time::sleep(std::time::Duration::from_millis(30)).await; + } + let _ = conn2.send_datagram( + touch(InputKind::TouchUp, cx + r, cy) + .encode() + .to_vec() + .into(), + ); + let _ = loop_i; + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + tracing::info!("touch-test: done"); + }); + } + // Closed-flag for the blocking receive loop. let closed = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); { diff --git a/crates/punktfunk-core/src/input.rs b/crates/punktfunk-core/src/input.rs index 84a38ca..f56755f 100644 --- a/crates/punktfunk-core/src/input.rs +++ b/crates/punktfunk-core/src/input.rs @@ -29,6 +29,14 @@ pub enum InputKind { /// Sticks are i16 range (−32768..32767) in the XInput/Moonlight convention — **+y = /// up** (unlike mouse coordinates); triggers 0..255. GamepadAxis = 8, + /// Touch begins. `code` = touch id (which finger; reusable after `TouchUp`), `x`/`y` = + /// pixel coordinates and `flags` = `(width << 16) | height` of the client's touch surface + /// — the same absolute mapping as [`MouseMoveAbs`](Self::MouseMoveAbs). + TouchDown = 9, + /// Touch moves. Same field meaning as [`TouchDown`](Self::TouchDown). + TouchMove = 10, + /// Touch ends. Only `code` (the touch id) is used. + TouchUp = 11, } /// The gamepad wire contract for [`InputKind::GamepadButton`]/[`InputKind::GamepadAxis`]. @@ -79,6 +87,9 @@ impl InputKind { 6 => MouseScroll, 7 => GamepadButton, 8 => GamepadAxis, + 9 => TouchDown, + 10 => TouchMove, + 11 => TouchUp, _ => return None, }) } @@ -148,4 +159,26 @@ mod tests { assert_eq!(InputEvent::decode(&e.encode()), Some(e)); assert!(InputEvent::decode(&[0u8; INPUT_WIRE_LEN]).is_none()); // bad magic } + + #[test] + fn touch_kinds_roundtrip() { + for kind in [ + InputKind::TouchDown, + InputKind::TouchMove, + InputKind::TouchUp, + ] { + assert_eq!(InputKind::from_u8(kind as u8), Some(kind)); + let e = InputEvent { + kind, + _pad: [0; 3], + code: 2, // touch id + x: 640, + y: 360, + flags: (1280u32 << 16) | 720, // client surface w/h + }; + assert_eq!(InputEvent::decode(&e.encode()), Some(e)); + } + // 12 (one past TouchUp) is not a valid kind. + assert_eq!(InputKind::from_u8(12), None); + } } diff --git a/crates/punktfunk-host/src/inject/libei.rs b/crates/punktfunk-host/src/inject/libei.rs index 0ca5b90..35accba 100644 --- a/crates/punktfunk-host/src/inject/libei.rs +++ b/crates/punktfunk-host/src/inject/libei.rs @@ -178,7 +178,7 @@ async fn connect_portal() -> Result<( rd.select_devices( &session, SelectDevicesOptions::default() - .set_devices(DeviceType::Keyboard | DeviceType::Pointer) + .set_devices(DeviceType::Keyboard | DeviceType::Pointer | DeviceType::Touchscreen) .set_persist_mode(PersistMode::DoNot), ) .await @@ -263,7 +263,8 @@ impl EiState { | DeviceCapability::PointerAbsolute | DeviceCapability::Keyboard | DeviceCapability::Scroll - | DeviceCapability::Button, + | DeviceCapability::Button + | DeviceCapability::Touch, ); let _ = ctx.flush(); } @@ -321,10 +322,31 @@ impl EiState { InputKind::MouseButtonDown | InputKind::MouseButtonUp => DeviceCapability::Button, InputKind::MouseScroll => DeviceCapability::Scroll, InputKind::KeyDown | InputKind::KeyUp => DeviceCapability::Keyboard, + InputKind::TouchDown | InputKind::TouchMove | InputKind::TouchUp => { + DeviceCapability::Touch + } InputKind::GamepadButton | InputKind::GamepadAxis => return, // uinput path (later) }; let Some(idx) = self.device_for(cap) else { - return; // no resumed device with this capability yet + // No resumed device with this capability yet. For touch this is usually permanent on + // this compositor — the RemoteDesktop portal may grant the Touchscreen *device type* + // while the EIS server never creates a touchscreen *device* (observed on headless + // KWin). Surface it once so touch silently going nowhere is diagnosable. + if matches!( + ev.kind, + InputKind::TouchDown | InputKind::TouchMove | InputKind::TouchUp + ) { + static WARNED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + if !WARNED.swap(true, std::sync::atomic::Ordering::Relaxed) { + tracing::warn!( + "touch received but the compositor's EIS exposed no touchscreen device — \ + touch is dropped (KWin's libei may not implement ei_touchscreen yet; \ + gamescope / a newer compositor may)" + ); + } + } + return; }; let dev = self.devices[idx].device.device().clone(); self.ensure_emulating(idx, &dev); @@ -398,6 +420,32 @@ impl EiState { } } } + // Touch: `code` is the touch id, `x`/`y` are client pixels and `flags` packs the + // client surface w/h — mapped into the device's region exactly like MouseMoveAbs. + // One InputEvent = one frame, which satisfies the ei_touchscreen rule that a down / + // motion / up must not share a frame. + InputKind::TouchDown | InputKind::TouchMove => { + let w = ((ev.flags >> 16) & 0xffff) as f32; + let h = (ev.flags & 0xffff) as f32; + match (slot.interface::(), slot.regions().first()) { + (Some(t), Some(region)) if w > 0.0 && h > 0.0 => { + let nx = (ev.x as f32 / w).clamp(0.0, 1.0); + let ny = (ev.y as f32 / h).clamp(0.0, 1.0); + let x = region.x as f32 + nx * region.width as f32; + let y = region.y as f32 + ny * region.height as f32; + if ev.kind == InputKind::TouchDown { + t.down(ev.code, x, y); + } else { + t.motion(ev.code, x, y); + } + } + _ => emitted = false, + } + } + InputKind::TouchUp => match slot.interface::() { + Some(t) => t.up(ev.code), + None => emitted = false, + }, InputKind::GamepadButton | InputKind::GamepadAxis => emitted = false, } diff --git a/crates/punktfunk-host/src/inject/wlr.rs b/crates/punktfunk-host/src/inject/wlr.rs index e28c596..6b21e50 100644 --- a/crates/punktfunk-host/src/inject/wlr.rs +++ b/crates/punktfunk-host/src/inject/wlr.rs @@ -249,6 +249,8 @@ impl InputInjector for WlrootsInjector { } } InputKind::GamepadButton | InputKind::GamepadAxis => {} // not yet injected + // wlroots has no virtual-touch protocol wired here; touch is the libei path only. + InputKind::TouchDown | InputKind::TouchMove | InputKind::TouchUp => {} } // Surface protocol errors / disconnects, then push the batch to the compositor. self.queue diff --git a/docs/roadmap.md b/docs/roadmap.md index cf27e3b..092aedb 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -69,9 +69,14 @@ select = a `pw_stream` with `Direction::Output` + `media.class=Audio/Source`. ## 5. Touch + rich DualSense *(decision: commit to full UHID DualSense)* -- **Touch (M):** `reis` already exposes `ei_touchscreen` — add Touch InputKinds + wire - `ei::Touchscreen` in `inject/libei.rs` (reuse the abs-pointer region mapping). Multi-touch - on KWin/Mutter; single-pointer fallback elsewhere. +- **Touch — implemented (host path), pending a backend that lands it.** `TouchDown/Move/Up` + InputKinds (reuse the abs-pointer `flags=(w<<16)|h` mapping, `code`=touch id); host + `inject/libei.rs` requests the `Touchscreen` device type + binds the `Touch` capability and + injects `ei_touchscreen` down/motion/up; `punktfunk-client-rs --touch-test` drags a finger. + **Validated:** KWin's RemoteDesktop portal *grants* the Touchscreen device type, but its EIS + server creates **no touchscreen device** (headless KWin) — so touch currently no-ops on KWin + (now logged once). The code is correct; it needs a backend that exposes `ei_touchscreen` + (gamescope / newer KWin / the real iPad client path) to land. wlroots: no virtual-touch wired. - **Rich DualSense (XL, committed):** uinput can't carry HID output reports. Use **UHID** + the kernel `hid-playstation` driver (the inputtino/Wolf approach): present a genuine DualSense (real report descriptor, vendor 054C/0CE6, BT mode + CRC32) so games drive LED + diff --git a/include/punktfunk_core.h b/include/punktfunk_core.h index 88a4cf8..df6ed8f 100644 --- a/include/punktfunk_core.h +++ b/include/punktfunk_core.h @@ -201,6 +201,14 @@ enum PunktfunkInputKind // Sticks are i16 range (−32768..32767) in the XInput/Moonlight convention — **+y = // up** (unlike mouse coordinates); triggers 0..255. PUNKTFUNK_INPUT_KIND_GAMEPAD_AXIS = 8, + // Touch begins. `code` = touch id (which finger; reusable after `TouchUp`), `x`/`y` = + // pixel coordinates and `flags` = `(width << 16) | height` of the client's touch surface + // — the same absolute mapping as [`MouseMoveAbs`](Self::MouseMoveAbs). + PUNKTFUNK_INPUT_KIND_TOUCH_DOWN = 9, + // Touch moves. Same field meaning as [`TouchDown`](Self::TouchDown). + PUNKTFUNK_INPUT_KIND_TOUCH_MOVE = 10, + // Touch ends. Only `code` (the touch id) is used. + PUNKTFUNK_INPUT_KIND_TOUCH_UP = 11, }; #ifndef __cplusplus #if __STDC_VERSION__ >= 202311L