diff --git a/Cargo.lock b/Cargo.lock index 9192bef..8b4d284 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -932,6 +932,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -939,6 +954,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -947,6 +963,17 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.32" @@ -995,9 +1022,13 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -1468,6 +1499,7 @@ dependencies = [ "axum-server", "cbc", "ffmpeg-next", + "futures-util", "hex", "libc", "lumen-core", @@ -1476,6 +1508,7 @@ dependencies = [ "pipewire", "rand 0.8.6", "rcgen", + "reis", "rsa", "rustls", "rustls-pemfile", @@ -2172,6 +2205,19 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "reis" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aee09758db35e73eb1398c6ef321973ce7f4aad86573ba08c1c8ef98d350b46" +dependencies = [ + "enumflags2", + "futures", + "log", + "rustix", + "tokio", +] + [[package]] name = "ring" version = "0.17.14" diff --git a/crates/lumen-host/Cargo.toml b/crates/lumen-host/Cargo.toml index 6ebbdd1..b487dfb 100644 --- a/crates/lumen-host/Cargo.toml +++ b/crates/lumen-host/Cargo.toml @@ -31,10 +31,11 @@ rustls-pemfile = "2" rusty_enet = "0.4" [target.'cfg(target_os = "linux")'.dependencies] -# `screencast` gates the ScreenCast portal module; `tokio` is the default runtime. +# `screencast` gates the ScreenCast portal module; `remote_desktop` adds the RemoteDesktop +# portal we use for libei input on KWin/GNOME; `tokio` is the default runtime. # `open_pipe_wire_remote` is unconditional, so ashpd's own `pipewire` feature is not # needed — we drive PipeWire with the `pipewire` crate below. -ashpd = { version = "0.13", features = ["screencast"] } +ashpd = { version = "0.13", features = ["screencast", "remote_desktop"] } ffmpeg-next = "7" libc = "0.2" # Must match the pipewire crate ashpd 0.13 links (libspa/pipewire-sys `links` key is @@ -52,3 +53,8 @@ wayland-protocols-misc = { version = "0.3", features = ["client"] } xkbcommon = "0.8" # Opus encode for the GameStream audio stream (links system libopus). opus = "0.3" +# libei (EI sender) for the portable input path on KWin/GNOME (RemoteDesktop portal). +# The `tokio` feature wires reis's event stream into tokio's reactor. +reis = { version = "0.6.1", features = ["tokio"] } +# `StreamExt::next` on reis's tokio event stream in the libei worker loop. +futures-util = "0.3" diff --git a/crates/lumen-host/src/gamestream/control.rs b/crates/lumen-host/src/gamestream/control.rs index 9688280..3584cbe 100644 --- a/crates/lumen-host/src/gamestream/control.rs +++ b/crates/lumen-host/src/gamestream/control.rs @@ -146,10 +146,16 @@ fn on_receive( return; // keepalive / QoS / unhandled input kind } - // Open the injector on demand — by the first input event Sway's Wayland socket is up. + // Open the injector on demand — by the first input event the compositor session is up. + // Backend auto-selects per desktop (wlr on Sway, libei on KWin/GNOME); override with + // LUMEN_INPUT_BACKEND. if injector.is_none() { - match crate::inject::open(crate::inject::Backend::WlrVirtual) { - Ok(i) => *injector = Some(i), + let backend = crate::inject::default_backend(); + match crate::inject::open(backend) { + Ok(i) => { + tracing::info!(?backend, "input injection backend opened"); + *injector = Some(i); + } Err(e) => { tracing::error!(error = %format!("{e:#}"), "input injection unavailable"); return; diff --git a/crates/lumen-host/src/inject.rs b/crates/lumen-host/src/inject.rs index 6089752..2bd0675 100644 --- a/crates/lumen-host/src/inject.rs +++ b/crates/lumen-host/src/inject.rs @@ -42,7 +42,42 @@ pub fn open(backend: Backend) -> Result> { anyhow::bail!("wlroots virtual input requires Linux + a Wayland compositor") } } - other => anyhow::bail!("injection backend {other:?} not implemented; use WlrVirtual"), + Backend::Libei => { + #[cfg(target_os = "linux")] + { + Ok(Box::new(libei::LibeiInjector::open()?)) + } + #[cfg(not(target_os = "linux"))] + { + anyhow::bail!("libei input requires Linux + a RemoteDesktop portal") + } + } + other => anyhow::bail!("injection backend {other:?} not implemented"), + } +} + +/// Pick the injection backend for the current session. wlroots/Sway only implements the +/// ScreenCast portal (no RemoteDesktop), so libei can't run there — use the wlr virtual-input +/// protocols. KWin and GNOME implement RemoteDesktop but not the wlr protocols, so use libei. +/// `LUMEN_INPUT_BACKEND=wlr|libei` overrides the auto-detection. +pub fn default_backend() -> Backend { + if let Ok(v) = std::env::var("LUMEN_INPUT_BACKEND") { + match v.trim().to_ascii_lowercase().as_str() { + "wlr" | "wlroots" | "wlrvirtual" => return Backend::WlrVirtual, + "libei" | "ei" | "portal" => return Backend::Libei, + "uinput" => return Backend::Uinput, + other => tracing::warn!( + value = other, + "unknown LUMEN_INPUT_BACKEND — auto-detecting" + ), + } + } + let desktop = std::env::var("XDG_CURRENT_DESKTOP").unwrap_or_default(); + let d = desktop.to_ascii_uppercase(); + if d.contains("KDE") || d.contains("GNOME") { + Backend::Libei + } else { + Backend::WlrVirtual } } @@ -195,266 +230,6 @@ fn gs_button_to_evdev(b: u32) -> Option { } #[cfg(target_os = "linux")] -mod wlr { - use super::{gs_button_to_evdev, vk_to_evdev, InputEvent, InputInjector}; - use anyhow::{bail, Context, Result}; - use lumen_core::input::InputKind; - use std::io::Write; - use std::os::fd::{AsFd, FromRawFd}; - use std::time::Instant; - use wayland_client::protocol::{wl_output::WlOutput, wl_pointer, wl_registry, wl_seat::WlSeat}; - use wayland_client::{Connection, Dispatch, EventQueue, Proxy, QueueHandle}; - use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{ - zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1, - zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1, - }; - use wayland_protocols_wlr::virtual_pointer::v1::client::{ - zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1, - zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1, - }; - use xkbcommon::xkb; - - /// `code` value marking a horizontal scroll event (mirrors `gamestream::input`). - const SCROLL_HORIZONTAL: u32 = 1; - - /// Globals bound from the registry (the Wayland dispatch state). - #[derive(Default)] - struct Globals { - pointer_mgr: Option, - keyboard_mgr: Option, - seat: Option, - output: Option, - } - - impl Dispatch for Globals { - fn event( - state: &mut Self, - registry: &wl_registry::WlRegistry, - event: wl_registry::Event, - _: &(), - _: &Connection, - qh: &QueueHandle, - ) { - if let wl_registry::Event::Global { - name, - interface, - version, - } = event - { - match interface.as_str() { - "zwlr_virtual_pointer_manager_v1" => { - state.pointer_mgr = Some(registry.bind(name, version.min(2), qh, ())); - } - "zwp_virtual_keyboard_manager_v1" => { - state.keyboard_mgr = Some(registry.bind(name, version.min(1), qh, ())); - } - "wl_seat" => { - state.seat = Some(registry.bind(name, version.min(7), qh, ())); - } - "wl_output" if state.output.is_none() => { - state.output = Some(registry.bind(name, version.min(3), qh, ())); - } - _ => {} - } - } - } - } - - // The managers, the two virtual devices, the seat and the output emit no events we use. - macro_rules! ignore_events { - ($($t:ty),* $(,)?) => {$( - impl Dispatch<$t, ()> for Globals { - fn event(_: &mut Self, _: &$t, _: <$t as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle) {} - } - )*}; - } - ignore_events!( - WlSeat, - WlOutput, - ZwlrVirtualPointerManagerV1, - ZwlrVirtualPointerV1, - ZwpVirtualKeyboardManagerV1, - ZwpVirtualKeyboardV1, - ); - - pub struct WlrootsInjector { - conn: Connection, - queue: EventQueue, - globals: Globals, - pointer: ZwlrVirtualPointerV1, - keyboard: ZwpVirtualKeyboardV1, - xkb_state: xkb::State, - _keymap_file: std::fs::File, // keep the memfd alive for the compositor's mmap - start: Instant, - } - - impl WlrootsInjector { - pub fn open() -> Result { - let conn = Connection::connect_to_env().context( - "connect to Wayland (is Sway up + WAYLAND_DISPLAY/XDG_RUNTIME_DIR set?)", - )?; - let mut queue = conn.new_event_queue(); - let qh = queue.handle(); - let _registry = conn.display().get_registry(&qh, ()); - let mut globals = Globals::default(); - queue - .roundtrip(&mut globals) - .context("Wayland registry roundtrip")?; - - let pointer_mgr = globals - .pointer_mgr - .clone() - .context("compositor lacks zwlr_virtual_pointer_manager_v1")?; - let keyboard_mgr = globals - .keyboard_mgr - .clone() - .context("compositor lacks zwp_virtual_keyboard_manager_v1")?; - let seat = globals - .seat - .clone() - .context("compositor advertised no wl_seat")?; - - let pointer = pointer_mgr.create_virtual_pointer_with_output( - Some(&seat), - globals.output.as_ref(), - &qh, - (), - ); - let keyboard = keyboard_mgr.create_virtual_keyboard(&seat, &qh, ()); - - // A standard evdev/US keymap so raw evdev keycodes resolve to the right keysyms. - let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); - let keymap = xkb::Keymap::new_from_names( - &ctx, - "evdev", - "pc105", - "us", - "", - None, - xkb::KEYMAP_COMPILE_NO_FLAGS, - ) - .context("compile xkb keymap")?; - let keymap_str = keymap.get_as_string(xkb::KEYMAP_FORMAT_TEXT_V1); - let xkb_state = xkb::State::new(&keymap); - - let file = memfd_with(&keymap_str)?; - let size = keymap_str.len() as u32 + 1; // include the trailing NUL - keyboard.keymap(1 /* XKB_V1 */, file.as_fd(), size); - queue - .roundtrip(&mut globals) - .context("keymap upload roundtrip")?; - conn.flush().ok(); - - tracing::info!( - output = globals.output.is_some(), - "wlroots virtual input ready (pointer + keyboard)" - ); - Ok(Self { - conn, - queue, - globals, - pointer, - keyboard, - xkb_state, - _keymap_file: file, - start: Instant::now(), - }) - } - - fn now_ms(&self) -> u32 { - self.start.elapsed().as_millis() as u32 - } - - /// Update xkb state for a key and tell the compositor the resulting modifier mask. - fn send_modifiers(&mut self, evdev: u16, down: bool) { - let kc = xkb::Keycode::new(evdev as u32 + 8); // evdev -> xkb keycode - let dir = if down { - xkb::KeyDirection::Down - } else { - xkb::KeyDirection::Up - }; - self.xkb_state.update_key(kc, dir); - let depressed = self.xkb_state.serialize_mods(xkb::STATE_MODS_DEPRESSED); - let latched = self.xkb_state.serialize_mods(xkb::STATE_MODS_LATCHED); - let locked = self.xkb_state.serialize_mods(xkb::STATE_MODS_LOCKED); - let group = self.xkb_state.serialize_layout(xkb::STATE_LAYOUT_EFFECTIVE); - self.keyboard.modifiers(depressed, latched, locked, group); - } - } - - impl InputInjector for WlrootsInjector { - fn inject(&mut self, event: &InputEvent) -> Result<()> { - let t = self.now_ms(); - match event.kind { - InputKind::MouseMove => { - self.pointer.motion(t, event.x as f64, event.y as f64); - self.pointer.frame(); - } - InputKind::MouseMoveAbs => { - let w = (event.flags >> 16) & 0xffff; - let h = event.flags & 0xffff; - if w > 0 && h > 0 { - let x = event.x.clamp(0, w as i32) as u32; - let y = event.y.clamp(0, h as i32) as u32; - self.pointer.motion_absolute(t, x, y, w, h); - self.pointer.frame(); - } - } - InputKind::MouseButtonDown | InputKind::MouseButtonUp => { - if let Some(btn) = gs_button_to_evdev(event.code) { - let st = if event.kind == InputKind::MouseButtonDown { - wl_pointer::ButtonState::Pressed - } else { - wl_pointer::ButtonState::Released - }; - self.pointer.button(t, btn, st); - self.pointer.frame(); - } - } - InputKind::MouseScroll => { - let axis = if event.code == SCROLL_HORIZONTAL { - wl_pointer::Axis::HorizontalScroll - } else { - wl_pointer::Axis::VerticalScroll - }; - // GameStream sends WHEEL_DELTA(120)-scaled units; a notch ≈ 15px. Positive - // GameStream = scroll up, which is negative on the Wayland axis. - let notches = event.x as f64 / 120.0; - self.pointer.axis_source(wl_pointer::AxisSource::Wheel); - self.pointer.axis(t, axis, -notches * 15.0); - self.pointer.frame(); - } - InputKind::KeyDown | InputKind::KeyUp => { - let down = event.kind == InputKind::KeyDown; - if let Some(evdev) = vk_to_evdev(event.code as u8) { - self.keyboard.key(t, evdev as u32, if down { 1 } else { 0 }); - self.send_modifiers(evdev, down); - } else { - tracing::debug!(vk = event.code, "unmapped VK keycode — dropped"); - } - } - InputKind::GamepadButton | InputKind::GamepadAxis => {} // not yet injected - } - // Surface protocol errors / disconnects, then push the batch to the compositor. - self.queue - .dispatch_pending(&mut self.globals) - .context("wayland dispatch")?; - self.conn.flush().context("wayland flush")?; - Ok(()) - } - } - - /// Create an anonymous in-memory file holding `s` + a trailing NUL (for the keymap fd). - fn memfd_with(s: &str) -> Result { - let name = b"lumen-keymap\0"; - let fd = - unsafe { libc::memfd_create(name.as_ptr() as *const libc::c_char, libc::MFD_CLOEXEC) }; - if fd < 0 { - bail!("memfd_create failed: {}", std::io::Error::last_os_error()); - } - let mut f = unsafe { std::fs::File::from_raw_fd(fd) }; - f.write_all(s.as_bytes()).context("write keymap")?; - f.write_all(&[0]).context("write keymap NUL")?; - Ok(f) - } -} +mod libei; +#[cfg(target_os = "linux")] +mod wlr; diff --git a/crates/lumen-host/src/inject/libei.rs b/crates/lumen-host/src/inject/libei.rs new file mode 100644 index 0000000..0aaaa00 --- /dev/null +++ b/crates/lumen-host/src/inject/libei.rs @@ -0,0 +1,354 @@ +//! libei input injection via the RemoteDesktop portal — the portable path for KWin and +//! GNOME/Mutter, which (unlike wlroots/Sway) implement `org.freedesktop.portal.RemoteDesktop` +//! and route emulated input through libei/EIS rather than the wlr virtual-input protocols. +//! +//! We use `ashpd` to open a RemoteDesktop session and obtain the EIS socket fd, then `reis` to +//! drive it as an EI *sender*: bind the seat's pointer/keyboard/scroll/button capabilities and, +//! per device, `start_emulating` → emit → `frame`. The portal session and the EIS connection +//! must stay alive and the event stream must be polled continuously (resume/pause/ping/modifier +//! traffic), so the whole thing runs on a dedicated thread with its own tokio runtime; the +//! synchronous control thread reaches it through an unbounded channel and [`LibeiInjector::inject`] +//! merely enqueues. +//! +//! Keyboard codes are Linux evdev (the same space our VK→evdev table produces) and the +//! compositor supplies the keymap, so — unlike the wlr path — there is no keymap to upload and +//! no modifier mask to serialize: pressing the modifier *keys* (which Moonlight sends as normal +//! key events) is enough. + +use super::{gs_button_to_evdev, vk_to_evdev, InputInjector}; +use anyhow::{anyhow, Result}; +use ashpd::desktop::{ + remote_desktop::{ + ConnectToEISOptions, DeviceType, RemoteDesktop, SelectDevicesOptions, StartOptions, + }, + CreateSessionOptions, PersistMode, +}; +use futures_util::StreamExt; +use lumen_core::input::{InputEvent, InputKind}; +use reis::ei; +use reis::event::{DeviceCapability, EiEvent}; +use std::os::unix::net::UnixStream; +use std::time::{Duration, Instant}; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; + +/// `code` value marking a horizontal scroll event (mirrors `gamestream::input`). +const SCROLL_HORIZONTAL: u32 = 1; + +/// Handle held by the control thread; forwards events to the libei worker thread. +pub struct LibeiInjector { + tx: UnboundedSender, +} + +impl LibeiInjector { + pub fn open() -> Result { + let (tx, rx) = unbounded_channel::(); + std::thread::Builder::new() + .name("lumen-libei".into()) + .spawn(move || worker(rx)) + .map_err(|e| anyhow!("spawn libei worker thread: {e}"))?; + // Return immediately — the portal handshake must NOT run on the caller's (control) + // thread, or a slow/denied portal would freeze the ENet control stream and drop the + // client. The worker establishes the session asynchronously and logs its status; + // events enqueue until devices resume (a few startup events may be dropped). + Ok(Self { tx }) + } +} + +impl InputInjector for LibeiInjector { + fn inject(&mut self, event: &InputEvent) -> Result<()> { + self.tx + .send(*event) + .map_err(|_| anyhow!("libei worker thread has exited")) + } +} + +/// Worker thread entry: build a tokio runtime and run the session to completion. +fn worker(rx: UnboundedReceiver) { + let rt = match tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + { + Ok(rt) => rt, + Err(e) => { + tracing::error!(error = %e, "libei: build tokio runtime failed"); + return; + } + }; + rt.block_on(session_main(rx)); +} + +/// Open the portal + EIS (bounded), then pump events until disconnect or shutdown. +async fn session_main(mut rx: UnboundedReceiver) { + // Keep `_rd`/`_session` bound for the whole loop — dropping the portal session closes the + // EIS connection. Bound the setup so a headless approval dialog (un-bypassed grant) can't + // hang the worker forever. + let (_rd, _session, context, mut events) = match tokio::time::timeout( + Duration::from_secs(30), + connect(), + ) + .await + { + Ok(Ok(t)) => t, + Ok(Err(e)) => { + tracing::error!(error = %format!("{e:#}"), "libei: portal/EIS setup failed"); + return; + } + Err(_) => { + tracing::error!( + "libei: portal setup timed out (headless approval needed, or kde-authorized grant not seeded / app-id mismatch)" + ); + return; + } + }; + tracing::info!("libei: EIS connected — awaiting devices"); + + let mut state = EiState::new(); + loop { + tokio::select! { + ei = events.next() => match ei { + Some(Ok(ev)) => state.handle_ei(ev, &context), + Some(Err(e)) => { tracing::warn!(error = %e, "libei: event stream error"); break; } + None => { tracing::info!("libei: EIS disconnected"); break; } + }, + msg = rx.recv() => match msg { + Some(input) => state.inject(&input, &context), + None => { tracing::info!("libei: injector closed — ending session"); break; } + }, + } + } +} + +/// Tie down the verbose tuple the connect step returns. +type Connected = ( + RemoteDesktop, + ashpd::desktop::Session, + ei::Context, + reis::tokio::EiConvertEventStream, +); + +/// Open a RemoteDesktop portal session (pointer + keyboard), connect to EIS, and run the EI +/// sender handshake. Returns the live portal + EI objects. +async fn connect() -> Result { + let rd = RemoteDesktop::new() + .await + .map_err(|e| anyhow!("open RemoteDesktop portal (is xdg-desktop-portal-kde/gnome running and XDG_CURRENT_DESKTOP set?): {e}"))?; + let session = rd + .create_session(CreateSessionOptions::default()) + .await + .map_err(|e| anyhow!("create RemoteDesktop session: {e}"))?; + rd.select_devices( + &session, + SelectDevicesOptions::default() + .set_devices(DeviceType::Keyboard | DeviceType::Pointer) + .set_persist_mode(PersistMode::DoNot), + ) + .await + .map_err(|e| anyhow!("select_devices: {e}"))? + .response() + .map_err(|e| anyhow!("select_devices response: {e}"))?; + let started = rd + .start(&session, None, StartOptions::default()) + .await + .map_err(|e| anyhow!("start RemoteDesktop session: {e}"))?; + let granted = started + .response() + .map_err(|e| anyhow!("RemoteDesktop start denied: {e}"))?; + tracing::info!(devices = ?granted.devices(), "libei: portal granted devices"); + + let fd = rd + .connect_to_eis(&session, ConnectToEISOptions::default()) + .await + .map_err(|e| anyhow!("connect_to_eis (RemoteDesktop portal version < 2?): {e}"))?; + let context = + ei::Context::new(UnixStream::from(fd)).map_err(|e| anyhow!("reis EI context: {e}"))?; + let (_conn, events) = context + .handshake_tokio("lumen-host", ei::handshake::ContextType::Sender) + .await + .map_err(|e| anyhow!("EI handshake: {e}"))?; + + Ok((rd, session, context, events)) +} + +/// One EI device and its emulation state. +struct DeviceSlot { + device: reis::event::Device, + /// The device is resumed (allowed to emit). Devices arrive paused and may pause again. + resumed: bool, + /// We have issued `start_emulating` since the last resume. + emulating: bool, +} + +/// Tracks bound devices + the serial/sequence/timebase the EI protocol requires. +struct EiState { + devices: Vec, + last_serial: u32, + sequence: u32, + start: Instant, +} + +impl EiState { + fn new() -> Self { + Self { + devices: Vec::new(), + last_serial: 0, + sequence: 0, + start: Instant::now(), + } + } + + fn now_us(&self) -> u64 { + self.start.elapsed().as_micros() as u64 + } + + /// Apply a server event: bind capabilities, track devices, and follow resume/pause. + fn handle_ei(&mut self, ev: EiEvent, ctx: &ei::Context) { + match ev { + EiEvent::SeatAdded(e) => { + e.seat.bind_capabilities( + DeviceCapability::Pointer + | DeviceCapability::PointerAbsolute + | DeviceCapability::Keyboard + | DeviceCapability::Scroll + | DeviceCapability::Button, + ); + let _ = ctx.flush(); + } + EiEvent::DeviceAdded(e) => { + tracing::info!(device = ?e.device.name(), ty = ?e.device.device_type(), "libei: device added"); + self.devices.push(DeviceSlot { + device: e.device, + resumed: false, + emulating: false, + }); + } + EiEvent::DeviceRemoved(e) => { + self.devices.retain(|d| d.device != e.device); + } + EiEvent::DeviceResumed(e) => { + self.last_serial = e.serial; + if let Some(d) = self.devices.iter_mut().find(|d| d.device == e.device) { + d.resumed = true; + d.emulating = false; // must re-issue start_emulating after a resume + } + } + EiEvent::DevicePaused(e) => { + if let Some(d) = self.devices.iter_mut().find(|d| d.device == e.device) { + d.resumed = false; + d.emulating = false; + } + } + // Informational: the server reports resulting modifier/group state; we don't set it. + EiEvent::KeyboardModifiers(e) => self.last_serial = e.serial, + _ => {} + } + } + + /// Index of a resumed device exposing `cap`. + fn device_for(&self, cap: DeviceCapability) -> Option { + self.devices + .iter() + .position(|d| d.resumed && d.device.has_capability(cap)) + } + + /// Ensure the device at `idx` is in `start_emulating` state before we emit on it. + fn ensure_emulating(&mut self, idx: usize, dev: &ei::Device) { + if !self.devices[idx].emulating { + dev.start_emulating(self.last_serial, self.sequence); + self.sequence = self.sequence.wrapping_add(1); + self.devices[idx].emulating = true; + } + } + + /// Translate and emit one client input event, committing it as a single `frame`. + fn inject(&mut self, ev: &InputEvent, ctx: &ei::Context) { + let cap = match ev.kind { + InputKind::MouseMove => DeviceCapability::Pointer, + InputKind::MouseMoveAbs => DeviceCapability::PointerAbsolute, + InputKind::MouseButtonDown | InputKind::MouseButtonUp => DeviceCapability::Button, + InputKind::MouseScroll => DeviceCapability::Scroll, + InputKind::KeyDown | InputKind::KeyUp => DeviceCapability::Keyboard, + InputKind::GamepadButton | InputKind::GamepadAxis => return, // uinput path (later) + }; + let Some(idx) = self.device_for(cap) else { + return; // no resumed device with this capability yet + }; + let dev = self.devices[idx].device.device().clone(); + self.ensure_emulating(idx, &dev); + + let mut emitted = true; + let slot = &self.devices[idx].device; + match ev.kind { + InputKind::MouseMove => match slot.interface::() { + Some(p) => p.motion_relative(ev.x as f32, ev.y as f32), + None => emitted = false, + }, + InputKind::MouseMoveAbs => { + let w = ((ev.flags >> 16) & 0xffff) as f32; + let h = (ev.flags & 0xffff) as f32; + match ( + slot.interface::(), + slot.regions().first(), + ) { + (Some(p), Some(region)) if w > 0.0 && h > 0.0 => { + // Map the normalized client position into the device's first region. + let nx = (ev.x as f32 / w).clamp(0.0, 1.0); + let ny = (ev.y as f32 / h).clamp(0.0, 1.0); + let x = region.x as f32 + nx * region.width as f32; + let y = region.y as f32 + ny * region.height as f32; + p.motion_absolute(x, y); + } + _ => emitted = false, + } + } + InputKind::MouseButtonDown | InputKind::MouseButtonUp => { + match (slot.interface::(), gs_button_to_evdev(ev.code)) { + (Some(b), Some(btn)) => { + let st = if ev.kind == InputKind::MouseButtonDown { + ei::button::ButtonState::Press + } else { + ei::button::ButtonState::Released + }; + b.button(btn, st); + } + _ => emitted = false, + } + } + InputKind::MouseScroll => match slot.interface::() { + Some(s) => { + // GameStream sends WHEEL_DELTA(120)-scaled deltas in `x`; ei scroll_discrete + // uses the same 120-per-detent unit. Positive GameStream = up/left, which is + // negative on the ei axis (matches wl_pointer). + if ev.code == SCROLL_HORIZONTAL { + s.scroll_discrete(-ev.x, 0); + } else { + s.scroll_discrete(0, -ev.x); + } + } + None => emitted = false, + }, + InputKind::KeyDown | InputKind::KeyUp => { + match (slot.interface::(), vk_to_evdev(ev.code as u8)) { + (Some(k), Some(evdev)) => { + let st = if ev.kind == InputKind::KeyDown { + ei::keyboard::KeyState::Press + } else { + ei::keyboard::KeyState::Released + }; + k.key(evdev as u32, st); + } + _ => { + emitted = false; + tracing::debug!(vk = ev.code, "libei: unmapped VK keycode — dropped"); + } + } + } + InputKind::GamepadButton | InputKind::GamepadAxis => emitted = false, + } + + if emitted { + dev.frame(self.last_serial, self.now_us()); + } + let _ = ctx.flush(); + } +} diff --git a/crates/lumen-host/src/inject/wlr.rs b/crates/lumen-host/src/inject/wlr.rs new file mode 100644 index 0000000..8700ded --- /dev/null +++ b/crates/lumen-host/src/inject/wlr.rs @@ -0,0 +1,266 @@ +//! Input injection through the wlroots virtual-input Wayland protocols +//! (`zwlr_virtual_pointer_manager_v1` + `zwp_virtual_keyboard_manager_v1`) — the headless-Sway +//! path. We connect as an ordinary Wayland client (the host inherits Sway's +//! `WAYLAND_DISPLAY`/`XDG_RUNTIME_DIR`), bind the two managers, upload a standard evdev/US xkb +//! keymap, and translate events into virtual pointer/keyboard requests, tracking modifier state +//! so the compositor resolves shifted keysyms correctly. + +use super::{gs_button_to_evdev, vk_to_evdev, InputEvent, InputInjector}; +use anyhow::{bail, Context, Result}; +use lumen_core::input::InputKind; +use std::io::Write; +use std::os::fd::{AsFd, FromRawFd}; +use std::time::Instant; +use wayland_client::protocol::{wl_output::WlOutput, wl_pointer, wl_registry, wl_seat::WlSeat}; +use wayland_client::{Connection, Dispatch, EventQueue, Proxy, QueueHandle}; +use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{ + zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1, + zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1, +}; +use wayland_protocols_wlr::virtual_pointer::v1::client::{ + zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1, + zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1, +}; +use xkbcommon::xkb; + +/// `code` value marking a horizontal scroll event (mirrors `gamestream::input`). +const SCROLL_HORIZONTAL: u32 = 1; + +/// Globals bound from the registry (the Wayland dispatch state). +#[derive(Default)] +struct Globals { + pointer_mgr: Option, + keyboard_mgr: Option, + seat: Option, + output: Option, +} + +impl Dispatch for Globals { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { + name, + interface, + version, + } = event + { + match interface.as_str() { + "zwlr_virtual_pointer_manager_v1" => { + state.pointer_mgr = Some(registry.bind(name, version.min(2), qh, ())); + } + "zwp_virtual_keyboard_manager_v1" => { + state.keyboard_mgr = Some(registry.bind(name, version.min(1), qh, ())); + } + "wl_seat" => { + state.seat = Some(registry.bind(name, version.min(7), qh, ())); + } + "wl_output" if state.output.is_none() => { + state.output = Some(registry.bind(name, version.min(3), qh, ())); + } + _ => {} + } + } + } +} + +// The managers, the two virtual devices, the seat and the output emit no events we use. +macro_rules! ignore_events { + ($($t:ty),* $(,)?) => {$( + impl Dispatch<$t, ()> for Globals { + fn event(_: &mut Self, _: &$t, _: <$t as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle) {} + } + )*}; +} +ignore_events!( + WlSeat, + WlOutput, + ZwlrVirtualPointerManagerV1, + ZwlrVirtualPointerV1, + ZwpVirtualKeyboardManagerV1, + ZwpVirtualKeyboardV1, +); + +pub struct WlrootsInjector { + conn: Connection, + queue: EventQueue, + globals: Globals, + pointer: ZwlrVirtualPointerV1, + keyboard: ZwpVirtualKeyboardV1, + xkb_state: xkb::State, + _keymap_file: std::fs::File, // keep the memfd alive for the compositor's mmap + start: Instant, +} + +impl WlrootsInjector { + pub fn open() -> Result { + let conn = Connection::connect_to_env() + .context("connect to Wayland (is Sway up + WAYLAND_DISPLAY/XDG_RUNTIME_DIR set?)")?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut globals = Globals::default(); + queue + .roundtrip(&mut globals) + .context("Wayland registry roundtrip")?; + + let pointer_mgr = globals + .pointer_mgr + .clone() + .context("compositor lacks zwlr_virtual_pointer_manager_v1")?; + let keyboard_mgr = globals + .keyboard_mgr + .clone() + .context("compositor lacks zwp_virtual_keyboard_manager_v1")?; + let seat = globals + .seat + .clone() + .context("compositor advertised no wl_seat")?; + + let pointer = pointer_mgr.create_virtual_pointer_with_output( + Some(&seat), + globals.output.as_ref(), + &qh, + (), + ); + let keyboard = keyboard_mgr.create_virtual_keyboard(&seat, &qh, ()); + + // A standard evdev/US keymap so raw evdev keycodes resolve to the right keysyms. + let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); + let keymap = xkb::Keymap::new_from_names( + &ctx, + "evdev", + "pc105", + "us", + "", + None, + xkb::KEYMAP_COMPILE_NO_FLAGS, + ) + .context("compile xkb keymap")?; + let keymap_str = keymap.get_as_string(xkb::KEYMAP_FORMAT_TEXT_V1); + let xkb_state = xkb::State::new(&keymap); + + let file = memfd_with(&keymap_str)?; + let size = keymap_str.len() as u32 + 1; // include the trailing NUL + keyboard.keymap(1 /* XKB_V1 */, file.as_fd(), size); + queue + .roundtrip(&mut globals) + .context("keymap upload roundtrip")?; + conn.flush().ok(); + + tracing::info!( + output = globals.output.is_some(), + "wlroots virtual input ready (pointer + keyboard)" + ); + Ok(Self { + conn, + queue, + globals, + pointer, + keyboard, + xkb_state, + _keymap_file: file, + start: Instant::now(), + }) + } + + fn now_ms(&self) -> u32 { + self.start.elapsed().as_millis() as u32 + } + + /// Update xkb state for a key and tell the compositor the resulting modifier mask. + fn send_modifiers(&mut self, evdev: u16, down: bool) { + let kc = xkb::Keycode::new(evdev as u32 + 8); // evdev -> xkb keycode + let dir = if down { + xkb::KeyDirection::Down + } else { + xkb::KeyDirection::Up + }; + self.xkb_state.update_key(kc, dir); + let depressed = self.xkb_state.serialize_mods(xkb::STATE_MODS_DEPRESSED); + let latched = self.xkb_state.serialize_mods(xkb::STATE_MODS_LATCHED); + let locked = self.xkb_state.serialize_mods(xkb::STATE_MODS_LOCKED); + let group = self.xkb_state.serialize_layout(xkb::STATE_LAYOUT_EFFECTIVE); + self.keyboard.modifiers(depressed, latched, locked, group); + } +} + +impl InputInjector for WlrootsInjector { + fn inject(&mut self, event: &InputEvent) -> Result<()> { + let t = self.now_ms(); + match event.kind { + InputKind::MouseMove => { + self.pointer.motion(t, event.x as f64, event.y as f64); + self.pointer.frame(); + } + InputKind::MouseMoveAbs => { + let w = (event.flags >> 16) & 0xffff; + let h = event.flags & 0xffff; + if w > 0 && h > 0 { + let x = event.x.clamp(0, w as i32) as u32; + let y = event.y.clamp(0, h as i32) as u32; + self.pointer.motion_absolute(t, x, y, w, h); + self.pointer.frame(); + } + } + InputKind::MouseButtonDown | InputKind::MouseButtonUp => { + if let Some(btn) = gs_button_to_evdev(event.code) { + let st = if event.kind == InputKind::MouseButtonDown { + wl_pointer::ButtonState::Pressed + } else { + wl_pointer::ButtonState::Released + }; + self.pointer.button(t, btn, st); + self.pointer.frame(); + } + } + InputKind::MouseScroll => { + let axis = if event.code == SCROLL_HORIZONTAL { + wl_pointer::Axis::HorizontalScroll + } else { + wl_pointer::Axis::VerticalScroll + }; + // GameStream sends WHEEL_DELTA(120)-scaled units; a notch ≈ 15px. Positive + // GameStream = scroll up, which is negative on the Wayland axis. + let notches = event.x as f64 / 120.0; + self.pointer.axis_source(wl_pointer::AxisSource::Wheel); + self.pointer.axis(t, axis, -notches * 15.0); + self.pointer.frame(); + } + InputKind::KeyDown | InputKind::KeyUp => { + let down = event.kind == InputKind::KeyDown; + if let Some(evdev) = vk_to_evdev(event.code as u8) { + self.keyboard.key(t, evdev as u32, if down { 1 } else { 0 }); + self.send_modifiers(evdev, down); + } else { + tracing::debug!(vk = event.code, "unmapped VK keycode — dropped"); + } + } + InputKind::GamepadButton | InputKind::GamepadAxis => {} // not yet injected + } + // Surface protocol errors / disconnects, then push the batch to the compositor. + self.queue + .dispatch_pending(&mut self.globals) + .context("wayland dispatch")?; + self.conn.flush().context("wayland flush")?; + Ok(()) + } +} + +/// Create an anonymous in-memory file holding `s` + a trailing NUL (for the keymap fd). +fn memfd_with(s: &str) -> Result { + let name = b"lumen-keymap\0"; + let fd = unsafe { libc::memfd_create(name.as_ptr() as *const libc::c_char, libc::MFD_CLOEXEC) }; + if fd < 0 { + bail!("memfd_create failed: {}", std::io::Error::last_os_error()); + } + let mut f = unsafe { std::fs::File::from_raw_fd(fd) }; + f.write_all(s.as_bytes()).context("write keymap")?; + f.write_all(&[0]).context("write keymap NUL")?; + Ok(f) +} diff --git a/crates/lumen-host/src/main.rs b/crates/lumen-host/src/main.rs index 44cf864..654225a 100644 --- a/crates/lumen-host/src/main.rs +++ b/crates/lumen-host/src/main.rs @@ -49,6 +49,9 @@ fn real_main() -> Result<()> { match args.first().map(String::as_str) { // M2 GameStream host control plane (P1.1: mDNS + serverinfo). Some("serve") => gamestream::serve(), + // Standalone input-injection smoke test (no client needed): open the session's input + // backend and inject a scripted mouse/keyboard pattern. Watch a focused app / `wev`. + Some("input-test") => input_test(), // M0 pipeline spike. Some("m0") => m0::run(parse_m0(&args[1..])?), Some("-h") | Some("--help") | Some("help") | None => { @@ -60,6 +63,56 @@ fn real_main() -> Result<()> { } } +/// Inject a scripted mouse + keyboard pattern through the session's input backend (libei on +/// KWin/GNOME, wlr on Sway). Lets us validate input injection without a Moonlight client. +#[cfg(target_os = "linux")] +fn input_test() -> Result<()> { + use lumen_core::input::{InputEvent, InputKind}; + use std::time::Duration; + + let backend = inject::default_backend(); + tracing::info!(?backend, "input-test: opening injector"); + let mut inj = inject::open(backend)?; + // An async backend (libei) needs a moment to establish its portal/EIS session + device + // resume; events injected before then are dropped. + std::thread::sleep(Duration::from_secs(4)); + + let ev = |kind, code, x, y| InputEvent { + kind, + _pad: [0; 3], + code, + x, + y, + flags: 0, + }; + tracing::info!("input-test: injecting a mouse square + 'A'/click taps for ~8s (watch wev / focused app)"); + for i in 0..160u32 { + let (dx, dy) = match (i / 10) % 4 { + 0 => (12, 0), + 1 => (0, 12), + 2 => (-12, 0), + _ => (0, -12), + }; + if let Err(e) = inj.inject(&ev(InputKind::MouseMove, 0, dx, dy)) { + tracing::warn!(error = %format!("{e:#}"), "input-test: inject failed"); + } + if i % 20 == 0 { + let _ = inj.inject(&ev(InputKind::KeyDown, 0x41, 0, 0)); // 'A' + let _ = inj.inject(&ev(InputKind::KeyUp, 0x41, 0, 0)); + let _ = inj.inject(&ev(InputKind::MouseButtonDown, 1, 0, 0)); // left click + let _ = inj.inject(&ev(InputKind::MouseButtonUp, 1, 0, 0)); + } + std::thread::sleep(Duration::from_millis(50)); + } + tracing::info!("input-test: done"); + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn input_test() -> Result<()> { + bail!("input-test requires Linux") +} + fn parse_m0(args: &[String]) -> Result { let mut source = Source::Portal; let mut width = 1920u32;