feat: M2 P1.7 — libei input backend (portable to KWin/GNOME)

Add a second input-injection backend that works on compositors implementing
the org.freedesktop.portal.RemoteDesktop interface (KWin, GNOME/Mutter), where
the wlroots virtual-input protocols are absent. Uses ashpd 0.13 to open a
RemoteDesktop session + EIS fd and reis 0.6.1 to drive it as an EI sender:
bind pointer/keyboard/scroll/button capabilities and, per device,
start_emulating → emit → frame. Runs on a dedicated thread with its own tokio
runtime (the portal session + EIS connection must stay alive and the event
stream must be polled continuously); open() returns immediately so a slow or
denied portal can never freeze the ENet control thread, with events enqueued
over an unbounded channel until devices resume.

Backend now auto-selects per session (inject::default_backend): wlr on Sway,
libei on KDE/GNOME; LUMEN_INPUT_BACKEND overrides. Refactor inject.rs into the
inject/{wlr,libei}.rs layout matching the capture/encode convention. Keyboard
codes are evdev (the same space our VK→evdev table produces) and the compositor
supplies the keymap, so no keymap upload and no modifier serialization — pressing
the modifier keys Moonlight sends is enough.

Add a `lumen-host input-test` subcommand that injects a scripted mouse+keyboard
pattern through the session backend, so input injection can be validated without
a Moonlight client.

Live-validated on headless KWin (Plasma 6.4): mouse motion, left click, and the
'A' key inject correctly and are delivered to the focused client.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-09 13:58:41 +00:00
parent 6de09fd822
commit 03a6a67354
7 changed files with 775 additions and 269 deletions
Generated
+46
View File
@@ -932,6 +932,21 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "futures"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]] [[package]]
name = "futures-channel" name = "futures-channel"
version = "0.3.32" version = "0.3.32"
@@ -939,6 +954,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink",
] ]
[[package]] [[package]]
@@ -947,6 +963,17 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
[[package]]
name = "futures-executor"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]] [[package]]
name = "futures-io" name = "futures-io"
version = "0.3.32" version = "0.3.32"
@@ -995,9 +1022,13 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
dependencies = [ dependencies = [
"futures-channel",
"futures-core", "futures-core",
"futures-io",
"futures-macro", "futures-macro",
"futures-sink",
"futures-task", "futures-task",
"memchr",
"pin-project-lite", "pin-project-lite",
"slab", "slab",
] ]
@@ -1468,6 +1499,7 @@ dependencies = [
"axum-server", "axum-server",
"cbc", "cbc",
"ffmpeg-next", "ffmpeg-next",
"futures-util",
"hex", "hex",
"libc", "libc",
"lumen-core", "lumen-core",
@@ -1476,6 +1508,7 @@ dependencies = [
"pipewire", "pipewire",
"rand 0.8.6", "rand 0.8.6",
"rcgen", "rcgen",
"reis",
"rsa", "rsa",
"rustls", "rustls",
"rustls-pemfile", "rustls-pemfile",
@@ -2172,6 +2205,19 @@ version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
[[package]]
name = "reis"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2aee09758db35e73eb1398c6ef321973ce7f4aad86573ba08c1c8ef98d350b46"
dependencies = [
"enumflags2",
"futures",
"log",
"rustix",
"tokio",
]
[[package]] [[package]]
name = "ring" name = "ring"
version = "0.17.14" version = "0.17.14"
+8 -2
View File
@@ -31,10 +31,11 @@ rustls-pemfile = "2"
rusty_enet = "0.4" rusty_enet = "0.4"
[target.'cfg(target_os = "linux")'.dependencies] [target.'cfg(target_os = "linux")'.dependencies]
# `screencast` gates the ScreenCast portal module; `tokio` is the default runtime. # `screencast` gates the ScreenCast portal module; `remote_desktop` adds the RemoteDesktop
# portal we use for libei input on KWin/GNOME; `tokio` is the default runtime.
# `open_pipe_wire_remote` is unconditional, so ashpd's own `pipewire` feature is not # `open_pipe_wire_remote` is unconditional, so ashpd's own `pipewire` feature is not
# needed — we drive PipeWire with the `pipewire` crate below. # needed — we drive PipeWire with the `pipewire` crate below.
ashpd = { version = "0.13", features = ["screencast"] } ashpd = { version = "0.13", features = ["screencast", "remote_desktop"] }
ffmpeg-next = "7" ffmpeg-next = "7"
libc = "0.2" libc = "0.2"
# Must match the pipewire crate ashpd 0.13 links (libspa/pipewire-sys `links` key is # Must match the pipewire crate ashpd 0.13 links (libspa/pipewire-sys `links` key is
@@ -52,3 +53,8 @@ wayland-protocols-misc = { version = "0.3", features = ["client"] }
xkbcommon = "0.8" xkbcommon = "0.8"
# Opus encode for the GameStream audio stream (links system libopus). # Opus encode for the GameStream audio stream (links system libopus).
opus = "0.3" opus = "0.3"
# libei (EI sender) for the portable input path on KWin/GNOME (RemoteDesktop portal).
# The `tokio` feature wires reis's event stream into tokio's reactor.
reis = { version = "0.6.1", features = ["tokio"] }
# `StreamExt::next` on reis's tokio event stream in the libei worker loop.
futures-util = "0.3"
+9 -3
View File
@@ -146,10 +146,16 @@ fn on_receive(
return; // keepalive / QoS / unhandled input kind return; // keepalive / QoS / unhandled input kind
} }
// Open the injector on demand — by the first input event Sway's Wayland socket is up. // Open the injector on demand — by the first input event the compositor session is up.
// Backend auto-selects per desktop (wlr on Sway, libei on KWin/GNOME); override with
// LUMEN_INPUT_BACKEND.
if injector.is_none() { if injector.is_none() {
match crate::inject::open(crate::inject::Backend::WlrVirtual) { let backend = crate::inject::default_backend();
Ok(i) => *injector = Some(i), match crate::inject::open(backend) {
Ok(i) => {
tracing::info!(?backend, "input injection backend opened");
*injector = Some(i);
}
Err(e) => { Err(e) => {
tracing::error!(error = %format!("{e:#}"), "input injection unavailable"); tracing::error!(error = %format!("{e:#}"), "input injection unavailable");
return; return;
+39 -264
View File
@@ -42,7 +42,42 @@ pub fn open(backend: Backend) -> Result<Box<dyn InputInjector>> {
anyhow::bail!("wlroots virtual input requires Linux + a Wayland compositor") anyhow::bail!("wlroots virtual input requires Linux + a Wayland compositor")
} }
} }
other => anyhow::bail!("injection backend {other:?} not implemented; use WlrVirtual"), Backend::Libei => {
#[cfg(target_os = "linux")]
{
Ok(Box::new(libei::LibeiInjector::open()?))
}
#[cfg(not(target_os = "linux"))]
{
anyhow::bail!("libei input requires Linux + a RemoteDesktop portal")
}
}
other => anyhow::bail!("injection backend {other:?} not implemented"),
}
}
/// Pick the injection backend for the current session. wlroots/Sway only implements the
/// ScreenCast portal (no RemoteDesktop), so libei can't run there — use the wlr virtual-input
/// protocols. KWin and GNOME implement RemoteDesktop but not the wlr protocols, so use libei.
/// `LUMEN_INPUT_BACKEND=wlr|libei` overrides the auto-detection.
pub fn default_backend() -> Backend {
if let Ok(v) = std::env::var("LUMEN_INPUT_BACKEND") {
match v.trim().to_ascii_lowercase().as_str() {
"wlr" | "wlroots" | "wlrvirtual" => return Backend::WlrVirtual,
"libei" | "ei" | "portal" => return Backend::Libei,
"uinput" => return Backend::Uinput,
other => tracing::warn!(
value = other,
"unknown LUMEN_INPUT_BACKEND — auto-detecting"
),
}
}
let desktop = std::env::var("XDG_CURRENT_DESKTOP").unwrap_or_default();
let d = desktop.to_ascii_uppercase();
if d.contains("KDE") || d.contains("GNOME") {
Backend::Libei
} else {
Backend::WlrVirtual
} }
} }
@@ -195,266 +230,6 @@ fn gs_button_to_evdev(b: u32) -> Option<u32> {
} }
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
mod wlr { mod libei;
use super::{gs_button_to_evdev, vk_to_evdev, InputEvent, InputInjector}; #[cfg(target_os = "linux")]
use anyhow::{bail, Context, Result}; mod wlr;
use lumen_core::input::InputKind;
use std::io::Write;
use std::os::fd::{AsFd, FromRawFd};
use std::time::Instant;
use wayland_client::protocol::{wl_output::WlOutput, wl_pointer, wl_registry, wl_seat::WlSeat};
use wayland_client::{Connection, Dispatch, EventQueue, Proxy, QueueHandle};
use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{
zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1,
zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1,
};
use wayland_protocols_wlr::virtual_pointer::v1::client::{
zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1,
zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1,
};
use xkbcommon::xkb;
/// `code` value marking a horizontal scroll event (mirrors `gamestream::input`).
const SCROLL_HORIZONTAL: u32 = 1;
/// Globals bound from the registry (the Wayland dispatch state).
#[derive(Default)]
struct Globals {
pointer_mgr: Option<ZwlrVirtualPointerManagerV1>,
keyboard_mgr: Option<ZwpVirtualKeyboardManagerV1>,
seat: Option<WlSeat>,
output: Option<WlOutput>,
}
impl Dispatch<wl_registry::WlRegistry, ()> for Globals {
fn event(
state: &mut Self,
registry: &wl_registry::WlRegistry,
event: wl_registry::Event,
_: &(),
_: &Connection,
qh: &QueueHandle<Self>,
) {
if let wl_registry::Event::Global {
name,
interface,
version,
} = event
{
match interface.as_str() {
"zwlr_virtual_pointer_manager_v1" => {
state.pointer_mgr = Some(registry.bind(name, version.min(2), qh, ()));
}
"zwp_virtual_keyboard_manager_v1" => {
state.keyboard_mgr = Some(registry.bind(name, version.min(1), qh, ()));
}
"wl_seat" => {
state.seat = Some(registry.bind(name, version.min(7), qh, ()));
}
"wl_output" if state.output.is_none() => {
state.output = Some(registry.bind(name, version.min(3), qh, ()));
}
_ => {}
}
}
}
}
// The managers, the two virtual devices, the seat and the output emit no events we use.
macro_rules! ignore_events {
($($t:ty),* $(,)?) => {$(
impl Dispatch<$t, ()> for Globals {
fn event(_: &mut Self, _: &$t, _: <$t as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}
}
)*};
}
ignore_events!(
WlSeat,
WlOutput,
ZwlrVirtualPointerManagerV1,
ZwlrVirtualPointerV1,
ZwpVirtualKeyboardManagerV1,
ZwpVirtualKeyboardV1,
);
pub struct WlrootsInjector {
conn: Connection,
queue: EventQueue<Globals>,
globals: Globals,
pointer: ZwlrVirtualPointerV1,
keyboard: ZwpVirtualKeyboardV1,
xkb_state: xkb::State,
_keymap_file: std::fs::File, // keep the memfd alive for the compositor's mmap
start: Instant,
}
impl WlrootsInjector {
pub fn open() -> Result<Self> {
let conn = Connection::connect_to_env().context(
"connect to Wayland (is Sway up + WAYLAND_DISPLAY/XDG_RUNTIME_DIR set?)",
)?;
let mut queue = conn.new_event_queue();
let qh = queue.handle();
let _registry = conn.display().get_registry(&qh, ());
let mut globals = Globals::default();
queue
.roundtrip(&mut globals)
.context("Wayland registry roundtrip")?;
let pointer_mgr = globals
.pointer_mgr
.clone()
.context("compositor lacks zwlr_virtual_pointer_manager_v1")?;
let keyboard_mgr = globals
.keyboard_mgr
.clone()
.context("compositor lacks zwp_virtual_keyboard_manager_v1")?;
let seat = globals
.seat
.clone()
.context("compositor advertised no wl_seat")?;
let pointer = pointer_mgr.create_virtual_pointer_with_output(
Some(&seat),
globals.output.as_ref(),
&qh,
(),
);
let keyboard = keyboard_mgr.create_virtual_keyboard(&seat, &qh, ());
// A standard evdev/US keymap so raw evdev keycodes resolve to the right keysyms.
let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS);
let keymap = xkb::Keymap::new_from_names(
&ctx,
"evdev",
"pc105",
"us",
"",
None,
xkb::KEYMAP_COMPILE_NO_FLAGS,
)
.context("compile xkb keymap")?;
let keymap_str = keymap.get_as_string(xkb::KEYMAP_FORMAT_TEXT_V1);
let xkb_state = xkb::State::new(&keymap);
let file = memfd_with(&keymap_str)?;
let size = keymap_str.len() as u32 + 1; // include the trailing NUL
keyboard.keymap(1 /* XKB_V1 */, file.as_fd(), size);
queue
.roundtrip(&mut globals)
.context("keymap upload roundtrip")?;
conn.flush().ok();
tracing::info!(
output = globals.output.is_some(),
"wlroots virtual input ready (pointer + keyboard)"
);
Ok(Self {
conn,
queue,
globals,
pointer,
keyboard,
xkb_state,
_keymap_file: file,
start: Instant::now(),
})
}
fn now_ms(&self) -> u32 {
self.start.elapsed().as_millis() as u32
}
/// Update xkb state for a key and tell the compositor the resulting modifier mask.
fn send_modifiers(&mut self, evdev: u16, down: bool) {
let kc = xkb::Keycode::new(evdev as u32 + 8); // evdev -> xkb keycode
let dir = if down {
xkb::KeyDirection::Down
} else {
xkb::KeyDirection::Up
};
self.xkb_state.update_key(kc, dir);
let depressed = self.xkb_state.serialize_mods(xkb::STATE_MODS_DEPRESSED);
let latched = self.xkb_state.serialize_mods(xkb::STATE_MODS_LATCHED);
let locked = self.xkb_state.serialize_mods(xkb::STATE_MODS_LOCKED);
let group = self.xkb_state.serialize_layout(xkb::STATE_LAYOUT_EFFECTIVE);
self.keyboard.modifiers(depressed, latched, locked, group);
}
}
impl InputInjector for WlrootsInjector {
fn inject(&mut self, event: &InputEvent) -> Result<()> {
let t = self.now_ms();
match event.kind {
InputKind::MouseMove => {
self.pointer.motion(t, event.x as f64, event.y as f64);
self.pointer.frame();
}
InputKind::MouseMoveAbs => {
let w = (event.flags >> 16) & 0xffff;
let h = event.flags & 0xffff;
if w > 0 && h > 0 {
let x = event.x.clamp(0, w as i32) as u32;
let y = event.y.clamp(0, h as i32) as u32;
self.pointer.motion_absolute(t, x, y, w, h);
self.pointer.frame();
}
}
InputKind::MouseButtonDown | InputKind::MouseButtonUp => {
if let Some(btn) = gs_button_to_evdev(event.code) {
let st = if event.kind == InputKind::MouseButtonDown {
wl_pointer::ButtonState::Pressed
} else {
wl_pointer::ButtonState::Released
};
self.pointer.button(t, btn, st);
self.pointer.frame();
}
}
InputKind::MouseScroll => {
let axis = if event.code == SCROLL_HORIZONTAL {
wl_pointer::Axis::HorizontalScroll
} else {
wl_pointer::Axis::VerticalScroll
};
// GameStream sends WHEEL_DELTA(120)-scaled units; a notch ≈ 15px. Positive
// GameStream = scroll up, which is negative on the Wayland axis.
let notches = event.x as f64 / 120.0;
self.pointer.axis_source(wl_pointer::AxisSource::Wheel);
self.pointer.axis(t, axis, -notches * 15.0);
self.pointer.frame();
}
InputKind::KeyDown | InputKind::KeyUp => {
let down = event.kind == InputKind::KeyDown;
if let Some(evdev) = vk_to_evdev(event.code as u8) {
self.keyboard.key(t, evdev as u32, if down { 1 } else { 0 });
self.send_modifiers(evdev, down);
} else {
tracing::debug!(vk = event.code, "unmapped VK keycode — dropped");
}
}
InputKind::GamepadButton | InputKind::GamepadAxis => {} // not yet injected
}
// Surface protocol errors / disconnects, then push the batch to the compositor.
self.queue
.dispatch_pending(&mut self.globals)
.context("wayland dispatch")?;
self.conn.flush().context("wayland flush")?;
Ok(())
}
}
/// Create an anonymous in-memory file holding `s` + a trailing NUL (for the keymap fd).
fn memfd_with(s: &str) -> Result<std::fs::File> {
let name = b"lumen-keymap\0";
let fd =
unsafe { libc::memfd_create(name.as_ptr() as *const libc::c_char, libc::MFD_CLOEXEC) };
if fd < 0 {
bail!("memfd_create failed: {}", std::io::Error::last_os_error());
}
let mut f = unsafe { std::fs::File::from_raw_fd(fd) };
f.write_all(s.as_bytes()).context("write keymap")?;
f.write_all(&[0]).context("write keymap NUL")?;
Ok(f)
}
}
+354
View File
@@ -0,0 +1,354 @@
//! libei input injection via the RemoteDesktop portal — the portable path for KWin and
//! GNOME/Mutter, which (unlike wlroots/Sway) implement `org.freedesktop.portal.RemoteDesktop`
//! and route emulated input through libei/EIS rather than the wlr virtual-input protocols.
//!
//! We use `ashpd` to open a RemoteDesktop session and obtain the EIS socket fd, then `reis` to
//! drive it as an EI *sender*: bind the seat's pointer/keyboard/scroll/button capabilities and,
//! per device, `start_emulating` → emit → `frame`. The portal session and the EIS connection
//! must stay alive and the event stream must be polled continuously (resume/pause/ping/modifier
//! traffic), so the whole thing runs on a dedicated thread with its own tokio runtime; the
//! synchronous control thread reaches it through an unbounded channel and [`LibeiInjector::inject`]
//! merely enqueues.
//!
//! Keyboard codes are Linux evdev (the same space our VK→evdev table produces) and the
//! compositor supplies the keymap, so — unlike the wlr path — there is no keymap to upload and
//! no modifier mask to serialize: pressing the modifier *keys* (which Moonlight sends as normal
//! key events) is enough.
use super::{gs_button_to_evdev, vk_to_evdev, InputInjector};
use anyhow::{anyhow, Result};
use ashpd::desktop::{
remote_desktop::{
ConnectToEISOptions, DeviceType, RemoteDesktop, SelectDevicesOptions, StartOptions,
},
CreateSessionOptions, PersistMode,
};
use futures_util::StreamExt;
use lumen_core::input::{InputEvent, InputKind};
use reis::ei;
use reis::event::{DeviceCapability, EiEvent};
use std::os::unix::net::UnixStream;
use std::time::{Duration, Instant};
use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
/// `code` value marking a horizontal scroll event (mirrors `gamestream::input`).
const SCROLL_HORIZONTAL: u32 = 1;
/// Handle held by the control thread; forwards events to the libei worker thread.
pub struct LibeiInjector {
tx: UnboundedSender<InputEvent>,
}
impl LibeiInjector {
pub fn open() -> Result<Self> {
let (tx, rx) = unbounded_channel::<InputEvent>();
std::thread::Builder::new()
.name("lumen-libei".into())
.spawn(move || worker(rx))
.map_err(|e| anyhow!("spawn libei worker thread: {e}"))?;
// Return immediately — the portal handshake must NOT run on the caller's (control)
// thread, or a slow/denied portal would freeze the ENet control stream and drop the
// client. The worker establishes the session asynchronously and logs its status;
// events enqueue until devices resume (a few startup events may be dropped).
Ok(Self { tx })
}
}
impl InputInjector for LibeiInjector {
fn inject(&mut self, event: &InputEvent) -> Result<()> {
self.tx
.send(*event)
.map_err(|_| anyhow!("libei worker thread has exited"))
}
}
/// Worker thread entry: build a tokio runtime and run the session to completion.
fn worker(rx: UnboundedReceiver<InputEvent>) {
let rt = match tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.enable_all()
.build()
{
Ok(rt) => rt,
Err(e) => {
tracing::error!(error = %e, "libei: build tokio runtime failed");
return;
}
};
rt.block_on(session_main(rx));
}
/// Open the portal + EIS (bounded), then pump events until disconnect or shutdown.
async fn session_main(mut rx: UnboundedReceiver<InputEvent>) {
// Keep `_rd`/`_session` bound for the whole loop — dropping the portal session closes the
// EIS connection. Bound the setup so a headless approval dialog (un-bypassed grant) can't
// hang the worker forever.
let (_rd, _session, context, mut events) = match tokio::time::timeout(
Duration::from_secs(30),
connect(),
)
.await
{
Ok(Ok(t)) => t,
Ok(Err(e)) => {
tracing::error!(error = %format!("{e:#}"), "libei: portal/EIS setup failed");
return;
}
Err(_) => {
tracing::error!(
"libei: portal setup timed out (headless approval needed, or kde-authorized grant not seeded / app-id mismatch)"
);
return;
}
};
tracing::info!("libei: EIS connected — awaiting devices");
let mut state = EiState::new();
loop {
tokio::select! {
ei = events.next() => match ei {
Some(Ok(ev)) => state.handle_ei(ev, &context),
Some(Err(e)) => { tracing::warn!(error = %e, "libei: event stream error"); break; }
None => { tracing::info!("libei: EIS disconnected"); break; }
},
msg = rx.recv() => match msg {
Some(input) => state.inject(&input, &context),
None => { tracing::info!("libei: injector closed — ending session"); break; }
},
}
}
}
/// Tie down the verbose tuple the connect step returns.
type Connected = (
RemoteDesktop,
ashpd::desktop::Session<RemoteDesktop>,
ei::Context,
reis::tokio::EiConvertEventStream,
);
/// Open a RemoteDesktop portal session (pointer + keyboard), connect to EIS, and run the EI
/// sender handshake. Returns the live portal + EI objects.
async fn connect() -> Result<Connected> {
let rd = RemoteDesktop::new()
.await
.map_err(|e| anyhow!("open RemoteDesktop portal (is xdg-desktop-portal-kde/gnome running and XDG_CURRENT_DESKTOP set?): {e}"))?;
let session = rd
.create_session(CreateSessionOptions::default())
.await
.map_err(|e| anyhow!("create RemoteDesktop session: {e}"))?;
rd.select_devices(
&session,
SelectDevicesOptions::default()
.set_devices(DeviceType::Keyboard | DeviceType::Pointer)
.set_persist_mode(PersistMode::DoNot),
)
.await
.map_err(|e| anyhow!("select_devices: {e}"))?
.response()
.map_err(|e| anyhow!("select_devices response: {e}"))?;
let started = rd
.start(&session, None, StartOptions::default())
.await
.map_err(|e| anyhow!("start RemoteDesktop session: {e}"))?;
let granted = started
.response()
.map_err(|e| anyhow!("RemoteDesktop start denied: {e}"))?;
tracing::info!(devices = ?granted.devices(), "libei: portal granted devices");
let fd = rd
.connect_to_eis(&session, ConnectToEISOptions::default())
.await
.map_err(|e| anyhow!("connect_to_eis (RemoteDesktop portal version < 2?): {e}"))?;
let context =
ei::Context::new(UnixStream::from(fd)).map_err(|e| anyhow!("reis EI context: {e}"))?;
let (_conn, events) = context
.handshake_tokio("lumen-host", ei::handshake::ContextType::Sender)
.await
.map_err(|e| anyhow!("EI handshake: {e}"))?;
Ok((rd, session, context, events))
}
/// One EI device and its emulation state.
struct DeviceSlot {
device: reis::event::Device,
/// The device is resumed (allowed to emit). Devices arrive paused and may pause again.
resumed: bool,
/// We have issued `start_emulating` since the last resume.
emulating: bool,
}
/// Tracks bound devices + the serial/sequence/timebase the EI protocol requires.
struct EiState {
devices: Vec<DeviceSlot>,
last_serial: u32,
sequence: u32,
start: Instant,
}
impl EiState {
fn new() -> Self {
Self {
devices: Vec::new(),
last_serial: 0,
sequence: 0,
start: Instant::now(),
}
}
fn now_us(&self) -> u64 {
self.start.elapsed().as_micros() as u64
}
/// Apply a server event: bind capabilities, track devices, and follow resume/pause.
fn handle_ei(&mut self, ev: EiEvent, ctx: &ei::Context) {
match ev {
EiEvent::SeatAdded(e) => {
e.seat.bind_capabilities(
DeviceCapability::Pointer
| DeviceCapability::PointerAbsolute
| DeviceCapability::Keyboard
| DeviceCapability::Scroll
| DeviceCapability::Button,
);
let _ = ctx.flush();
}
EiEvent::DeviceAdded(e) => {
tracing::info!(device = ?e.device.name(), ty = ?e.device.device_type(), "libei: device added");
self.devices.push(DeviceSlot {
device: e.device,
resumed: false,
emulating: false,
});
}
EiEvent::DeviceRemoved(e) => {
self.devices.retain(|d| d.device != e.device);
}
EiEvent::DeviceResumed(e) => {
self.last_serial = e.serial;
if let Some(d) = self.devices.iter_mut().find(|d| d.device == e.device) {
d.resumed = true;
d.emulating = false; // must re-issue start_emulating after a resume
}
}
EiEvent::DevicePaused(e) => {
if let Some(d) = self.devices.iter_mut().find(|d| d.device == e.device) {
d.resumed = false;
d.emulating = false;
}
}
// Informational: the server reports resulting modifier/group state; we don't set it.
EiEvent::KeyboardModifiers(e) => self.last_serial = e.serial,
_ => {}
}
}
/// Index of a resumed device exposing `cap`.
fn device_for(&self, cap: DeviceCapability) -> Option<usize> {
self.devices
.iter()
.position(|d| d.resumed && d.device.has_capability(cap))
}
/// Ensure the device at `idx` is in `start_emulating` state before we emit on it.
fn ensure_emulating(&mut self, idx: usize, dev: &ei::Device) {
if !self.devices[idx].emulating {
dev.start_emulating(self.last_serial, self.sequence);
self.sequence = self.sequence.wrapping_add(1);
self.devices[idx].emulating = true;
}
}
/// Translate and emit one client input event, committing it as a single `frame`.
fn inject(&mut self, ev: &InputEvent, ctx: &ei::Context) {
let cap = match ev.kind {
InputKind::MouseMove => DeviceCapability::Pointer,
InputKind::MouseMoveAbs => DeviceCapability::PointerAbsolute,
InputKind::MouseButtonDown | InputKind::MouseButtonUp => DeviceCapability::Button,
InputKind::MouseScroll => DeviceCapability::Scroll,
InputKind::KeyDown | InputKind::KeyUp => DeviceCapability::Keyboard,
InputKind::GamepadButton | InputKind::GamepadAxis => return, // uinput path (later)
};
let Some(idx) = self.device_for(cap) else {
return; // no resumed device with this capability yet
};
let dev = self.devices[idx].device.device().clone();
self.ensure_emulating(idx, &dev);
let mut emitted = true;
let slot = &self.devices[idx].device;
match ev.kind {
InputKind::MouseMove => match slot.interface::<ei::Pointer>() {
Some(p) => p.motion_relative(ev.x as f32, ev.y as f32),
None => emitted = false,
},
InputKind::MouseMoveAbs => {
let w = ((ev.flags >> 16) & 0xffff) as f32;
let h = (ev.flags & 0xffff) as f32;
match (
slot.interface::<ei::PointerAbsolute>(),
slot.regions().first(),
) {
(Some(p), Some(region)) if w > 0.0 && h > 0.0 => {
// Map the normalized client position into the device's first region.
let nx = (ev.x as f32 / w).clamp(0.0, 1.0);
let ny = (ev.y as f32 / h).clamp(0.0, 1.0);
let x = region.x as f32 + nx * region.width as f32;
let y = region.y as f32 + ny * region.height as f32;
p.motion_absolute(x, y);
}
_ => emitted = false,
}
}
InputKind::MouseButtonDown | InputKind::MouseButtonUp => {
match (slot.interface::<ei::Button>(), gs_button_to_evdev(ev.code)) {
(Some(b), Some(btn)) => {
let st = if ev.kind == InputKind::MouseButtonDown {
ei::button::ButtonState::Press
} else {
ei::button::ButtonState::Released
};
b.button(btn, st);
}
_ => emitted = false,
}
}
InputKind::MouseScroll => match slot.interface::<ei::Scroll>() {
Some(s) => {
// GameStream sends WHEEL_DELTA(120)-scaled deltas in `x`; ei scroll_discrete
// uses the same 120-per-detent unit. Positive GameStream = up/left, which is
// negative on the ei axis (matches wl_pointer).
if ev.code == SCROLL_HORIZONTAL {
s.scroll_discrete(-ev.x, 0);
} else {
s.scroll_discrete(0, -ev.x);
}
}
None => emitted = false,
},
InputKind::KeyDown | InputKind::KeyUp => {
match (slot.interface::<ei::Keyboard>(), vk_to_evdev(ev.code as u8)) {
(Some(k), Some(evdev)) => {
let st = if ev.kind == InputKind::KeyDown {
ei::keyboard::KeyState::Press
} else {
ei::keyboard::KeyState::Released
};
k.key(evdev as u32, st);
}
_ => {
emitted = false;
tracing::debug!(vk = ev.code, "libei: unmapped VK keycode — dropped");
}
}
}
InputKind::GamepadButton | InputKind::GamepadAxis => emitted = false,
}
if emitted {
dev.frame(self.last_serial, self.now_us());
}
let _ = ctx.flush();
}
}
+266
View File
@@ -0,0 +1,266 @@
//! Input injection through the wlroots virtual-input Wayland protocols
//! (`zwlr_virtual_pointer_manager_v1` + `zwp_virtual_keyboard_manager_v1`) — the headless-Sway
//! path. We connect as an ordinary Wayland client (the host inherits Sway's
//! `WAYLAND_DISPLAY`/`XDG_RUNTIME_DIR`), bind the two managers, upload a standard evdev/US xkb
//! keymap, and translate events into virtual pointer/keyboard requests, tracking modifier state
//! so the compositor resolves shifted keysyms correctly.
use super::{gs_button_to_evdev, vk_to_evdev, InputEvent, InputInjector};
use anyhow::{bail, Context, Result};
use lumen_core::input::InputKind;
use std::io::Write;
use std::os::fd::{AsFd, FromRawFd};
use std::time::Instant;
use wayland_client::protocol::{wl_output::WlOutput, wl_pointer, wl_registry, wl_seat::WlSeat};
use wayland_client::{Connection, Dispatch, EventQueue, Proxy, QueueHandle};
use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{
zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1,
zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1,
};
use wayland_protocols_wlr::virtual_pointer::v1::client::{
zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1,
zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1,
};
use xkbcommon::xkb;
/// `code` value marking a horizontal scroll event (mirrors `gamestream::input`).
const SCROLL_HORIZONTAL: u32 = 1;
/// Globals bound from the registry (the Wayland dispatch state).
#[derive(Default)]
struct Globals {
pointer_mgr: Option<ZwlrVirtualPointerManagerV1>,
keyboard_mgr: Option<ZwpVirtualKeyboardManagerV1>,
seat: Option<WlSeat>,
output: Option<WlOutput>,
}
impl Dispatch<wl_registry::WlRegistry, ()> for Globals {
fn event(
state: &mut Self,
registry: &wl_registry::WlRegistry,
event: wl_registry::Event,
_: &(),
_: &Connection,
qh: &QueueHandle<Self>,
) {
if let wl_registry::Event::Global {
name,
interface,
version,
} = event
{
match interface.as_str() {
"zwlr_virtual_pointer_manager_v1" => {
state.pointer_mgr = Some(registry.bind(name, version.min(2), qh, ()));
}
"zwp_virtual_keyboard_manager_v1" => {
state.keyboard_mgr = Some(registry.bind(name, version.min(1), qh, ()));
}
"wl_seat" => {
state.seat = Some(registry.bind(name, version.min(7), qh, ()));
}
"wl_output" if state.output.is_none() => {
state.output = Some(registry.bind(name, version.min(3), qh, ()));
}
_ => {}
}
}
}
}
// The managers, the two virtual devices, the seat and the output emit no events we use.
macro_rules! ignore_events {
($($t:ty),* $(,)?) => {$(
impl Dispatch<$t, ()> for Globals {
fn event(_: &mut Self, _: &$t, _: <$t as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}
}
)*};
}
ignore_events!(
WlSeat,
WlOutput,
ZwlrVirtualPointerManagerV1,
ZwlrVirtualPointerV1,
ZwpVirtualKeyboardManagerV1,
ZwpVirtualKeyboardV1,
);
pub struct WlrootsInjector {
conn: Connection,
queue: EventQueue<Globals>,
globals: Globals,
pointer: ZwlrVirtualPointerV1,
keyboard: ZwpVirtualKeyboardV1,
xkb_state: xkb::State,
_keymap_file: std::fs::File, // keep the memfd alive for the compositor's mmap
start: Instant,
}
impl WlrootsInjector {
pub fn open() -> Result<Self> {
let conn = Connection::connect_to_env()
.context("connect to Wayland (is Sway up + WAYLAND_DISPLAY/XDG_RUNTIME_DIR set?)")?;
let mut queue = conn.new_event_queue();
let qh = queue.handle();
let _registry = conn.display().get_registry(&qh, ());
let mut globals = Globals::default();
queue
.roundtrip(&mut globals)
.context("Wayland registry roundtrip")?;
let pointer_mgr = globals
.pointer_mgr
.clone()
.context("compositor lacks zwlr_virtual_pointer_manager_v1")?;
let keyboard_mgr = globals
.keyboard_mgr
.clone()
.context("compositor lacks zwp_virtual_keyboard_manager_v1")?;
let seat = globals
.seat
.clone()
.context("compositor advertised no wl_seat")?;
let pointer = pointer_mgr.create_virtual_pointer_with_output(
Some(&seat),
globals.output.as_ref(),
&qh,
(),
);
let keyboard = keyboard_mgr.create_virtual_keyboard(&seat, &qh, ());
// A standard evdev/US keymap so raw evdev keycodes resolve to the right keysyms.
let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS);
let keymap = xkb::Keymap::new_from_names(
&ctx,
"evdev",
"pc105",
"us",
"",
None,
xkb::KEYMAP_COMPILE_NO_FLAGS,
)
.context("compile xkb keymap")?;
let keymap_str = keymap.get_as_string(xkb::KEYMAP_FORMAT_TEXT_V1);
let xkb_state = xkb::State::new(&keymap);
let file = memfd_with(&keymap_str)?;
let size = keymap_str.len() as u32 + 1; // include the trailing NUL
keyboard.keymap(1 /* XKB_V1 */, file.as_fd(), size);
queue
.roundtrip(&mut globals)
.context("keymap upload roundtrip")?;
conn.flush().ok();
tracing::info!(
output = globals.output.is_some(),
"wlroots virtual input ready (pointer + keyboard)"
);
Ok(Self {
conn,
queue,
globals,
pointer,
keyboard,
xkb_state,
_keymap_file: file,
start: Instant::now(),
})
}
fn now_ms(&self) -> u32 {
self.start.elapsed().as_millis() as u32
}
/// Update xkb state for a key and tell the compositor the resulting modifier mask.
fn send_modifiers(&mut self, evdev: u16, down: bool) {
let kc = xkb::Keycode::new(evdev as u32 + 8); // evdev -> xkb keycode
let dir = if down {
xkb::KeyDirection::Down
} else {
xkb::KeyDirection::Up
};
self.xkb_state.update_key(kc, dir);
let depressed = self.xkb_state.serialize_mods(xkb::STATE_MODS_DEPRESSED);
let latched = self.xkb_state.serialize_mods(xkb::STATE_MODS_LATCHED);
let locked = self.xkb_state.serialize_mods(xkb::STATE_MODS_LOCKED);
let group = self.xkb_state.serialize_layout(xkb::STATE_LAYOUT_EFFECTIVE);
self.keyboard.modifiers(depressed, latched, locked, group);
}
}
impl InputInjector for WlrootsInjector {
fn inject(&mut self, event: &InputEvent) -> Result<()> {
let t = self.now_ms();
match event.kind {
InputKind::MouseMove => {
self.pointer.motion(t, event.x as f64, event.y as f64);
self.pointer.frame();
}
InputKind::MouseMoveAbs => {
let w = (event.flags >> 16) & 0xffff;
let h = event.flags & 0xffff;
if w > 0 && h > 0 {
let x = event.x.clamp(0, w as i32) as u32;
let y = event.y.clamp(0, h as i32) as u32;
self.pointer.motion_absolute(t, x, y, w, h);
self.pointer.frame();
}
}
InputKind::MouseButtonDown | InputKind::MouseButtonUp => {
if let Some(btn) = gs_button_to_evdev(event.code) {
let st = if event.kind == InputKind::MouseButtonDown {
wl_pointer::ButtonState::Pressed
} else {
wl_pointer::ButtonState::Released
};
self.pointer.button(t, btn, st);
self.pointer.frame();
}
}
InputKind::MouseScroll => {
let axis = if event.code == SCROLL_HORIZONTAL {
wl_pointer::Axis::HorizontalScroll
} else {
wl_pointer::Axis::VerticalScroll
};
// GameStream sends WHEEL_DELTA(120)-scaled units; a notch ≈ 15px. Positive
// GameStream = scroll up, which is negative on the Wayland axis.
let notches = event.x as f64 / 120.0;
self.pointer.axis_source(wl_pointer::AxisSource::Wheel);
self.pointer.axis(t, axis, -notches * 15.0);
self.pointer.frame();
}
InputKind::KeyDown | InputKind::KeyUp => {
let down = event.kind == InputKind::KeyDown;
if let Some(evdev) = vk_to_evdev(event.code as u8) {
self.keyboard.key(t, evdev as u32, if down { 1 } else { 0 });
self.send_modifiers(evdev, down);
} else {
tracing::debug!(vk = event.code, "unmapped VK keycode — dropped");
}
}
InputKind::GamepadButton | InputKind::GamepadAxis => {} // not yet injected
}
// Surface protocol errors / disconnects, then push the batch to the compositor.
self.queue
.dispatch_pending(&mut self.globals)
.context("wayland dispatch")?;
self.conn.flush().context("wayland flush")?;
Ok(())
}
}
/// Create an anonymous in-memory file holding `s` + a trailing NUL (for the keymap fd).
fn memfd_with(s: &str) -> Result<std::fs::File> {
let name = b"lumen-keymap\0";
let fd = unsafe { libc::memfd_create(name.as_ptr() as *const libc::c_char, libc::MFD_CLOEXEC) };
if fd < 0 {
bail!("memfd_create failed: {}", std::io::Error::last_os_error());
}
let mut f = unsafe { std::fs::File::from_raw_fd(fd) };
f.write_all(s.as_bytes()).context("write keymap")?;
f.write_all(&[0]).context("write keymap NUL")?;
Ok(f)
}
+53
View File
@@ -49,6 +49,9 @@ fn real_main() -> Result<()> {
match args.first().map(String::as_str) { match args.first().map(String::as_str) {
// M2 GameStream host control plane (P1.1: mDNS + serverinfo). // M2 GameStream host control plane (P1.1: mDNS + serverinfo).
Some("serve") => gamestream::serve(), Some("serve") => gamestream::serve(),
// Standalone input-injection smoke test (no client needed): open the session's input
// backend and inject a scripted mouse/keyboard pattern. Watch a focused app / `wev`.
Some("input-test") => input_test(),
// M0 pipeline spike. // M0 pipeline spike.
Some("m0") => m0::run(parse_m0(&args[1..])?), Some("m0") => m0::run(parse_m0(&args[1..])?),
Some("-h") | Some("--help") | Some("help") | None => { Some("-h") | Some("--help") | Some("help") | None => {
@@ -60,6 +63,56 @@ fn real_main() -> Result<()> {
} }
} }
/// Inject a scripted mouse + keyboard pattern through the session's input backend (libei on
/// KWin/GNOME, wlr on Sway). Lets us validate input injection without a Moonlight client.
#[cfg(target_os = "linux")]
fn input_test() -> Result<()> {
use lumen_core::input::{InputEvent, InputKind};
use std::time::Duration;
let backend = inject::default_backend();
tracing::info!(?backend, "input-test: opening injector");
let mut inj = inject::open(backend)?;
// An async backend (libei) needs a moment to establish its portal/EIS session + device
// resume; events injected before then are dropped.
std::thread::sleep(Duration::from_secs(4));
let ev = |kind, code, x, y| InputEvent {
kind,
_pad: [0; 3],
code,
x,
y,
flags: 0,
};
tracing::info!("input-test: injecting a mouse square + 'A'/click taps for ~8s (watch wev / focused app)");
for i in 0..160u32 {
let (dx, dy) = match (i / 10) % 4 {
0 => (12, 0),
1 => (0, 12),
2 => (-12, 0),
_ => (0, -12),
};
if let Err(e) = inj.inject(&ev(InputKind::MouseMove, 0, dx, dy)) {
tracing::warn!(error = %format!("{e:#}"), "input-test: inject failed");
}
if i % 20 == 0 {
let _ = inj.inject(&ev(InputKind::KeyDown, 0x41, 0, 0)); // 'A'
let _ = inj.inject(&ev(InputKind::KeyUp, 0x41, 0, 0));
let _ = inj.inject(&ev(InputKind::MouseButtonDown, 1, 0, 0)); // left click
let _ = inj.inject(&ev(InputKind::MouseButtonUp, 1, 0, 0));
}
std::thread::sleep(Duration::from_millis(50));
}
tracing::info!("input-test: done");
Ok(())
}
#[cfg(not(target_os = "linux"))]
fn input_test() -> Result<()> {
bail!("input-test requires Linux")
}
fn parse_m0(args: &[String]) -> Result<Options> { fn parse_m0(args: &[String]) -> Result<Options> {
let mut source = Source::Portal; let mut source = Source::Portal;
let mut width = 1920u32; let mut width = 1920u32;