diff --git a/Cargo.lock b/Cargo.lock index 2d1cf48..a26f7dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -709,6 +709,12 @@ dependencies = [ "syn", ] +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dunce" version = "1.0.5" @@ -1443,6 +1449,7 @@ name = "lumen-host" version = "0.0.1" dependencies = [ "aes", + "aes-gcm", "anyhow", "ashpd", "axum", @@ -1463,7 +1470,11 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "wayland-client", + "wayland-protocols-misc", + "wayland-protocols-wlr", "x509-parser", + "xkbcommon", ] [[package]] @@ -1502,6 +1513,15 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -1910,6 +1930,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.9" @@ -3193,6 +3222,89 @@ dependencies = [ "semver", ] +[[package]] +name = "wayland-backend" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2857dd20b54e916ec7253b3d6b4d5c4d7d4ca2c33c2e11c6c76a99bd8744755d" +dependencies = [ + "cc", + "downcast-rs", + "rustix", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144" +dependencies = [ + "bitflags 2.13.0", + "rustix", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols" +version = "0.32.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "563a85523cade2429938e790815fd7319062103b9f4a2dc806e9b53b95982d8f" +dependencies = [ + "bitflags 2.13.0", + "wayland-backend", + "wayland-client", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-misc" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9567599ef23e09b8dad6e429e5738d4509dfc46b3b21f32841a304d16b29c8" +dependencies = [ + "bitflags 2.13.0", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-wlr" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" +dependencies = [ + "bitflags 2.13.0", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c324a910fd86ebdc364a3e61ec1f11737d3b1d6c273c0239ee8ff4bc0d24b4a" +dependencies = [ + "proc-macro2", + "quick-xml", + "quote", +] + +[[package]] +name = "wayland-sys" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" +dependencies = [ + "pkg-config", +] + [[package]] name = "web-time" version = "1.1.0" @@ -3597,6 +3709,23 @@ dependencies = [ "time", ] +[[package]] +name = "xkbcommon" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d66ca9352cbd4eecbbc40871d8a11b4ac8107cfc528a6e14d7c19c69d0e1ac9" +dependencies = [ + "libc", + "memmap2", + "xkeysym", +] + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + [[package]] name = "yasna" version = "0.5.2" diff --git a/crates/lumen-host/Cargo.toml b/crates/lumen-host/Cargo.toml index 08d186c..1f593a2 100644 --- a/crates/lumen-host/Cargo.toml +++ b/crates/lumen-host/Cargo.toml @@ -19,6 +19,7 @@ tokio = { version = "1", features = ["full"] } rsa = "0.9" sha2 = { version = "0.10", features = ["oid"] } aes = "0.8" +aes-gcm = "0.10" rand = "0.8" hex = "0.4" rcgen = { version = "0.13", default-features = false, features = ["aws_lc_rs", "pem"] } @@ -41,3 +42,10 @@ pipewire = "0.9" # ashpd 0.13 uses the tokio runtime; a current-thread runtime drives the one-time # portal handshake (control plane — never the per-frame path). tokio = { version = "1", features = ["rt", "rt-multi-thread", "net", "time"] } +# Input injection into headless Sway via the wlroots virtual-input Wayland protocols +# (uinput won't reach a compositor running with WLR_LIBINPUT_NO_DEVICES=1). +wayland-client = "0.31" +wayland-protocols-wlr = { version = "0.3", features = ["client"] } +wayland-protocols-misc = { version = "0.3", features = ["client"] } +# Builds/validates the xkb keymap uploaded to the virtual keyboard + tracks modifier state. +xkbcommon = "0.8" diff --git a/crates/lumen-host/src/gamestream/control.rs b/crates/lumen-host/src/gamestream/control.rs index f8be93b..3451950 100644 --- a/crates/lumen-host/src/gamestream/control.rs +++ b/crates/lumen-host/src/gamestream/control.rs @@ -1,20 +1,37 @@ //! The GameStream control stream: an ENet host on UDP 47999. Moonlight connects this //! BEFORE the video stream starts (`STAGE_CONTROL_STREAM_START` precedes -//! `STAGE_VIDEO_STREAM_START`), so it must be up or the whole connection aborts. P1.4 here -//! just accepts the connection + services ENet (keepalive/timeouts) so video can flow; -//! decoding control messages into input injection (mouse/keyboard/gamepad) is the next step. +//! `STAGE_VIDEO_STREAM_START`), so it must be up or the whole connection aborts. It carries +//! input (mouse/keyboard/gamepad), keepalives, and QoS feedback. //! -//! Plaintext for now (we negotiate `encryptionEnabled=0` in DESCRIBE); the encrypted -//! SS_ENC_CONTROL_V2 framing is P1.5. Runs on its own native thread for the host's lifetime. +//! Sunshine-mode hosts (we advertise `state=SUNSHINE_SERVER_FREE`) make Moonlight encrypt the +//! control stream with AES-128-GCM under the `/launch` `rikey`, even though we negotiate no +//! media encryption. Wire framing (all little-endian): +//! +//! ```text +//! u16 encType = 0x0001 | u16 length | u32 seq | [16-byte GCM tag] | ciphertext +//! length = sizeof(seq) + 16 (tag) + plaintext +//! ``` +//! +//! The GCM nonce depends on what Moonlight negotiated (`encryptControlMessage` in +//! moonlight-common-c). For `SS_ENC_CONTROL_V2` it is a 12-byte nonce with `seq` (LE) in bytes +//! [0..4] and `b"CC"` (client→host) at [10..12]. For the legacy path — which we hit, since we +//! advertise no encryption — it is a 16-byte nonce with only `iv[0] = seq & 0xff` and the rest +//! zero. The tag is prepended to the ciphertext; there is no AAD; the key is the forward +//! `hex::decode(rikey)`. We auto-detect the exact scheme via [`decrypt_control`] on the first +//! packet that authenticates, since GCM gives no partial credit. +//! +//! Runs on its own native thread for the host's lifetime. -use super::CONTROL_PORT; +use super::{AppState, CONTROL_PORT}; +use crate::inject::InputInjector; use anyhow::{anyhow, Context, Result}; use rusty_enet::{Event, Host, HostSettings}; use std::net::UdpSocket; +use std::sync::Arc; use std::time::Duration; /// Bind the ENet control host on 47999 and service it forever on a dedicated thread. -pub fn spawn() -> Result<()> { +pub fn spawn(state: Arc) -> Result<()> { let socket = UdpSocket::bind(("0.0.0.0", CONTROL_PORT)).context("bind control UDP")?; socket .set_nonblocking(true) @@ -32,44 +49,263 @@ pub fn spawn() -> Result<()> { std::thread::Builder::new() .name("lumen-control".into()) - .spawn(move || loop { + .spawn(move || { + // Thread-local (the injector owns non-Send Wayland/xkb state, so it must be + // created and live here rather than be captured into the closure). + // GCM scheme detected from the first authenticating packet; reused thereafter. + let mut detected: Option = None; + // Lazily opened on the first input event (Sway's Wayland socket is up by then). + let mut injector: Option> = None; loop { - match host.service() { - Ok(Some(event)) => match event { - Event::Connect { .. } => { - tracing::info!("control: client connected"); + loop { + match host.service() { + Ok(Some(event)) => match event { + Event::Connect { .. } => { + tracing::info!("control: client connected"); + } + Event::Disconnect { .. } => { + tracing::info!("control: client disconnected"); + detected = None; + } + Event::Receive { + channel_id, packet, .. + } => { + on_receive( + &state, + channel_id, + packet.data(), + &mut detected, + &mut injector, + ); + } + }, + Ok(None) => break, + Err(e) => { + tracing::warn!(error = %format!("{e:?}"), "control: service error"); + break; } - Event::Disconnect { .. } => { - tracing::info!("control: client disconnected"); - } - Event::Receive { - channel_id, packet, .. - } => { - let d = packet.data(); - let opcode = if d.len() >= 2 { - u16::from_le_bytes([d[0], d[1]]) - } else { - 0 - }; - // TODO(P1.4): decode input events (mouse/keyboard/gamepad) → inject.rs. - tracing::debug!( - channel_id, - len = d.len(), - opcode = format!("0x{opcode:04x}"), - "control: message" - ); - } - }, - Ok(None) => break, - Err(e) => { - tracing::warn!(error = %format!("{e:?}"), "control: service error"); - break; } } + // ENet needs frequent servicing for handshake/keepalive/retransmit. + std::thread::sleep(Duration::from_millis(2)); } - // ENet needs frequent servicing for handshake/keepalive/retransmit. - std::thread::sleep(Duration::from_millis(2)); }) .context("spawn control thread")?; Ok(()) } + +/// Handle one received control packet: decrypt it (learning the GCM scheme on the first one), +/// decode any input event, and inject it into the host session. +fn on_receive( + state: &AppState, + _channel_id: u8, + d: &[u8], + detected: &mut Option, + injector: &mut Option>, +) { + let Some(key) = state.launch.lock().unwrap().map(|s| s.gcm_key) else { + return; // control traffic before /launch — no key yet + }; + // Encrypted control packets begin with u16 LE encType = 0x0001 and an 8-byte header. + if d.len() < 8 || d[0] != 0x01 || d[1] != 0x00 { + return; + } + + let pt = match decrypt_control(&key, d, detected) { + Some((scheme, pt)) => { + if detected.is_none() { + tracing::info!(?scheme, "control: GCM scheme locked in"); + } + *detected = Some(scheme); + pt + } + None => { + tracing::warn!(len = d.len(), "control: GCM decrypt failed"); + return; + } + }; + + let events = super::input::decode(&pt); + if events.is_empty() { + return; // keepalive / QoS / unhandled input kind + } + + // Open the injector on demand — by the first input event Sway's Wayland socket is up. + if injector.is_none() { + match crate::inject::open(crate::inject::Backend::WlrVirtual) { + Ok(i) => *injector = Some(i), + Err(e) => { + tracing::error!(error = %format!("{e:#}"), "input injection unavailable"); + return; + } + } + } + let inj = injector.as_mut().unwrap(); + for ev in events { + if let Err(e) = inj.inject(&ev) { + tracing::warn!(error = %format!("{e:#}"), "inject failed"); + } + } +} + +/// How a control packet's nonce is built — Moonlight picks one based on the negotiated flags. +#[derive(Clone, Copy, Debug)] +enum NonceKind { + /// `SS_ENC_CONTROL_V2`: 12-byte nonce, `seq` in [0..4], marker bytes at [10..12]. + V2 { seq_be: bool, marker: [u8; 2] }, + /// Legacy: 16-byte nonce, only `iv[0] = seq & 0xff` (the rest zero). + LegacyLowByte, + /// Legacy variant: 16-byte nonce, full `seq` in [0..4] (the rest zero). + Legacy16Seq { seq_be: bool }, +} + +impl NonceKind { + fn nonce(&self, seq: u32) -> Vec { + let seq_bytes = |be: bool| { + if be { + seq.to_be_bytes() + } else { + seq.to_le_bytes() + } + }; + match *self { + NonceKind::V2 { seq_be, marker } => { + let mut iv = vec![0u8; 12]; + iv[0..4].copy_from_slice(&seq_bytes(seq_be)); + iv[10] = marker[0]; + iv[11] = marker[1]; + iv + } + NonceKind::LegacyLowByte => { + let mut iv = vec![0u8; 16]; + iv[0] = (seq & 0xff) as u8; + iv + } + NonceKind::Legacy16Seq { seq_be } => { + let mut iv = vec![0u8; 16]; + iv[0..4].copy_from_slice(&seq_bytes(seq_be)); + iv + } + } + } +} + +/// The byte-exact GCM scheme that opened a control packet. Determined empirically once per +/// connection (AES-GCM gives no partial credit, so an authenticating combination is proof). +#[derive(Clone, Copy, Debug)] +struct Scheme { + /// `gcm_key` is byte-reversed before use (defensive; Sunshine's net effect is forward). + key_rev: bool, + nonce: NonceKind, + /// GCM tag sits before the ciphertext (vs after). + tag_first: bool, + aad: Aad, +} + +#[derive(Clone, Copy, Debug)] +enum Aad { + None, + /// The 4-byte cleartext header prefix (encType + length), `d[0..4]`. + Header4, +} + +impl Scheme { + fn key(&self, base: &[u8; 16]) -> [u8; 16] { + let mut k = *base; + if self.key_rev { + k.reverse(); + } + k + } +} + +/// Open an encrypted control packet `d` (8-byte cleartext header + `[tag?][ciphertext]`). If +/// `detected` is set only that scheme is tried (fast path); otherwise the full cross-product +/// of plausible schemes (nonce construction × key byte-order × tag position × AAD) is swept +/// and the combination whose GCM tag authenticates is returned. +fn decrypt_control( + key: &[u8; 16], + d: &[u8], + detected: &Option, +) -> Option<(Scheme, Vec)> { + let seq = u32::from_le_bytes([d[4], d[5], d[6], d[7]]); + let payload = &d[8..]; + if payload.len() < 16 { + return None; + } + + let attempt = |s: Scheme| -> Option> { + // aes-gcm wants `ciphertext || tag`; reassemble from whichever wire order this is. + let (ct, tag) = if s.tag_first { + (&payload[16..], &payload[..16]) + } else { + ( + &payload[..payload.len() - 16], + &payload[payload.len() - 16..], + ) + }; + let mut ct_tag = Vec::with_capacity(ct.len() + 16); + ct_tag.extend_from_slice(ct); + ct_tag.extend_from_slice(tag); + let aad: &[u8] = match s.aad { + Aad::None => &[], + Aad::Header4 => &d[0..4], + }; + gcm_open(&s.key(key), &s.nonce.nonce(seq), &ct_tag, aad) + }; + + if let Some(s) = *detected { + return attempt(s).map(|pt| (s, pt)); + } + + // Candidate nonce constructions, most-likely first. + const MARKERS: [[u8; 2]; 3] = [*b"CC", *b"HC", *b"CH"]; + let mut kinds: Vec = vec![NonceKind::LegacyLowByte]; + for seq_be in [false, true] { + for marker in MARKERS { + kinds.push(NonceKind::V2 { seq_be, marker }); + } + kinds.push(NonceKind::Legacy16Seq { seq_be }); + } + + for &nonce in &kinds { + for key_rev in [false, true] { + for tag_first in [true, false] { + for aad in [Aad::None, Aad::Header4] { + let s = Scheme { + key_rev, + nonce, + tag_first, + aad, + }; + if let Some(pt) = attempt(s) { + return Some((s, pt)); + } + } + } + } + } + None +} + +/// AES-128-GCM open with a 12- or 16-byte nonce and explicit AAD. Returns the plaintext iff +/// the tag authenticates. `ct_tag` is `ciphertext || tag` (aes-gcm's expected order). +fn gcm_open(key: &[u8; 16], nonce: &[u8], ct_tag: &[u8], aad: &[u8]) -> Option> { + use aes_gcm::aead::consts::{U12, U16}; + use aes_gcm::aead::generic_array::GenericArray; + use aes_gcm::aead::{Aead, KeyInit, Payload}; + use aes_gcm::{aes::Aes128, AesGcm}; + + let p = Payload { msg: ct_tag, aad }; + match nonce.len() { + 12 => AesGcm::::new_from_slice(key) + .ok()? + .decrypt(GenericArray::from_slice(nonce), p) + .ok(), + 16 => AesGcm::::new_from_slice(key) + .ok()? + .decrypt(GenericArray::from_slice(nonce), p) + .ok(), + _ => None, + } +} diff --git a/crates/lumen-host/src/gamestream/input.rs b/crates/lumen-host/src/gamestream/input.rs new file mode 100644 index 0000000..83470c0 --- /dev/null +++ b/crates/lumen-host/src/gamestream/input.rs @@ -0,0 +1,143 @@ +//! Decode the GameStream input wire format (carried AES-GCM-encrypted on the ENet control +//! stream — see [`super::control`]) into platform-agnostic +//! [`lumen_core::input::InputEvent`]s for injection. +//! +//! A decrypted control message is `[u16 type LE][u16 length LE][NV_INPUT packet]`. We only +//! handle the input type (`0x0206`); the packet is an 8-byte `NV_INPUT_HEADER` (`size` BE, +//! `magic` LE) followed by a magic-specific body. Multi-byte body fields are big-endian +//! (network order) except `magic` and the keyboard `keyCode` (little-endian). Struct layouts +//! mirror moonlight-common-c `Input.h`; the magic dispatch matches Sunshine `input.cpp` +//! (Gen5+, where scroll is `0x0A` and controllers are `0x0C`, so there's no ambiguity). + +use lumen_core::input::{InputEvent, InputKind}; + +/// Inner control-message type for input (moonlight `packetTypesGen7[IDX_INPUT_DATA]`). +const INPUT_DATA_TYPE: u16 = 0x0206; + +// NV_INPUT_HEADER.magic values (Input.h), with the Gen5+ variants where they differ. +const MAGIC_KEY_DOWN: u32 = 0x03; +const MAGIC_KEY_UP: u32 = 0x04; +const MAGIC_MOUSE_ABS: u32 = 0x05; +const MAGIC_MOUSE_REL: u32 = 0x06; +const MAGIC_MOUSE_REL_GEN5: u32 = 0x07; +const MAGIC_MOUSE_BTN_DOWN: u32 = 0x08; +const MAGIC_MOUSE_BTN_UP: u32 = 0x09; +const MAGIC_SCROLL_GEN5: u32 = 0x0A; +const MAGIC_UTF8: u32 = 0x17; +const MAGIC_HSCROLL: u32 = 0x5500_0001; + +/// `code` value marking a [`InputKind::MouseScroll`] as horizontal (vs `0` = vertical). +pub const SCROLL_HORIZONTAL: u32 = 1; + +/// Decode one decrypted control plaintext into zero or more input events. Non-input control +/// messages (keepalives, QoS) and unhandled input kinds (gamepad/pen/touch) yield nothing. +pub fn decode(plaintext: &[u8]) -> Vec { + if plaintext.len() < 4 || u16::from_le_bytes([plaintext[0], plaintext[1]]) != INPUT_DATA_TYPE { + return Vec::new(); + } + decode_input_packet(&plaintext[4..]).into_iter().collect() +} + +fn decode_input_packet(p: &[u8]) -> Option { + if p.len() < 8 { + return None; + } + // NV_INPUT_HEADER: size (BE u32, excludes itself) + magic (LE u32). Body follows. + let magic = u32::from_le_bytes([p[4], p[5], p[6], p[7]]); + let b = &p[8..]; + let be16 = |o: usize| -> Option { Some(i16::from_be_bytes([*b.get(o)?, *b.get(o + 1)?])) }; + + Some(match magic { + MAGIC_MOUSE_REL | MAGIC_MOUSE_REL_GEN5 => { + ev(InputKind::MouseMove, 0, be16(0)? as i32, be16(2)? as i32, 0) + } + MAGIC_MOUSE_ABS => { + // short x, y, unused, width, height (all BE). Carry the client's reference extent + // (width<<16 | height) in `flags` so the injector can scale to its output. + let (x, y) = (be16(0)? as i32, be16(2)? as i32); + let flags = ((be16(6)? as u16 as u32) << 16) | (be16(8)? as u16 as u32); + ev(InputKind::MouseMoveAbs, 0, x, y, flags) + } + MAGIC_MOUSE_BTN_DOWN => ev(InputKind::MouseButtonDown, *b.first()? as u32, 0, 0, 0), + MAGIC_MOUSE_BTN_UP => ev(InputKind::MouseButtonUp, *b.first()? as u32, 0, 0, 0), + MAGIC_SCROLL_GEN5 => ev(InputKind::MouseScroll, 0, be16(0)? as i32, 0, 0), + MAGIC_HSCROLL => ev( + InputKind::MouseScroll, + SCROLL_HORIZONTAL, + be16(0)? as i32, + 0, + 0, + ), + MAGIC_KEY_DOWN | MAGIC_KEY_UP => { + // char flags, short keyCode (LE), char modifiers, short zero2. The client stuffs a + // 0x80 high byte on key-down; Sunshine masks to the low-byte VK (`& 0xFF`). + let key_code = (u16::from_le_bytes([*b.get(1)?, *b.get(2)?]) & 0x00FF) as u32; + let modifiers = *b.get(3)? as u32; + let kind = if magic == MAGIC_KEY_DOWN { + InputKind::KeyDown + } else { + InputKind::KeyUp + }; + ev(kind, key_code, 0, 0, modifiers) + } + // UTF-8 text, gamepad, pen, touch, haptics — not yet injected. + _ => return None, + }) +} + +fn ev(kind: InputKind, code: u32, x: i32, y: i32, flags: u32) -> InputEvent { + InputEvent { + kind, + _pad: [0; 3], + code, + x, + y, + flags, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a control plaintext: inner header + NV_INPUT_HEADER + body. + fn wrap(magic: u32, body: &[u8]) -> Vec { + let mut inp = Vec::new(); + inp.extend_from_slice(&((4 + body.len()) as u32).to_be_bytes()); // size (excl. itself) + inp.extend_from_slice(&magic.to_le_bytes()); + inp.extend_from_slice(body); + let mut pt = Vec::new(); + pt.extend_from_slice(&INPUT_DATA_TYPE.to_le_bytes()); + pt.extend_from_slice(&(inp.len() as u16).to_le_bytes()); + pt.extend_from_slice(&inp); + pt + } + + #[test] + fn decodes_relative_mouse() { + // deltaX = -1 (ffff BE), deltaY = +2 (0002 BE) — matches a real captured packet. + let pt = wrap(MAGIC_MOUSE_REL_GEN5, &[0xff, 0xff, 0x00, 0x02]); + let ev = decode(&pt); + assert_eq!(ev.len(), 1); + assert_eq!(ev[0].kind, InputKind::MouseMove); + assert_eq!((ev[0].x, ev[0].y), (-1, 2)); + } + + #[test] + fn decodes_key_down_masking_high_byte() { + // keyCode 0x80A4 (LE a4 80) → VK 0xA4 (VK_LMENU); modifiers 0x04 (Alt). + let pt = wrap(MAGIC_KEY_DOWN, &[0x00, 0xa4, 0x80, 0x04, 0x00, 0x00]); + let ev = decode(&pt); + assert_eq!(ev.len(), 1); + assert_eq!(ev[0].kind, InputKind::KeyDown); + assert_eq!(ev[0].code, 0xA4); + assert_eq!(ev[0].flags, 0x04); + } + + #[test] + fn ignores_non_input_type() { + let mut pt = vec![0x00, 0x02]; // type 0x0200 (keepalive) + pt.extend_from_slice(&[0x08, 0x00, 0x04, 0, 0, 0, 0, 0, 0, 0]); + assert!(decode(&pt).is_empty()); + } +} diff --git a/crates/lumen-host/src/gamestream/mod.rs b/crates/lumen-host/src/gamestream/mod.rs index 06628fa..f85c570 100644 --- a/crates/lumen-host/src/gamestream/mod.rs +++ b/crates/lumen-host/src/gamestream/mod.rs @@ -9,6 +9,7 @@ mod cert; mod control; mod crypto; +mod input; mod mdns; mod nvhttp; mod pairing; @@ -112,7 +113,7 @@ pub fn serve() -> Result<()> { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let _advert = mdns::advertise(&state.host).context("mDNS advertise")?; rtsp::spawn(state.clone()).context("start RTSP server")?; - control::spawn().context("start ENet control server")?; + control::spawn(state.clone()).context("start ENet control server")?; nvhttp::run(state).await }) } diff --git a/crates/lumen-host/src/inject.rs b/crates/lumen-host/src/inject.rs index 61084c9..6089752 100644 --- a/crates/lumen-host/src/inject.rs +++ b/crates/lumen-host/src/inject.rs @@ -1,30 +1,460 @@ -//! Input injection (plan §4): turn client [`lumen_core::input::InputEvent`]s into host -//! input. Wayland-native via libei (`reis`) first; uinput as the universal fallback. +//! Input injection (plan §4): turn client [`lumen_core::input::InputEvent`]s into host input. +//! +//! The headless Sway compositor runs with `WLR_LIBINPUT_NO_DEVICES=1`, so kernel `uinput` +//! devices are never picked up. Instead we inject through the wlroots virtual-input Wayland +//! protocols — `zwlr_virtual_pointer_manager_v1` + `zwp_virtual_keyboard_manager_v1` — which +//! Sway always advertises. We connect as an ordinary Wayland client (the host process +//! inherits Sway's `WAYLAND_DISPLAY`/`XDG_RUNTIME_DIR`), bind the two managers, and translate +//! events into virtual pointer/keyboard requests. Keyboard codes are Linux evdev; we upload a +//! standard evdev/US xkb keymap and track modifier state so the compositor resolves shifted +//! keysyms correctly. use anyhow::Result; use lumen_core::input::InputEvent; -/// Injects input events into the host session. -pub trait InputInjector: Send { +/// Injects input events into the host session. Not `Send`: an injector owns compositor +/// resources (a Wayland connection, an xkb state) and lives entirely on the control thread +/// that creates it. +pub trait InputInjector { fn inject(&mut self, event: &InputEvent) -> Result<()>; } /// Preferred injection backend. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Backend { - /// libei via `reis` — Wayland-native (RemoteDesktop portal). + /// wlroots virtual pointer + keyboard Wayland protocols — the headless-Sway path. + WlrVirtual, + /// libei via `reis` — Wayland-native (RemoteDesktop portal). Not yet implemented. Libei, - /// `/dev/uinput` — universal fallback, always available. + /// `/dev/uinput` — universal fallback (but invisible to `WLR_LIBINPUT_NO_DEVICES=1`). Uinput, } -pub fn open(_backend: Backend) -> Result> { - #[cfg(target_os = "linux")] - { - anyhow::bail!("libei/uinput injection not yet implemented (M2)") - } - #[cfg(not(target_os = "linux"))] - { - anyhow::bail!("input injection requires Linux (libei/uinput)") +pub fn open(backend: Backend) -> Result> { + match backend { + Backend::WlrVirtual => { + #[cfg(target_os = "linux")] + { + Ok(Box::new(wlr::WlrootsInjector::open()?)) + } + #[cfg(not(target_os = "linux"))] + { + anyhow::bail!("wlroots virtual input requires Linux + a Wayland compositor") + } + } + other => anyhow::bail!("injection backend {other:?} not implemented; use WlrVirtual"), + } +} + +/// Map a Windows Virtual-Key code (as sent by Moonlight/GameStream) to a Linux evdev key code. +pub fn vk_to_evdev(vk: u8) -> Option { + match vk { + // --- Navigation / editing / whitespace --- + 0x08 => Some(14), // VK_BACK -> KEY_BACKSPACE + 0x09 => Some(15), // VK_TAB -> KEY_TAB + 0x0D => Some(28), // VK_RETURN -> KEY_ENTER + 0x13 => Some(119), // VK_PAUSE -> KEY_PAUSE + 0x14 => Some(58), // VK_CAPITAL -> KEY_CAPSLOCK + 0x1B => Some(1), // VK_ESCAPE -> KEY_ESC + 0x20 => Some(57), // VK_SPACE -> KEY_SPACE + 0x21 => Some(104), // VK_PRIOR -> KEY_PAGEUP + 0x22 => Some(109), // VK_NEXT -> KEY_PAGEDOWN + 0x23 => Some(107), // VK_END -> KEY_END + 0x24 => Some(102), // VK_HOME -> KEY_HOME + 0x25 => Some(105), // VK_LEFT -> KEY_LEFT + 0x26 => Some(103), // VK_UP -> KEY_UP + 0x27 => Some(106), // VK_RIGHT -> KEY_RIGHT + 0x28 => Some(108), // VK_DOWN -> KEY_DOWN + 0x2C => Some(99), // VK_SNAPSHOT -> KEY_SYSRQ + 0x2D => Some(110), // VK_INSERT -> KEY_INSERT + 0x2E => Some(111), // VK_DELETE -> KEY_DELETE + + // --- Generic modifiers --- + 0x10 => Some(42), // VK_SHIFT -> KEY_LEFTSHIFT + 0x11 => Some(29), // VK_CONTROL -> KEY_LEFTCTRL + 0x12 => Some(56), // VK_MENU -> KEY_LEFTALT + + // --- Digit row (KEY_0 is 11, KEY_1..KEY_9 are 2..10) --- + 0x30 => Some(11), // VK_0 + 0x31 => Some(2), // VK_1 + 0x32 => Some(3), // VK_2 + 0x33 => Some(4), // VK_3 + 0x34 => Some(5), // VK_4 + 0x35 => Some(6), // VK_5 + 0x36 => Some(7), // VK_6 + 0x37 => Some(8), // VK_7 + 0x38 => Some(9), // VK_8 + 0x39 => Some(10), // VK_9 + + // --- Letters A-Z (NOT sequential in evdev) --- + 0x41 => Some(30), // A + 0x42 => Some(48), // B + 0x43 => Some(46), // C + 0x44 => Some(32), // D + 0x45 => Some(18), // E + 0x46 => Some(33), // F + 0x47 => Some(34), // G + 0x48 => Some(35), // H + 0x49 => Some(23), // I + 0x4A => Some(36), // J + 0x4B => Some(37), // K + 0x4C => Some(38), // L + 0x4D => Some(50), // M + 0x4E => Some(49), // N + 0x4F => Some(24), // O + 0x50 => Some(25), // P + 0x51 => Some(16), // Q + 0x52 => Some(19), // R + 0x53 => Some(31), // S + 0x54 => Some(20), // T + 0x55 => Some(22), // U + 0x56 => Some(47), // V + 0x57 => Some(17), // W + 0x58 => Some(45), // X + 0x59 => Some(21), // Y + 0x5A => Some(44), // Z + + // --- Meta / context-menu --- + 0x5B => Some(125), // VK_LWIN -> KEY_LEFTMETA + 0x5C => Some(126), // VK_RWIN -> KEY_RIGHTMETA + 0x5D => Some(127), // VK_APPS -> KEY_COMPOSE + + // --- Numpad --- + 0x60 => Some(82), // KP0 + 0x61 => Some(79), // KP1 + 0x62 => Some(80), // KP2 + 0x63 => Some(81), // KP3 + 0x64 => Some(75), // KP4 + 0x65 => Some(76), // KP5 + 0x66 => Some(77), // KP6 + 0x67 => Some(71), // KP7 + 0x68 => Some(72), // KP8 + 0x69 => Some(73), // KP9 + 0x6A => Some(55), // VK_MULTIPLY -> KEY_KPASTERISK + 0x6B => Some(78), // VK_ADD -> KEY_KPPLUS + 0x6C => Some(96), // VK_SEPARATOR -> KEY_KPENTER + 0x6D => Some(74), // VK_SUBTRACT -> KEY_KPMINUS + 0x6E => Some(83), // VK_DECIMAL -> KEY_KPDOT + 0x6F => Some(98), // VK_DIVIDE -> KEY_KPSLASH + + // --- Function keys (F1..F10 = 59..68, F11/F12 = 87/88) --- + 0x70 => Some(59), + 0x71 => Some(60), + 0x72 => Some(61), + 0x73 => Some(62), + 0x74 => Some(63), + 0x75 => Some(64), + 0x76 => Some(65), + 0x77 => Some(66), + 0x78 => Some(67), + 0x79 => Some(68), + 0x7A => Some(87), + 0x7B => Some(88), + + // --- Locks --- + 0x90 => Some(69), // VK_NUMLOCK -> KEY_NUMLOCK + 0x91 => Some(70), // VK_SCROLL -> KEY_SCROLLLOCK + + // --- Left/right modifiers --- + 0xA0 => Some(42), // VK_LSHIFT -> KEY_LEFTSHIFT + 0xA1 => Some(54), // VK_RSHIFT -> KEY_RIGHTSHIFT + 0xA2 => Some(29), // VK_LCONTROL -> KEY_LEFTCTRL + 0xA3 => Some(97), // VK_RCONTROL -> KEY_RIGHTCTRL + 0xA4 => Some(56), // VK_LMENU -> KEY_LEFTALT + 0xA5 => Some(100), // VK_RMENU -> KEY_RIGHTALT + + // --- OEM punctuation (US layout) --- + 0xBA => Some(39), // VK_OEM_1 -> KEY_SEMICOLON + 0xBB => Some(13), // VK_OEM_PLUS -> KEY_EQUAL + 0xBC => Some(51), // VK_OEM_COMMA -> KEY_COMMA + 0xBD => Some(12), // VK_OEM_MINUS -> KEY_MINUS + 0xBE => Some(52), // VK_OEM_PERIOD -> KEY_DOT + 0xBF => Some(53), // VK_OEM_2 -> KEY_SLASH + 0xC0 => Some(41), // VK_OEM_3 -> KEY_GRAVE + 0xDB => Some(26), // VK_OEM_4 -> KEY_LEFTBRACE + 0xDC => Some(43), // VK_OEM_5 -> KEY_BACKSLASH + 0xDD => Some(27), // VK_OEM_6 -> KEY_RIGHTBRACE + 0xDE => Some(40), // VK_OEM_7 -> KEY_APOSTROPHE + 0xE2 => Some(86), // VK_OEM_102 -> KEY_102ND + + _ => None, + } +} + +/// Map a GameStream mouse button id (1=left … 5=X2) to a Linux evdev `BTN_*` code. +#[cfg(target_os = "linux")] +fn gs_button_to_evdev(b: u32) -> Option { + Some(match b { + 1 => 0x110, // BTN_LEFT + 2 => 0x112, // BTN_MIDDLE + 3 => 0x111, // BTN_RIGHT + 4 => 0x113, // BTN_SIDE (X1) + 5 => 0x114, // BTN_EXTRA (X2) + _ => return None, + }) +} + +#[cfg(target_os = "linux")] +mod wlr { + use super::{gs_button_to_evdev, vk_to_evdev, InputEvent, InputInjector}; + use anyhow::{bail, Context, Result}; + use lumen_core::input::InputKind; + use std::io::Write; + use std::os::fd::{AsFd, FromRawFd}; + use std::time::Instant; + use wayland_client::protocol::{wl_output::WlOutput, wl_pointer, wl_registry, wl_seat::WlSeat}; + use wayland_client::{Connection, Dispatch, EventQueue, Proxy, QueueHandle}; + use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{ + zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1, + zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1, + }; + use wayland_protocols_wlr::virtual_pointer::v1::client::{ + zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1, + zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1, + }; + use xkbcommon::xkb; + + /// `code` value marking a horizontal scroll event (mirrors `gamestream::input`). + const SCROLL_HORIZONTAL: u32 = 1; + + /// Globals bound from the registry (the Wayland dispatch state). + #[derive(Default)] + struct Globals { + pointer_mgr: Option, + keyboard_mgr: Option, + seat: Option, + output: Option, + } + + impl Dispatch for Globals { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { + name, + interface, + version, + } = event + { + match interface.as_str() { + "zwlr_virtual_pointer_manager_v1" => { + state.pointer_mgr = Some(registry.bind(name, version.min(2), qh, ())); + } + "zwp_virtual_keyboard_manager_v1" => { + state.keyboard_mgr = Some(registry.bind(name, version.min(1), qh, ())); + } + "wl_seat" => { + state.seat = Some(registry.bind(name, version.min(7), qh, ())); + } + "wl_output" if state.output.is_none() => { + state.output = Some(registry.bind(name, version.min(3), qh, ())); + } + _ => {} + } + } + } + } + + // The managers, the two virtual devices, the seat and the output emit no events we use. + macro_rules! ignore_events { + ($($t:ty),* $(,)?) => {$( + impl Dispatch<$t, ()> for Globals { + fn event(_: &mut Self, _: &$t, _: <$t as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle) {} + } + )*}; + } + ignore_events!( + WlSeat, + WlOutput, + ZwlrVirtualPointerManagerV1, + ZwlrVirtualPointerV1, + ZwpVirtualKeyboardManagerV1, + ZwpVirtualKeyboardV1, + ); + + pub struct WlrootsInjector { + conn: Connection, + queue: EventQueue, + globals: Globals, + pointer: ZwlrVirtualPointerV1, + keyboard: ZwpVirtualKeyboardV1, + xkb_state: xkb::State, + _keymap_file: std::fs::File, // keep the memfd alive for the compositor's mmap + start: Instant, + } + + impl WlrootsInjector { + pub fn open() -> Result { + let conn = Connection::connect_to_env().context( + "connect to Wayland (is Sway up + WAYLAND_DISPLAY/XDG_RUNTIME_DIR set?)", + )?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut globals = Globals::default(); + queue + .roundtrip(&mut globals) + .context("Wayland registry roundtrip")?; + + let pointer_mgr = globals + .pointer_mgr + .clone() + .context("compositor lacks zwlr_virtual_pointer_manager_v1")?; + let keyboard_mgr = globals + .keyboard_mgr + .clone() + .context("compositor lacks zwp_virtual_keyboard_manager_v1")?; + let seat = globals + .seat + .clone() + .context("compositor advertised no wl_seat")?; + + let pointer = pointer_mgr.create_virtual_pointer_with_output( + Some(&seat), + globals.output.as_ref(), + &qh, + (), + ); + let keyboard = keyboard_mgr.create_virtual_keyboard(&seat, &qh, ()); + + // A standard evdev/US keymap so raw evdev keycodes resolve to the right keysyms. + let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); + let keymap = xkb::Keymap::new_from_names( + &ctx, + "evdev", + "pc105", + "us", + "", + None, + xkb::KEYMAP_COMPILE_NO_FLAGS, + ) + .context("compile xkb keymap")?; + let keymap_str = keymap.get_as_string(xkb::KEYMAP_FORMAT_TEXT_V1); + let xkb_state = xkb::State::new(&keymap); + + let file = memfd_with(&keymap_str)?; + let size = keymap_str.len() as u32 + 1; // include the trailing NUL + keyboard.keymap(1 /* XKB_V1 */, file.as_fd(), size); + queue + .roundtrip(&mut globals) + .context("keymap upload roundtrip")?; + conn.flush().ok(); + + tracing::info!( + output = globals.output.is_some(), + "wlroots virtual input ready (pointer + keyboard)" + ); + Ok(Self { + conn, + queue, + globals, + pointer, + keyboard, + xkb_state, + _keymap_file: file, + start: Instant::now(), + }) + } + + fn now_ms(&self) -> u32 { + self.start.elapsed().as_millis() as u32 + } + + /// Update xkb state for a key and tell the compositor the resulting modifier mask. + fn send_modifiers(&mut self, evdev: u16, down: bool) { + let kc = xkb::Keycode::new(evdev as u32 + 8); // evdev -> xkb keycode + let dir = if down { + xkb::KeyDirection::Down + } else { + xkb::KeyDirection::Up + }; + self.xkb_state.update_key(kc, dir); + let depressed = self.xkb_state.serialize_mods(xkb::STATE_MODS_DEPRESSED); + let latched = self.xkb_state.serialize_mods(xkb::STATE_MODS_LATCHED); + let locked = self.xkb_state.serialize_mods(xkb::STATE_MODS_LOCKED); + let group = self.xkb_state.serialize_layout(xkb::STATE_LAYOUT_EFFECTIVE); + self.keyboard.modifiers(depressed, latched, locked, group); + } + } + + impl InputInjector for WlrootsInjector { + fn inject(&mut self, event: &InputEvent) -> Result<()> { + let t = self.now_ms(); + match event.kind { + InputKind::MouseMove => { + self.pointer.motion(t, event.x as f64, event.y as f64); + self.pointer.frame(); + } + InputKind::MouseMoveAbs => { + let w = (event.flags >> 16) & 0xffff; + let h = event.flags & 0xffff; + if w > 0 && h > 0 { + let x = event.x.clamp(0, w as i32) as u32; + let y = event.y.clamp(0, h as i32) as u32; + self.pointer.motion_absolute(t, x, y, w, h); + self.pointer.frame(); + } + } + InputKind::MouseButtonDown | InputKind::MouseButtonUp => { + if let Some(btn) = gs_button_to_evdev(event.code) { + let st = if event.kind == InputKind::MouseButtonDown { + wl_pointer::ButtonState::Pressed + } else { + wl_pointer::ButtonState::Released + }; + self.pointer.button(t, btn, st); + self.pointer.frame(); + } + } + InputKind::MouseScroll => { + let axis = if event.code == SCROLL_HORIZONTAL { + wl_pointer::Axis::HorizontalScroll + } else { + wl_pointer::Axis::VerticalScroll + }; + // GameStream sends WHEEL_DELTA(120)-scaled units; a notch ≈ 15px. Positive + // GameStream = scroll up, which is negative on the Wayland axis. + let notches = event.x as f64 / 120.0; + self.pointer.axis_source(wl_pointer::AxisSource::Wheel); + self.pointer.axis(t, axis, -notches * 15.0); + self.pointer.frame(); + } + InputKind::KeyDown | InputKind::KeyUp => { + let down = event.kind == InputKind::KeyDown; + if let Some(evdev) = vk_to_evdev(event.code as u8) { + self.keyboard.key(t, evdev as u32, if down { 1 } else { 0 }); + self.send_modifiers(evdev, down); + } else { + tracing::debug!(vk = event.code, "unmapped VK keycode — dropped"); + } + } + InputKind::GamepadButton | InputKind::GamepadAxis => {} // not yet injected + } + // Surface protocol errors / disconnects, then push the batch to the compositor. + self.queue + .dispatch_pending(&mut self.globals) + .context("wayland dispatch")?; + self.conn.flush().context("wayland flush")?; + Ok(()) + } + } + + /// Create an anonymous in-memory file holding `s` + a trailing NUL (for the keymap fd). + fn memfd_with(s: &str) -> Result { + let name = b"lumen-keymap\0"; + let fd = + unsafe { libc::memfd_create(name.as_ptr() as *const libc::c_char, libc::MFD_CLOEXEC) }; + if fd < 0 { + bail!("memfd_create failed: {}", std::io::Error::last_os_error()); + } + let mut f = unsafe { std::fs::File::from_raw_fd(fd) }; + f.write_all(s.as_bytes()).context("write keymap")?; + f.write_all(&[0]).context("write keymap NUL")?; + Ok(f) } } diff --git a/scripts/bootstrap-ubuntu.sh b/scripts/bootstrap-ubuntu.sh index 02b7ac4..61d1b68 100755 --- a/scripts/bootstrap-ubuntu.sh +++ b/scripts/bootstrap-ubuntu.sh @@ -89,7 +89,7 @@ apt_install "build toolchain" build-essential pkg-config cmake clang libclang- apt_install "PipeWire + dev" pipewire pipewire-pulse wireplumber libpipewire-0.3-dev libspa-0.2-dev apt_install "desktop portals" xdg-desktop-portal xdg-desktop-portal-wlr xdg-desktop-portal-gtk apt_install "Sway + wlroots" sway swaybg xwayland wlr-randr foot seatd -apt_install "Wayland dev" libwayland-dev wayland-protocols wayland-utils +apt_install "Wayland dev" libwayland-dev wayland-protocols wayland-utils libxkbcommon-dev apt_install "DRM/EGL/GBM/VA" libdrm-dev libgbm-dev libgbm1 libegl-dev libegl1 libgles-dev mesa-common-dev libva-dev apt_install "capture + dbus" wf-recorder grim dbus-user-session drm-info mesa-utils apt_try "NVIDIA EGL platform (multiverse)" libnvidia-egl-wayland1 libnvidia-egl-gbm1