feat(core+host+android): committed-text input plane (IME path) — InputKind::TextInput

The VK key-event vocabulary cannot express text an input method COMMITS
(autocorrect, gesture typing, non-Latin scripts, emoji). Add a first-class
text event and negotiate it:

- punktfunk-core: InputKind::TextInput (= 15) carries one Unicode scalar per
  event in `code`; HOST_CAP_TEXT_INPUT (0x04) in Welcome::host_caps.
- Host advertises the cap only where the session's inject backend can type
  text: Windows SendInput (KEYEVENTF_UNICODE, surrogate-pair aware) and the
  Linux wlroots backend — a dedicated second zwp_virtual_keyboard whose xkb
  keymap grows Unicode keysyms on demand (the wtype model), so keymap
  re-uploads never disturb the main device's layout/modifier state. The
  KWin-fake-input/libei/gamescope backends can only press layout keycodes, so
  those sessions don't set the bit and clients keep the VK fallback.
- GameStream plane: Moonlight's UTF-8 text packet (MAGIC_UTF8, previously
  recognized-and-dropped) now decodes to the same TextInput events.
- Android: KeyCaptureView picks a real editable InputConnection when the host
  has the cap — the IME runs its full machinery, mirrored to the host live via
  common-prefix diffs of the composition (backspaces + new suffix), with
  setComposingRegion adopting committed text so autocorrect-revert flows diff
  instead of retyping; newline→Enter, deleteSurroundingText→Backspace/Delete.
  Older hosts keep the TYPE_NULL raw-key path unchanged.
- keymap: media VKs (0xB0-0xB3) → evdev so the Android media keys land on
  Linux hosts too.

Verified: punktfunk-core + host gamestream + pf-inject tests green on Linux
(Ubuntu box), clippy clean; Android app+native builds.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

crates/pf-inject/src/inject/keymap.rs

@@ -36,6 +36,12 @@ pub fn vk_to_evdev(vk: u8) -> Option<u16> {
         0x2D => Some(110), // VK_INSERT   -> KEY_INSERT
         0x2E => Some(111), // VK_DELETE   -> KEY_DELETE
 
+        // --- Consumer/media keys (Android TV remotes, keyboard media rows) ---
+        0xB0 => Some(163), // VK_MEDIA_NEXT_TRACK -> KEY_NEXTSONG
+        0xB1 => Some(165), // VK_MEDIA_PREV_TRACK -> KEY_PREVIOUSSONG
+        0xB2 => Some(166), // VK_MEDIA_STOP       -> KEY_STOPCD
+        0xB3 => Some(164), // VK_MEDIA_PLAY_PAUSE -> KEY_PLAYPAUSE
+
         // --- Generic modifiers ---
         0x10 => Some(42), // VK_SHIFT   -> KEY_LEFTSHIFT
         0x11 => Some(29), // VK_CONTROL -> KEY_LEFTCTRL

crates/pf-inject/src/inject/linux/kwin_fake_input.rs

@@ -424,6 +424,9 @@ impl InputInjector for KwinFakeInjector {
                 self.fake.touch_up(event.code);
                 self.fake.touch_frame();
             }
+            // fake_input can only press host-layout keycodes — no committed-text path (the
+            // HOST_CAP_TEXT_INPUT cap is not advertised on this backend).
+            InputKind::TextInput => {}
             // Gamepads are injected through uinput, not the compositor.
             InputKind::GamepadState
             | InputKind::GamepadButton

crates/pf-inject/src/inject/linux/libei.rs

@@ -438,6 +438,7 @@ fn kind_bit(kind: InputKind) -> u32 {
         InputKind::GamepadState => 12,
         InputKind::GamepadRemove => 13,
         InputKind::GamepadArrival => 14,
+        InputKind::TextInput => 15,
     };
     1 << i
 }
@@ -670,6 +671,9 @@ impl EiState {
             | InputKind::GamepadAxis
             | InputKind::GamepadRemove
             | InputKind::GamepadArrival => return, // uinput path (later)
+            // libei presses keycodes against the server's negotiated keymap — no committed-text
+            // path (the HOST_CAP_TEXT_INPUT cap is not advertised on this backend).
+            InputKind::TextInput => return,
         };
         self.injected += 1;
         let n = self.injected;
@@ -844,7 +848,8 @@ impl EiState {
             | InputKind::GamepadButton
             | InputKind::GamepadAxis
             | InputKind::GamepadRemove
-            | InputKind::GamepadArrival => emitted = false,
+            | InputKind::GamepadArrival
+            | InputKind::TextInput => emitted = false,
         }
 
         if emitted {

crates/pf-inject/src/inject/linux/wlr.rs

@@ -98,9 +98,26 @@ pub struct WlrootsInjector {
     keyboard: ZwpVirtualKeyboardV1,
     xkb_state: xkb::State,
     _keymap_file: std::fs::File, // keep the memfd alive for the compositor's mmap
+    /// Dedicated committed-text device ([`InputKind::TextInput`]), created on first use.
+    text: Option<TextKeyboard>,
     start: Instant,
 }
 
+/// Cap on distinct characters the dynamic text keymap holds before it restarts from scratch
+/// (keycodes grow upward from 9; xkb tops out at 255, so stay well under).
+const TEXT_KEYMAP_MAX: usize = 200;
+
+/// The dedicated **text** virtual keyboard: types committed IME text (`InputKind::TextInput`,
+/// one Unicode scalar per event) by growing a keymap of Unicode keysyms on demand and pressing
+/// the character's keycode — the `wtype` model. A separate `zwp_virtual_keyboard` so keymap
+/// re-uploads never disturb the main device's layout/modifier state that VK key events ride on.
+struct TextKeyboard {
+    keyboard: ZwpVirtualKeyboardV1,
+    /// Characters in keycode order: `chars[i]` types on wire keycode `i + 1` (xkb `i + 9`).
+    chars: Vec<char>,
+    _keymap_file: Option<std::fs::File>, // keep the memfd alive for the compositor's mmap
+}
+
 impl WlrootsInjector {
     pub fn open() -> Result<Self> {
         let conn = Connection::connect_to_env()
@@ -171,6 +188,7 @@ impl WlrootsInjector {
             keyboard,
             xkb_state,
             _keymap_file: file,
+            text: None,
             start: Instant::now(),
         })
     }
@@ -179,6 +197,54 @@ impl WlrootsInjector {
         self.start.elapsed().as_millis() as u32
     }
 
+    /// Type one committed-text Unicode scalar on the dedicated text device (created lazily),
+    /// growing its keymap when the character is new. Control characters are dropped — Enter,
+    /// Backspace and Tab ride the VK key-event path.
+    fn type_text(&mut self, cp: u32) -> Result<()> {
+        let Some(ch) = char::from_u32(cp) else {
+            return Ok(()); // lone surrogate / out of range
+        };
+        if ch.is_control() {
+            return Ok(());
+        }
+        if self.text.is_none() {
+            let (Some(mgr), Some(seat)) =
+                (self.globals.keyboard_mgr.clone(), self.globals.seat.clone())
+            else {
+                return Ok(());
+            };
+            let kb = mgr.create_virtual_keyboard(&seat, &self.queue.handle(), ());
+            self.text = Some(TextKeyboard {
+                keyboard: kb,
+                chars: Vec::new(),
+                _keymap_file: None,
+            });
+        }
+        let t = self.now_ms();
+        let text = self.text.as_mut().expect("created above");
+        let code = match text.chars.iter().position(|&c| c == ch) {
+            Some(i) => (i + 1) as u32,
+            None => {
+                if text.chars.len() >= TEXT_KEYMAP_MAX {
+                    text.chars.clear(); // restart the map; old codes are re-assigned lazily
+                }
+                text.chars.push(ch);
+                let keymap_str = text_keymap(&text.chars);
+                let file = memfd_with(&keymap_str)?;
+                text.keyboard.keymap(
+                    1, /* XKB_V1 */
+                    file.as_fd(),
+                    keymap_str.len() as u32 + 1,
+                );
+                text._keymap_file = Some(file);
+                text.chars.len() as u32
+            }
+        };
+        text.keyboard.key(t, code, 1);
+        text.keyboard.key(t, code, 0);
+        Ok(())
+    }
+
     /// Update xkb state for a key and tell the compositor the resulting modifier mask.
     fn send_modifiers(&mut self, evdev: u16, down: bool) {
         let kc = xkb::Keycode::new(evdev as u32 + 8); // evdev -> xkb keycode
@@ -254,6 +320,9 @@ impl InputInjector for WlrootsInjector {
                     tracing::debug!(vk = event.code, "unmapped VK keycode — dropped");
                 }
             }
+            InputKind::TextInput => {
+                self.type_text(event.code)?;
+            }
             InputKind::GamepadState
             | InputKind::GamepadButton
             | InputKind::GamepadAxis
@@ -271,6 +340,33 @@ impl InputInjector for WlrootsInjector {
     }
 }
 
+/// Build a minimal xkb keymap whose keycode `i + 9` (wire code `i + 1`) types `chars[i]`, using
+/// Unicode keysym names (`U<hex>` — xkbcommon resolves them for any scalar, emoji included).
+/// Types/compat `include "complete"` mirrors `wtype`'s generated keymap — proven on wlroots
+/// compositors, and the system XKB data is present (the main keymap compiled from it in `open`).
+fn text_keymap(chars: &[char]) -> String {
+    use std::fmt::Write as _;
+    let mut keycodes = String::new();
+    let mut symbols = String::new();
+    for (i, ch) in chars.iter().enumerate() {
+        let _ = writeln!(keycodes, "        <T{i}> = {};", i + 9);
+        let _ = writeln!(symbols, "        key <T{i}> {{ [ U{:04X} ] }};", *ch as u32);
+    }
+    format!(
+        "xkb_keymap {{\n\
+             xkb_keycodes \"punktfunk-text\" {{\n\
+                 minimum = 8;\n\
+                 maximum = {};\n\
+         {keycodes}\
+             }};\n\
+             xkb_types \"punktfunk-text\" {{ include \"complete\" }};\n\
+             xkb_compatibility \"punktfunk-text\" {{ include \"complete\" }};\n\
+             xkb_symbols \"punktfunk-text\" {{\n{symbols}    }};\n\
+         }};\n",
+        chars.len() + 9,
+    )
+}
+
 /// Create an anonymous in-memory file holding `s` + a trailing NUL (for the keymap fd).
 fn memfd_with(s: &str) -> Result<std::fs::File> {
     let name = b"punktfunk-keymap\0";

crates/pf-inject/src/inject/windows/sendinput.rs

@@ -27,10 +27,10 @@ use windows::Win32::System::StationsAndDesktops::{
 use windows::Win32::UI::Input::KeyboardAndMouse::{
     GetKeyboardLayout, MapVirtualKeyExW, SendInput, HKL, INPUT, INPUT_0, INPUT_KEYBOARD,
     INPUT_MOUSE, KEYBDINPUT, KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP, KEYEVENTF_SCANCODE,
-    MAPVK_VK_TO_VSC_EX, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_HWHEEL, MOUSEEVENTF_LEFTDOWN,
-    MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE,
-    MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK, MOUSEEVENTF_WHEEL,
-    MOUSEEVENTF_XDOWN, MOUSEEVENTF_XUP, MOUSEINPUT, VIRTUAL_KEY,
+    KEYEVENTF_UNICODE, MAPVK_VK_TO_VSC_EX, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_HWHEEL,
+    MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP,
+    MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK,
+    MOUSEEVENTF_WHEEL, MOUSEEVENTF_XDOWN, MOUSEEVENTF_XUP, MOUSEINPUT, VIRTUAL_KEY,
 };
 use windows::Win32::UI::WindowsAndMessaging::{
     GetForegroundWindow, GetSystemMetrics, GetWindowThreadProcessId, SM_CXVIRTUALSCREEN,
@@ -297,6 +297,33 @@ impl InputInjector for SendInputInjector {
                 };
                 self.send(&[key(ki)])
             }
+            InputKind::TextInput => {
+                // Committed IME text: one Unicode scalar per event, injected as
+                // `KEYEVENTF_UNICODE` packets (wScan = UTF-16 unit, no scancode/layout involved
+                // — the receiving app gets the character verbatim via WM_CHAR). An astral-plane
+                // scalar (emoji) is its surrogate pair, each unit down+up in order — exactly how
+                // Windows expects supplementary characters from unicode injection.
+                let Some(ch) = char::from_u32(event.code) else {
+                    return Ok(()); // lone surrogate / out of range — drop
+                };
+                if ch.is_control() {
+                    return Ok(()); // control chars ride the VK path (Enter/Backspace/Tab)
+                }
+                let mut units = [0u16; 2];
+                let mut inputs: Vec<INPUT> = Vec::with_capacity(4);
+                for &unit in ch.encode_utf16(&mut units).iter() {
+                    for flags in [KEYEVENTF_UNICODE, KEYEVENTF_UNICODE | KEYEVENTF_KEYUP] {
+                        inputs.push(key(KEYBDINPUT {
+                            wVk: VIRTUAL_KEY(0),
+                            wScan: unit,
+                            dwFlags: flags,
+                            time: 0,
+                            dwExtraInfo: 0,
+                        }));
+                    }
+                }
+                self.send(&inputs)
+            }
             // Gamepad goes through the XUSB backend. Touch: no SendInput equivalent -> no-op.
             InputKind::GamepadButton
             | InputKind::GamepadAxis

crates/pf-inject/src/lib.rs

@@ -174,6 +174,29 @@ pub fn default_backend() -> Backend {
     Backend::Unsupported
 }
 
+/// Whether the session's inject backend can type **committed text**
+/// ([`InputKind::TextInput`] — see `HOST_CAP_TEXT_INPUT`): Windows always (`KEYEVENTF_UNICODE`);
+/// Linux only on the wlroots backend (a dedicated virtual keyboard with a dynamically-grown
+/// Unicode keymap) — KWin fake-input/libei/gamescope can only press keycodes of the host layout.
+/// Consulted at Welcome time to advertise the cap; a mid-session backend switch away from a
+/// capable one just degrades to dropped text events (input is lossy by design).
+#[cfg(target_os = "windows")]
+pub fn text_input_supported() -> bool {
+    true
+}
+
+/// See the Windows variant: Linux types text only through the wlroots virtual-keyboard backend.
+#[cfg(target_os = "linux")]
+pub fn text_input_supported() -> bool {
+    matches!(default_backend(), Backend::WlrVirtual)
+}
+
+/// No injector ⇒ no text.
+#[cfg(not(any(target_os = "linux", target_os = "windows")))]
+pub fn text_input_supported() -> bool {
+    false
+}
+
 #[path = "inject/service.rs"]
 mod service;
 pub use service::InjectorService;

crates/punktfunk-core/src/input.rs

@@ -73,6 +73,17 @@ pub enum InputKind {
     /// [`HOST_CAP_GAMEPAD_STATE`](crate::quic::HOST_CAP_GAMEPAD_STATE); an older host ignores the
     /// unknown tag (every pad then uses the session-default kind — the pre-existing behaviour).
     GamepadArrival = 14,
+    /// One Unicode scalar of **committed text** — `code` = the scalar value, everything else 0.
+    ///
+    /// The IME path: the layout-independent VK key events cannot express text an input method
+    /// *commits* (autocorrect, gesture typing, non-Latin scripts, emoji), so a capable client
+    /// sends the committed characters verbatim and the host injects them directly (Windows
+    /// `KEYEVENTF_UNICODE`; Linux wlroots via a dynamically-grown Unicode keymap on a dedicated
+    /// virtual keyboard). A multi-character commit is consecutive events in order. Sent only when
+    /// the host advertised [`HOST_CAP_TEXT_INPUT`](crate::quic::HOST_CAP_TEXT_INPUT) — toward an
+    /// older host (or one whose inject backend can't type text) clients keep the best-effort VK
+    /// synthesis, and an older host ignores the unknown tag entirely.
+    TextInput = 15,
 }
 
 /// Pack a [`InputKind::GamepadRemove`] `flags` word (`seq << 24 | pad`) — the same low-byte-pad /
@@ -158,6 +169,7 @@ impl InputKind {
             12 => GamepadState,
             13 => GamepadRemove,
             14 => GamepadArrival,
+            15 => TextInput,
             _ => return None,
         })
     }
@@ -392,10 +404,27 @@ mod tests {
             };
             assert_eq!(InputEvent::decode(&e.encode()), Some(e));
         }
-        // GamepadRemove + GamepadArrival are valid kinds; 15 (one past them) is not.
+        // GamepadRemove/GamepadArrival/TextInput are valid kinds; 16 (one past them) is not.
         assert_eq!(InputKind::from_u8(13), Some(InputKind::GamepadRemove));
         assert_eq!(InputKind::from_u8(14), Some(InputKind::GamepadArrival));
-        assert_eq!(InputKind::from_u8(15), None);
+        assert_eq!(InputKind::from_u8(15), Some(InputKind::TextInput));
+        assert_eq!(InputKind::from_u8(16), None);
+    }
+
+    #[test]
+    fn text_input_roundtrip() {
+        // One Unicode scalar per event — BMP and astral (emoji) alike.
+        for cp in ['a' as u32, 'ß' as u32, '語' as u32, 0x1F600 /* 😀 */] {
+            let e = InputEvent {
+                kind: InputKind::TextInput,
+                _pad: [0; 3],
+                code: cp,
+                x: 0,
+                y: 0,
+                flags: 0,
+            };
+            assert_eq!(InputEvent::decode(&e.encode()), Some(e));
+        }
     }
 
     #[test]

crates/punktfunk-core/src/quic/caps.rs

@@ -71,6 +71,16 @@ pub const HOST_CAP_GAMEPAD_STATE: u8 = 0x01;
 /// trailing `host_caps` byte — no wire-layout change.
 pub const HOST_CAP_CLIPBOARD: u8 = 0x02;
 
+/// [`Welcome::host_caps`] bit: the host's active inject backend can type **committed text**
+/// ([`InputKind::TextInput`](crate::input::InputKind::TextInput) — one Unicode scalar per event):
+/// Windows (`KEYEVENTF_UNICODE`) and Linux wlroots (dynamic Unicode keymap on a dedicated virtual
+/// keyboard); the KWin/libei/gamescope backends can only press layout keycodes, so those sessions
+/// don't set it. A capable client routes its IME's committed text (autocorrect, gesture typing,
+/// non-Latin scripts, emoji) through `TextInput` instead of lossy VK synthesis; absent the bit it
+/// keeps the VK fallback. Packs into the existing trailing `host_caps` byte — no wire-layout
+/// change; an older host ignores the unknown input tag anyway (input is lossy by design).
+pub const HOST_CAP_TEXT_INPUT: u8 = 0x04;
+
 /// [`Hello::video_codecs`] bit: the client can decode H.264 / AVC. The GPU-less **software**
 /// encode path (openh264) emits H.264, so a client that wants to stream from a software host MUST
 /// advertise this.

crates/punktfunk-host/src/gamestream/input.rs

@@ -35,7 +35,24 @@ pub fn decode(plaintext: &[u8]) -> Vec<InputEvent> {
     if plaintext.len() < 4 || u16::from_le_bytes([plaintext[0], plaintext[1]]) != INPUT_DATA_TYPE {
         return Vec::new();
     }
-    decode_input_packet(&plaintext[4..]).into_iter().collect()
+    let p = &plaintext[4..];
+    // UTF-8 text (Moonlight's client-side keyboard commit) expands to one `TextInput` event per
+    // Unicode scalar — the only magic yielding more than one event, so it's handled before the
+    // single-event dispatch. Injected the same way as the native plane's IME text.
+    if p.len() >= 8 && u32::from_le_bytes([p[4], p[5], p[6], p[7]]) == MAGIC_UTF8 {
+        // NV_INPUT_HEADER.size (BE, excludes itself) counts magic + body.
+        let size = u32::from_be_bytes([p[0], p[1], p[2], p[3]]) as usize;
+        let body_len = size.saturating_sub(4).min(p.len() - 8);
+        return match std::str::from_utf8(&p[8..8 + body_len]) {
+            Ok(s) => s
+                .chars()
+                .filter(|c| !c.is_control())
+                .map(|c| ev(InputKind::TextInput, c as u32, 0, 0, 0))
+                .collect(),
+            Err(_) => Vec::new(),
+        };
+    }
+    decode_input_packet(p).into_iter().collect()
 }
 
 fn decode_input_packet(p: &[u8]) -> Option<InputEvent> {
@@ -89,7 +106,7 @@ fn decode_input_packet(p: &[u8]) -> Option<InputEvent> {
                 modifiers | crate::inject::KEY_FLAG_SEMANTIC_VK,
             )
         }
-        // UTF-8 text, gamepad, pen, touch, haptics — not yet injected.
+        // Gamepad, pen, touch, haptics — not yet injected. (UTF-8 text is handled in `decode`.)
         _ => return None,
     })
 }
@@ -144,6 +161,21 @@ mod tests {
         assert_eq!(ev[0].flags, 0x04 | crate::inject::KEY_FLAG_SEMANTIC_VK);
     }
 
+    #[test]
+    fn decodes_utf8_text_per_scalar() {
+        // "aß😀" — ASCII, Latin-1, and an astral scalar; one TextInput event per scalar.
+        let pt = wrap(MAGIC_UTF8, "aß😀".as_bytes());
+        let ev = decode(&pt);
+        assert_eq!(ev.len(), 3);
+        assert!(ev.iter().all(|e| e.kind == InputKind::TextInput));
+        assert_eq!(ev[0].code, 'a' as u32);
+        assert_eq!(ev[1].code, 'ß' as u32);
+        assert_eq!(ev[2].code, 0x1F600);
+        // Truncated / invalid UTF-8 decodes to nothing rather than mojibake.
+        let bad = wrap(MAGIC_UTF8, &[0xff, 0xfe]);
+        assert!(decode(&bad).is_empty());
+    }
+
     #[test]
     fn ignores_non_input_type() {
         let mut pt = vec![0x00, 0x02]; // type 0x0200 (keepalive)

crates/punktfunk-host/src/native/handshake.rs

@@ -444,6 +444,15 @@ pub(super) async fn negotiate(
                 punktfunk_core::quic::HOST_CAP_CLIPBOARD
             } else {
                 0
+            }
+            // Committed-text injection (InputKind::TextInput): only where the session's inject
+            // backend can actually type text — Windows SendInput (KEYEVENTF_UNICODE) and the
+            // Linux wlroots virtual keyboard (dynamic Unicode keymap). Clients without the bit
+            // keep their VK-synthesis fallback for IME text.
+            | if crate::inject::text_input_supported() {
+                punktfunk_core::quic::HOST_CAP_TEXT_INPUT
+            } else {
+                0
             },
         // The negotiated session AEAD (resolved above) + its 32-byte key toward a ChaCha
         // client; toward everyone else cipher 0 keeps the Welcome byte-identical to the