feat(core+host+android): committed-text input plane (IME path) — InputKind::TextInput

The VK key-event vocabulary cannot express text an input method COMMITS
(autocorrect, gesture typing, non-Latin scripts, emoji). Add a first-class
text event and negotiate it:

- punktfunk-core: InputKind::TextInput (= 15) carries one Unicode scalar per
  event in `code`; HOST_CAP_TEXT_INPUT (0x04) in Welcome::host_caps.
- Host advertises the cap only where the session's inject backend can type
  text: Windows SendInput (KEYEVENTF_UNICODE, surrogate-pair aware) and the
  Linux wlroots backend — a dedicated second zwp_virtual_keyboard whose xkb
  keymap grows Unicode keysyms on demand (the wtype model), so keymap
  re-uploads never disturb the main device's layout/modifier state. The
  KWin-fake-input/libei/gamescope backends can only press layout keycodes, so
  those sessions don't set the bit and clients keep the VK fallback.
- GameStream plane: Moonlight's UTF-8 text packet (MAGIC_UTF8, previously
  recognized-and-dropped) now decodes to the same TextInput events.
- Android: KeyCaptureView picks a real editable InputConnection when the host
  has the cap — the IME runs its full machinery, mirrored to the host live via
  common-prefix diffs of the composition (backspaces + new suffix), with
  setComposingRegion adopting committed text so autocorrect-revert flows diff
  instead of retyping; newline→Enter, deleteSurroundingText→Backspace/Delete.
  Older hosts keep the TYPE_NULL raw-key path unchanged.
- keymap: media VKs (0xB0-0xB3) → evdev so the Android media keys land on
  Linux hosts too.

Verified: punktfunk-core + host gamestream + pf-inject tests green on Linux
(Ubuntu box), clippy clean; Android app+native builds.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-22 19:19:11 +02:00
co-authored by Claude Fable 5
parent ddb72edcba
commit abe6228b42
14 changed files with 460 additions and 20 deletions
@@ -424,6 +424,9 @@ impl InputInjector for KwinFakeInjector {
self.fake.touch_up(event.code);
self.fake.touch_frame();
}
// fake_input can only press host-layout keycodes — no committed-text path (the
// HOST_CAP_TEXT_INPUT cap is not advertised on this backend).
InputKind::TextInput => {}
// Gamepads are injected through uinput, not the compositor.
InputKind::GamepadState
| InputKind::GamepadButton
+6 -1
View File
@@ -438,6 +438,7 @@ fn kind_bit(kind: InputKind) -> u32 {
InputKind::GamepadState => 12,
InputKind::GamepadRemove => 13,
InputKind::GamepadArrival => 14,
InputKind::TextInput => 15,
};
1 << i
}
@@ -670,6 +671,9 @@ impl EiState {
| InputKind::GamepadAxis
| InputKind::GamepadRemove
| InputKind::GamepadArrival => return, // uinput path (later)
// libei presses keycodes against the server's negotiated keymap — no committed-text
// path (the HOST_CAP_TEXT_INPUT cap is not advertised on this backend).
InputKind::TextInput => return,
};
self.injected += 1;
let n = self.injected;
@@ -844,7 +848,8 @@ impl EiState {
| InputKind::GamepadButton
| InputKind::GamepadAxis
| InputKind::GamepadRemove
| InputKind::GamepadArrival => emitted = false,
| InputKind::GamepadArrival
| InputKind::TextInput => emitted = false,
}
if emitted {
+96
View File
@@ -98,9 +98,26 @@ pub struct WlrootsInjector {
keyboard: ZwpVirtualKeyboardV1,
xkb_state: xkb::State,
_keymap_file: std::fs::File, // keep the memfd alive for the compositor's mmap
/// Dedicated committed-text device ([`InputKind::TextInput`]), created on first use.
text: Option<TextKeyboard>,
start: Instant,
}
/// Cap on distinct characters the dynamic text keymap holds before it restarts from scratch
/// (keycodes grow upward from 9; xkb tops out at 255, so stay well under).
const TEXT_KEYMAP_MAX: usize = 200;
/// The dedicated **text** virtual keyboard: types committed IME text (`InputKind::TextInput`,
/// one Unicode scalar per event) by growing a keymap of Unicode keysyms on demand and pressing
/// the character's keycode — the `wtype` model. A separate `zwp_virtual_keyboard` so keymap
/// re-uploads never disturb the main device's layout/modifier state that VK key events ride on.
struct TextKeyboard {
keyboard: ZwpVirtualKeyboardV1,
/// Characters in keycode order: `chars[i]` types on wire keycode `i + 1` (xkb `i + 9`).
chars: Vec<char>,
_keymap_file: Option<std::fs::File>, // keep the memfd alive for the compositor's mmap
}
impl WlrootsInjector {
pub fn open() -> Result<Self> {
let conn = Connection::connect_to_env()
@@ -171,6 +188,7 @@ impl WlrootsInjector {
keyboard,
xkb_state,
_keymap_file: file,
text: None,
start: Instant::now(),
})
}
@@ -179,6 +197,54 @@ impl WlrootsInjector {
self.start.elapsed().as_millis() as u32
}
/// Type one committed-text Unicode scalar on the dedicated text device (created lazily),
/// growing its keymap when the character is new. Control characters are dropped — Enter,
/// Backspace and Tab ride the VK key-event path.
fn type_text(&mut self, cp: u32) -> Result<()> {
let Some(ch) = char::from_u32(cp) else {
return Ok(()); // lone surrogate / out of range
};
if ch.is_control() {
return Ok(());
}
if self.text.is_none() {
let (Some(mgr), Some(seat)) =
(self.globals.keyboard_mgr.clone(), self.globals.seat.clone())
else {
return Ok(());
};
let kb = mgr.create_virtual_keyboard(&seat, &self.queue.handle(), ());
self.text = Some(TextKeyboard {
keyboard: kb,
chars: Vec::new(),
_keymap_file: None,
});
}
let t = self.now_ms();
let text = self.text.as_mut().expect("created above");
let code = match text.chars.iter().position(|&c| c == ch) {
Some(i) => (i + 1) as u32,
None => {
if text.chars.len() >= TEXT_KEYMAP_MAX {
text.chars.clear(); // restart the map; old codes are re-assigned lazily
}
text.chars.push(ch);
let keymap_str = text_keymap(&text.chars);
let file = memfd_with(&keymap_str)?;
text.keyboard.keymap(
1, /* XKB_V1 */
file.as_fd(),
keymap_str.len() as u32 + 1,
);
text._keymap_file = Some(file);
text.chars.len() as u32
}
};
text.keyboard.key(t, code, 1);
text.keyboard.key(t, code, 0);
Ok(())
}
/// Update xkb state for a key and tell the compositor the resulting modifier mask.
fn send_modifiers(&mut self, evdev: u16, down: bool) {
let kc = xkb::Keycode::new(evdev as u32 + 8); // evdev -> xkb keycode
@@ -254,6 +320,9 @@ impl InputInjector for WlrootsInjector {
tracing::debug!(vk = event.code, "unmapped VK keycode — dropped");
}
}
InputKind::TextInput => {
self.type_text(event.code)?;
}
InputKind::GamepadState
| InputKind::GamepadButton
| InputKind::GamepadAxis
@@ -271,6 +340,33 @@ impl InputInjector for WlrootsInjector {
}
}
/// Build a minimal xkb keymap whose keycode `i + 9` (wire code `i + 1`) types `chars[i]`, using
/// Unicode keysym names (`U<hex>` — xkbcommon resolves them for any scalar, emoji included).
/// Types/compat `include "complete"` mirrors `wtype`'s generated keymap — proven on wlroots
/// compositors, and the system XKB data is present (the main keymap compiled from it in `open`).
fn text_keymap(chars: &[char]) -> String {
use std::fmt::Write as _;
let mut keycodes = String::new();
let mut symbols = String::new();
for (i, ch) in chars.iter().enumerate() {
let _ = writeln!(keycodes, " <T{i}> = {};", i + 9);
let _ = writeln!(symbols, " key <T{i}> {{ [ U{:04X} ] }};", *ch as u32);
}
format!(
"xkb_keymap {{\n\
xkb_keycodes \"punktfunk-text\" {{\n\
minimum = 8;\n\
maximum = {};\n\
{keycodes}\
}};\n\
xkb_types \"punktfunk-text\" {{ include \"complete\" }};\n\
xkb_compatibility \"punktfunk-text\" {{ include \"complete\" }};\n\
xkb_symbols \"punktfunk-text\" {{\n{symbols} }};\n\
}};\n",
chars.len() + 9,
)
}
/// Create an anonymous in-memory file holding `s` + a trailing NUL (for the keymap fd).
fn memfd_with(s: &str) -> Result<std::fs::File> {
let name = b"punktfunk-keymap\0";