diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt index a80f2774..9ebd870b 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt @@ -450,6 +450,9 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { factory = { ctx -> KeyCaptureView(ctx).also { v -> keyCapture = v + // Real IME text path when the host types committed text (see KeyCaptureView). + v.textHandle = + if (NativeBridge.nativeTextInputSupported(handle)) handle else 0L v.setOnCapturedPointerListener { _, ev -> (ctx as? MainActivity)?.mouseForwarder?.onCapturedPointer(ev) ?: false } @@ -513,12 +516,13 @@ private fun RemotePointerHint(modifier: Modifier = Modifier) { /** * Invisible focus anchor for typing on the host: the three-finger swipe summons the device IME - * onto this view. `TYPE_NULL` puts the IME in "dumb keyboard" mode — it delivers raw [KeyEvent]s - * (no composing text, no autocorrect), which flow through `MainActivity.dispatchKeyEvent` → - * `Keymap.toVk` → the host, the exact path a hardware keyboard takes. Text an IME insists on - * committing instead still arrives: the non-editable [BaseInputConnection] synthesizes KeyEvents - * for it via `KeyCharacterMap` (with Shift carried as meta state — see the IME-shift wrap in - * `MainActivity.dispatchKeyEvent`). + * onto this view. Two IME models, picked by the host's capabilities: + * * **Text path** ([textHandle] set — the host advertised `HOST_CAP_TEXT_INPUT`): a real + * editable [HostTextConnection], so the IME gives autocorrect, gesture typing, non-Latin + * composition and emoji, all mirrored to the host as committed text + diffs. + * * **Fallback** (older host): `TYPE_NULL` puts the IME in "dumb keyboard" mode — raw + * [KeyEvent]s flow through `MainActivity.dispatchKeyEvent` → `Keymap.toVk` → the host, the + * exact path a hardware keyboard takes (with the IME-shift wrap documented there). * * Doubles as the pointer-capture grab target: a grab needs a focusable view, and captured-pointer * events are delivered to it (routed to [MouseForwarder.onCapturedPointer] via the listener the @@ -530,6 +534,9 @@ private class KeyCaptureView(context: Context) : View(context) { isFocusableInTouchMode = true } + /** The session handle when the host types committed text; `0` = VK-only fallback. */ + var textHandle: Long = 0L + /** Whether [setImeVisible] last showed the IME — for toggle-style callers (remote pointer). */ var imeShown = false private set @@ -537,9 +544,16 @@ private class KeyCaptureView(context: Context) : View(context) { override fun onCheckIsTextEditor(): Boolean = true override fun onCreateInputConnection(outAttrs: EditorInfo): InputConnection { - outAttrs.inputType = InputType.TYPE_NULL - outAttrs.imeOptions = EditorInfo.IME_FLAG_NO_EXTRACT_UI or EditorInfo.IME_FLAG_NO_FULLSCREEN - return BaseInputConnection(this, false) + outAttrs.imeOptions = EditorInfo.IME_FLAG_NO_EXTRACT_UI or + EditorInfo.IME_FLAG_NO_FULLSCREEN or EditorInfo.IME_FLAG_NO_ENTER_ACTION + return if (textHandle != 0L) { + outAttrs.inputType = InputType.TYPE_CLASS_TEXT or + InputType.TYPE_TEXT_FLAG_AUTO_CORRECT or InputType.TYPE_TEXT_FLAG_MULTI_LINE + HostTextConnection(this, textHandle) + } else { + outAttrs.inputType = InputType.TYPE_NULL + BaseInputConnection(this, false) + } } fun setImeVisible(show: Boolean) { @@ -554,3 +568,113 @@ private class KeyCaptureView(context: Context) : View(context) { } } } + +/** + * IME → host text bridge (the `HOST_CAP_TEXT_INPUT` path): a real **editable** connection, so + * the IME runs its full machinery (autocorrect, gesture typing, non-Latin composition), mirrored + * to the host as it happens. The one piece of host-side state tracked is *what the host currently + * shows of the active composition* ([sentComposition]): composing updates send a common-prefix + * diff (backspaces + the new suffix) so corrections materialize live on the host; a commit + * settles it. [setComposingRegion] adopts already-committed text as the active composition + * (autocorrect-revert / backspace-into-word flows), so the next update diffs against it instead + * of retyping. Newlines become Enter taps; [deleteSurroundingText] becomes Backspace/Delete taps. + * + * Known approximation: diff lengths are counted in Unicode scalars, assuming one host Backspace + * deletes one scalar — true for the composition text IMEs actually produce (emoji and other + * multi-unit graphemes commit directly rather than composing). + */ +private class HostTextConnection( + view: KeyCaptureView, + private val handle: Long, +) : BaseInputConnection(view, true) { + /** What the host currently shows of the active composition ("" = none). */ + private var sentComposition = "" + + override fun commitText(text: CharSequence, newCursorPosition: Int): Boolean { + retype(text.toString()) + sentComposition = "" + val ok = super.commitText(text, newCursorPosition) + trimEditable() + return ok + } + + override fun setComposingText(text: CharSequence, newCursorPosition: Int): Boolean { + retype(text.toString()) + return super.setComposingText(text, newCursorPosition) + } + + override fun finishComposingText(): Boolean { + // The composition text stands as committed — the host already shows it verbatim. + sentComposition = "" + return super.finishComposingText() + } + + override fun setComposingRegion(start: Int, end: Int): Boolean { + val e = editable + if (e != null) { + val a = start.coerceIn(0, e.length) + val b = end.coerceIn(0, e.length) + sentComposition = e.subSequence(minOf(a, b), maxOf(a, b)).toString() + } + return super.setComposingRegion(start, end) + } + + override fun deleteSurroundingText(beforeLength: Int, afterLength: Int): Boolean { + repeat(beforeLength.coerceIn(0, MAX_TAPS)) { tapVk(VK_BACK) } + repeat(afterLength.coerceIn(0, MAX_TAPS)) { tapVk(VK_DELETE) } + return super.deleteSurroundingText(beforeLength, afterLength) + } + + override fun performEditorAction(actionCode: Int): Boolean { + tapVk(VK_RETURN) + return true + } + + /** Replace the host's view of the composition with [text] via a common-prefix diff. */ + private fun retype(text: String) { + var common = sentComposition.commonPrefixWith(text) + // Never split a surrogate pair mid-diff — back off to the pair boundary. + if (common.isNotEmpty() && common.last().isHighSurrogate()) { + common = common.dropLast(1) + } + val stale = sentComposition.substring(common.length) + repeat(stale.codePointCount(0, stale.length).coerceAtMost(MAX_TAPS)) { tapVk(VK_BACK) } + sendText(text.substring(common.length)) + sentComposition = text + } + + /** Forward literal text, turning newlines into Enter taps (control chars never ride text). */ + private fun sendText(s: String) { + var chunk = StringBuilder() + for (ch in s) { + if (ch == '\n') { + if (chunk.isNotEmpty()) { + NativeBridge.nativeSendText(handle, chunk.toString()) + chunk = StringBuilder() + } + tapVk(VK_RETURN) + } else { + chunk.append(ch) + } + } + if (chunk.isNotEmpty()) NativeBridge.nativeSendText(handle, chunk.toString()) + } + + private fun tapVk(vk: Int) { + NativeBridge.nativeSendKey(handle, vk, true, 0) + NativeBridge.nativeSendKey(handle, vk, false, 0) + } + + /** Bound the mirror buffer: once nothing is composing, old text serves no purpose. */ + private fun trimEditable() { + val e = editable ?: return + if (getComposingSpanStart(e) == -1 && e.length > 4000) e.clear() + } + + private companion object { + const val VK_BACK = 0x08 + const val VK_RETURN = 0x0D + const val VK_DELETE = 0x2E + const val MAX_TAPS = 256 + } +} diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index 5445f84d..a08469b8 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -287,6 +287,20 @@ object NativeBridge { /** One key transition. vk: Windows VK (0 = dropped by Rust). mods: VK modifier mask (0 for now). */ external fun nativeSendKey(handle: Long, vk: Int, down: Boolean, mods: Int) + /** + * Whether the host advertised committed-text injection (`HOST_CAP_TEXT_INPUT`) — its inject + * backend can type Unicode text directly. Picks the real IME `InputConnection` (autocorrect, + * gesture typing, non-Latin scripts) over the TYPE_NULL raw-key fallback. False on `0`. + */ + external fun nativeTextInputSupported(handle: Long): Boolean + + /** + * Committed IME text → one `TextInput` wire event per Unicode scalar, in order. Control + * characters are skipped natively (Enter/Backspace ride [nativeSendKey]). Only meaningful + * when [nativeTextInputSupported] returned true — older hosts ignore the events. + */ + external fun nativeSendText(handle: Long, text: String) + // ---- Gamepad: each controller forwarded on its own wire pad index (0..15, low byte of flags) ---- // The pad index is assigned per Android device by GamepadRouter; a single controller lands on 0, // so its wire is byte-identical to the old single-pad path. The core folds the per-transition diff --git a/clients/android/native/src/session/input.rs b/clients/android/native/src/session/input.rs index c082e6e0..ca1f2267 100644 --- a/clients/android/native/src/session/input.rs +++ b/clients/android/native/src/session/input.rs @@ -6,11 +6,11 @@ //! conventions: buttons 1=left/2=middle/3=right/4=X1/5=X2; scroll axis 0=vertical/1=horizontal, //! signed 120-unit delta, +=up/right; keys are Windows VK (mapped from KEYCODE_* on the Kotlin side). -use jni::objects::{JByteBuffer, JObject}; +use jni::objects::{JByteBuffer, JObject, JString}; use jni::sys::{jboolean, jint, jlong}; use jni::JNIEnv; use punktfunk_core::input::{InputEvent, InputKind}; -use punktfunk_core::quic::{RichInput, HID_REPORT_MAX}; +use punktfunk_core::quic::{RichInput, HID_REPORT_MAX, HOST_CAP_TEXT_INPUT}; use super::SessionHandle; @@ -145,6 +145,45 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendKey( send_event(handle, kind, vk as u32, 0, 0, mods as u32); } +/// `NativeBridge.nativeTextInputSupported(handle)` — whether the host advertised +/// `HOST_CAP_TEXT_INPUT` (its inject backend types committed text), so the Kotlin side can pick +/// the real IME `InputConnection` over the TYPE_NULL raw-key fallback. `0` handle → false. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeTextInputSupported( + _env: JNIEnv, + _this: JObject, + handle: jlong, +) -> jboolean { + if handle == 0 { + return 0; + } + // SAFETY: live handle per the nativeConnect/nativeClose contract; host_caps is &self. + let h = unsafe { &*(handle as *const SessionHandle) }; + u8::from(h.client.host_caps() & HOST_CAP_TEXT_INPUT != 0) +} + +/// `NativeBridge.nativeSendText(handle, text)` — committed IME text, one `TextInput` event per +/// Unicode scalar (`code` = the scalar; multi-char commits are consecutive events in order). +/// Control characters are skipped — Enter/Backspace/Tab ride the VK key path. Call only when +/// [`Java_io_unom_punktfunk_kit_NativeBridge_nativeTextInputSupported`] returned true. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendText( + mut env: JNIEnv, + _this: JObject, + handle: jlong, + text: JString, +) { + if handle == 0 { + return; + } + let Ok(s) = env.get_string(&text) else { + return; + }; + for ch in String::from(s).chars().filter(|c| !c.is_control()) { + send_event(handle, InputKind::TextInput, ch as u32, 0, 0, 0); + } +} + // ---- Gamepad: Kotlin captures (KeyEvent/MotionEvent) → NativeClient::send_input --------------- // Multi-pad model: each physical controller is forwarded on its own wire pad index (0..15), carried // in the low byte of `flags` on every per-pad event — the Kotlin side (`GamepadRouter`) assigns a diff --git a/crates/pf-inject/src/inject/keymap.rs b/crates/pf-inject/src/inject/keymap.rs index 401a54e3..07fbda97 100644 --- a/crates/pf-inject/src/inject/keymap.rs +++ b/crates/pf-inject/src/inject/keymap.rs @@ -36,6 +36,12 @@ pub fn vk_to_evdev(vk: u8) -> Option { 0x2D => Some(110), // VK_INSERT -> KEY_INSERT 0x2E => Some(111), // VK_DELETE -> KEY_DELETE + // --- Consumer/media keys (Android TV remotes, keyboard media rows) --- + 0xB0 => Some(163), // VK_MEDIA_NEXT_TRACK -> KEY_NEXTSONG + 0xB1 => Some(165), // VK_MEDIA_PREV_TRACK -> KEY_PREVIOUSSONG + 0xB2 => Some(166), // VK_MEDIA_STOP -> KEY_STOPCD + 0xB3 => Some(164), // VK_MEDIA_PLAY_PAUSE -> KEY_PLAYPAUSE + // --- Generic modifiers --- 0x10 => Some(42), // VK_SHIFT -> KEY_LEFTSHIFT 0x11 => Some(29), // VK_CONTROL -> KEY_LEFTCTRL diff --git a/crates/pf-inject/src/inject/linux/kwin_fake_input.rs b/crates/pf-inject/src/inject/linux/kwin_fake_input.rs index d2b5a641..0231b68f 100644 --- a/crates/pf-inject/src/inject/linux/kwin_fake_input.rs +++ b/crates/pf-inject/src/inject/linux/kwin_fake_input.rs @@ -424,6 +424,9 @@ impl InputInjector for KwinFakeInjector { self.fake.touch_up(event.code); self.fake.touch_frame(); } + // fake_input can only press host-layout keycodes — no committed-text path (the + // HOST_CAP_TEXT_INPUT cap is not advertised on this backend). + InputKind::TextInput => {} // Gamepads are injected through uinput, not the compositor. InputKind::GamepadState | InputKind::GamepadButton diff --git a/crates/pf-inject/src/inject/linux/libei.rs b/crates/pf-inject/src/inject/linux/libei.rs index 1737319a..20c0ba39 100644 --- a/crates/pf-inject/src/inject/linux/libei.rs +++ b/crates/pf-inject/src/inject/linux/libei.rs @@ -438,6 +438,7 @@ fn kind_bit(kind: InputKind) -> u32 { InputKind::GamepadState => 12, InputKind::GamepadRemove => 13, InputKind::GamepadArrival => 14, + InputKind::TextInput => 15, }; 1 << i } @@ -670,6 +671,9 @@ impl EiState { | InputKind::GamepadAxis | InputKind::GamepadRemove | InputKind::GamepadArrival => return, // uinput path (later) + // libei presses keycodes against the server's negotiated keymap — no committed-text + // path (the HOST_CAP_TEXT_INPUT cap is not advertised on this backend). + InputKind::TextInput => return, }; self.injected += 1; let n = self.injected; @@ -844,7 +848,8 @@ impl EiState { | InputKind::GamepadButton | InputKind::GamepadAxis | InputKind::GamepadRemove - | InputKind::GamepadArrival => emitted = false, + | InputKind::GamepadArrival + | InputKind::TextInput => emitted = false, } if emitted { diff --git a/crates/pf-inject/src/inject/linux/wlr.rs b/crates/pf-inject/src/inject/linux/wlr.rs index 7e66fe01..c506e5a8 100644 --- a/crates/pf-inject/src/inject/linux/wlr.rs +++ b/crates/pf-inject/src/inject/linux/wlr.rs @@ -98,9 +98,26 @@ pub struct WlrootsInjector { keyboard: ZwpVirtualKeyboardV1, xkb_state: xkb::State, _keymap_file: std::fs::File, // keep the memfd alive for the compositor's mmap + /// Dedicated committed-text device ([`InputKind::TextInput`]), created on first use. + text: Option, start: Instant, } +/// Cap on distinct characters the dynamic text keymap holds before it restarts from scratch +/// (keycodes grow upward from 9; xkb tops out at 255, so stay well under). +const TEXT_KEYMAP_MAX: usize = 200; + +/// The dedicated **text** virtual keyboard: types committed IME text (`InputKind::TextInput`, +/// one Unicode scalar per event) by growing a keymap of Unicode keysyms on demand and pressing +/// the character's keycode — the `wtype` model. A separate `zwp_virtual_keyboard` so keymap +/// re-uploads never disturb the main device's layout/modifier state that VK key events ride on. +struct TextKeyboard { + keyboard: ZwpVirtualKeyboardV1, + /// Characters in keycode order: `chars[i]` types on wire keycode `i + 1` (xkb `i + 9`). + chars: Vec, + _keymap_file: Option, // keep the memfd alive for the compositor's mmap +} + impl WlrootsInjector { pub fn open() -> Result { let conn = Connection::connect_to_env() @@ -171,6 +188,7 @@ impl WlrootsInjector { keyboard, xkb_state, _keymap_file: file, + text: None, start: Instant::now(), }) } @@ -179,6 +197,54 @@ impl WlrootsInjector { self.start.elapsed().as_millis() as u32 } + /// Type one committed-text Unicode scalar on the dedicated text device (created lazily), + /// growing its keymap when the character is new. Control characters are dropped — Enter, + /// Backspace and Tab ride the VK key-event path. + fn type_text(&mut self, cp: u32) -> Result<()> { + let Some(ch) = char::from_u32(cp) else { + return Ok(()); // lone surrogate / out of range + }; + if ch.is_control() { + return Ok(()); + } + if self.text.is_none() { + let (Some(mgr), Some(seat)) = + (self.globals.keyboard_mgr.clone(), self.globals.seat.clone()) + else { + return Ok(()); + }; + let kb = mgr.create_virtual_keyboard(&seat, &self.queue.handle(), ()); + self.text = Some(TextKeyboard { + keyboard: kb, + chars: Vec::new(), + _keymap_file: None, + }); + } + let t = self.now_ms(); + let text = self.text.as_mut().expect("created above"); + let code = match text.chars.iter().position(|&c| c == ch) { + Some(i) => (i + 1) as u32, + None => { + if text.chars.len() >= TEXT_KEYMAP_MAX { + text.chars.clear(); // restart the map; old codes are re-assigned lazily + } + text.chars.push(ch); + let keymap_str = text_keymap(&text.chars); + let file = memfd_with(&keymap_str)?; + text.keyboard.keymap( + 1, /* XKB_V1 */ + file.as_fd(), + keymap_str.len() as u32 + 1, + ); + text._keymap_file = Some(file); + text.chars.len() as u32 + } + }; + text.keyboard.key(t, code, 1); + text.keyboard.key(t, code, 0); + Ok(()) + } + /// Update xkb state for a key and tell the compositor the resulting modifier mask. fn send_modifiers(&mut self, evdev: u16, down: bool) { let kc = xkb::Keycode::new(evdev as u32 + 8); // evdev -> xkb keycode @@ -254,6 +320,9 @@ impl InputInjector for WlrootsInjector { tracing::debug!(vk = event.code, "unmapped VK keycode — dropped"); } } + InputKind::TextInput => { + self.type_text(event.code)?; + } InputKind::GamepadState | InputKind::GamepadButton | InputKind::GamepadAxis @@ -271,6 +340,33 @@ impl InputInjector for WlrootsInjector { } } +/// Build a minimal xkb keymap whose keycode `i + 9` (wire code `i + 1`) types `chars[i]`, using +/// Unicode keysym names (`U` — xkbcommon resolves them for any scalar, emoji included). +/// Types/compat `include "complete"` mirrors `wtype`'s generated keymap — proven on wlroots +/// compositors, and the system XKB data is present (the main keymap compiled from it in `open`). +fn text_keymap(chars: &[char]) -> String { + use std::fmt::Write as _; + let mut keycodes = String::new(); + let mut symbols = String::new(); + for (i, ch) in chars.iter().enumerate() { + let _ = writeln!(keycodes, " = {};", i + 9); + let _ = writeln!(symbols, " key {{ [ U{:04X} ] }};", *ch as u32); + } + format!( + "xkb_keymap {{\n\ + xkb_keycodes \"punktfunk-text\" {{\n\ + minimum = 8;\n\ + maximum = {};\n\ + {keycodes}\ + }};\n\ + xkb_types \"punktfunk-text\" {{ include \"complete\" }};\n\ + xkb_compatibility \"punktfunk-text\" {{ include \"complete\" }};\n\ + xkb_symbols \"punktfunk-text\" {{\n{symbols} }};\n\ + }};\n", + chars.len() + 9, + ) +} + /// Create an anonymous in-memory file holding `s` + a trailing NUL (for the keymap fd). fn memfd_with(s: &str) -> Result { let name = b"punktfunk-keymap\0"; diff --git a/crates/pf-inject/src/inject/windows/sendinput.rs b/crates/pf-inject/src/inject/windows/sendinput.rs index 64aad983..7aeea977 100644 --- a/crates/pf-inject/src/inject/windows/sendinput.rs +++ b/crates/pf-inject/src/inject/windows/sendinput.rs @@ -27,10 +27,10 @@ use windows::Win32::System::StationsAndDesktops::{ use windows::Win32::UI::Input::KeyboardAndMouse::{ GetKeyboardLayout, MapVirtualKeyExW, SendInput, HKL, INPUT, INPUT_0, INPUT_KEYBOARD, INPUT_MOUSE, KEYBDINPUT, KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP, KEYEVENTF_SCANCODE, - MAPVK_VK_TO_VSC_EX, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_HWHEEL, MOUSEEVENTF_LEFTDOWN, - MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE, - MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK, MOUSEEVENTF_WHEEL, - MOUSEEVENTF_XDOWN, MOUSEEVENTF_XUP, MOUSEINPUT, VIRTUAL_KEY, + KEYEVENTF_UNICODE, MAPVK_VK_TO_VSC_EX, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_HWHEEL, + MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, + MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK, + MOUSEEVENTF_WHEEL, MOUSEEVENTF_XDOWN, MOUSEEVENTF_XUP, MOUSEINPUT, VIRTUAL_KEY, }; use windows::Win32::UI::WindowsAndMessaging::{ GetForegroundWindow, GetSystemMetrics, GetWindowThreadProcessId, SM_CXVIRTUALSCREEN, @@ -297,6 +297,33 @@ impl InputInjector for SendInputInjector { }; self.send(&[key(ki)]) } + InputKind::TextInput => { + // Committed IME text: one Unicode scalar per event, injected as + // `KEYEVENTF_UNICODE` packets (wScan = UTF-16 unit, no scancode/layout involved + // — the receiving app gets the character verbatim via WM_CHAR). An astral-plane + // scalar (emoji) is its surrogate pair, each unit down+up in order — exactly how + // Windows expects supplementary characters from unicode injection. + let Some(ch) = char::from_u32(event.code) else { + return Ok(()); // lone surrogate / out of range — drop + }; + if ch.is_control() { + return Ok(()); // control chars ride the VK path (Enter/Backspace/Tab) + } + let mut units = [0u16; 2]; + let mut inputs: Vec = Vec::with_capacity(4); + for &unit in ch.encode_utf16(&mut units).iter() { + for flags in [KEYEVENTF_UNICODE, KEYEVENTF_UNICODE | KEYEVENTF_KEYUP] { + inputs.push(key(KEYBDINPUT { + wVk: VIRTUAL_KEY(0), + wScan: unit, + dwFlags: flags, + time: 0, + dwExtraInfo: 0, + })); + } + } + self.send(&inputs) + } // Gamepad goes through the XUSB backend. Touch: no SendInput equivalent -> no-op. InputKind::GamepadButton | InputKind::GamepadAxis diff --git a/crates/pf-inject/src/lib.rs b/crates/pf-inject/src/lib.rs index 6b96c030..62ba0e61 100644 --- a/crates/pf-inject/src/lib.rs +++ b/crates/pf-inject/src/lib.rs @@ -174,6 +174,29 @@ pub fn default_backend() -> Backend { Backend::Unsupported } +/// Whether the session's inject backend can type **committed text** +/// ([`InputKind::TextInput`] — see `HOST_CAP_TEXT_INPUT`): Windows always (`KEYEVENTF_UNICODE`); +/// Linux only on the wlroots backend (a dedicated virtual keyboard with a dynamically-grown +/// Unicode keymap) — KWin fake-input/libei/gamescope can only press keycodes of the host layout. +/// Consulted at Welcome time to advertise the cap; a mid-session backend switch away from a +/// capable one just degrades to dropped text events (input is lossy by design). +#[cfg(target_os = "windows")] +pub fn text_input_supported() -> bool { + true +} + +/// See the Windows variant: Linux types text only through the wlroots virtual-keyboard backend. +#[cfg(target_os = "linux")] +pub fn text_input_supported() -> bool { + matches!(default_backend(), Backend::WlrVirtual) +} + +/// No injector ⇒ no text. +#[cfg(not(any(target_os = "linux", target_os = "windows")))] +pub fn text_input_supported() -> bool { + false +} + #[path = "inject/service.rs"] mod service; pub use service::InjectorService; diff --git a/crates/punktfunk-core/src/input.rs b/crates/punktfunk-core/src/input.rs index 791c2161..49da2129 100644 --- a/crates/punktfunk-core/src/input.rs +++ b/crates/punktfunk-core/src/input.rs @@ -73,6 +73,17 @@ pub enum InputKind { /// [`HOST_CAP_GAMEPAD_STATE`](crate::quic::HOST_CAP_GAMEPAD_STATE); an older host ignores the /// unknown tag (every pad then uses the session-default kind — the pre-existing behaviour). GamepadArrival = 14, + /// One Unicode scalar of **committed text** — `code` = the scalar value, everything else 0. + /// + /// The IME path: the layout-independent VK key events cannot express text an input method + /// *commits* (autocorrect, gesture typing, non-Latin scripts, emoji), so a capable client + /// sends the committed characters verbatim and the host injects them directly (Windows + /// `KEYEVENTF_UNICODE`; Linux wlroots via a dynamically-grown Unicode keymap on a dedicated + /// virtual keyboard). A multi-character commit is consecutive events in order. Sent only when + /// the host advertised [`HOST_CAP_TEXT_INPUT`](crate::quic::HOST_CAP_TEXT_INPUT) — toward an + /// older host (or one whose inject backend can't type text) clients keep the best-effort VK + /// synthesis, and an older host ignores the unknown tag entirely. + TextInput = 15, } /// Pack a [`InputKind::GamepadRemove`] `flags` word (`seq << 24 | pad`) — the same low-byte-pad / @@ -158,6 +169,7 @@ impl InputKind { 12 => GamepadState, 13 => GamepadRemove, 14 => GamepadArrival, + 15 => TextInput, _ => return None, }) } @@ -392,10 +404,27 @@ mod tests { }; assert_eq!(InputEvent::decode(&e.encode()), Some(e)); } - // GamepadRemove + GamepadArrival are valid kinds; 15 (one past them) is not. + // GamepadRemove/GamepadArrival/TextInput are valid kinds; 16 (one past them) is not. assert_eq!(InputKind::from_u8(13), Some(InputKind::GamepadRemove)); assert_eq!(InputKind::from_u8(14), Some(InputKind::GamepadArrival)); - assert_eq!(InputKind::from_u8(15), None); + assert_eq!(InputKind::from_u8(15), Some(InputKind::TextInput)); + assert_eq!(InputKind::from_u8(16), None); + } + + #[test] + fn text_input_roundtrip() { + // One Unicode scalar per event — BMP and astral (emoji) alike. + for cp in ['a' as u32, 'ß' as u32, '語' as u32, 0x1F600 /* 😀 */] { + let e = InputEvent { + kind: InputKind::TextInput, + _pad: [0; 3], + code: cp, + x: 0, + y: 0, + flags: 0, + }; + assert_eq!(InputEvent::decode(&e.encode()), Some(e)); + } } #[test] diff --git a/crates/punktfunk-core/src/quic/caps.rs b/crates/punktfunk-core/src/quic/caps.rs index 034ba335..843a37a9 100644 --- a/crates/punktfunk-core/src/quic/caps.rs +++ b/crates/punktfunk-core/src/quic/caps.rs @@ -71,6 +71,16 @@ pub const HOST_CAP_GAMEPAD_STATE: u8 = 0x01; /// trailing `host_caps` byte — no wire-layout change. pub const HOST_CAP_CLIPBOARD: u8 = 0x02; +/// [`Welcome::host_caps`] bit: the host's active inject backend can type **committed text** +/// ([`InputKind::TextInput`](crate::input::InputKind::TextInput) — one Unicode scalar per event): +/// Windows (`KEYEVENTF_UNICODE`) and Linux wlroots (dynamic Unicode keymap on a dedicated virtual +/// keyboard); the KWin/libei/gamescope backends can only press layout keycodes, so those sessions +/// don't set it. A capable client routes its IME's committed text (autocorrect, gesture typing, +/// non-Latin scripts, emoji) through `TextInput` instead of lossy VK synthesis; absent the bit it +/// keeps the VK fallback. Packs into the existing trailing `host_caps` byte — no wire-layout +/// change; an older host ignores the unknown input tag anyway (input is lossy by design). +pub const HOST_CAP_TEXT_INPUT: u8 = 0x04; + /// [`Hello::video_codecs`] bit: the client can decode H.264 / AVC. The GPU-less **software** /// encode path (openh264) emits H.264, so a client that wants to stream from a software host MUST /// advertise this. diff --git a/crates/punktfunk-host/src/gamestream/input.rs b/crates/punktfunk-host/src/gamestream/input.rs index 77e84748..59924b26 100644 --- a/crates/punktfunk-host/src/gamestream/input.rs +++ b/crates/punktfunk-host/src/gamestream/input.rs @@ -35,7 +35,24 @@ pub fn decode(plaintext: &[u8]) -> Vec { if plaintext.len() < 4 || u16::from_le_bytes([plaintext[0], plaintext[1]]) != INPUT_DATA_TYPE { return Vec::new(); } - decode_input_packet(&plaintext[4..]).into_iter().collect() + let p = &plaintext[4..]; + // UTF-8 text (Moonlight's client-side keyboard commit) expands to one `TextInput` event per + // Unicode scalar — the only magic yielding more than one event, so it's handled before the + // single-event dispatch. Injected the same way as the native plane's IME text. + if p.len() >= 8 && u32::from_le_bytes([p[4], p[5], p[6], p[7]]) == MAGIC_UTF8 { + // NV_INPUT_HEADER.size (BE, excludes itself) counts magic + body. + let size = u32::from_be_bytes([p[0], p[1], p[2], p[3]]) as usize; + let body_len = size.saturating_sub(4).min(p.len() - 8); + return match std::str::from_utf8(&p[8..8 + body_len]) { + Ok(s) => s + .chars() + .filter(|c| !c.is_control()) + .map(|c| ev(InputKind::TextInput, c as u32, 0, 0, 0)) + .collect(), + Err(_) => Vec::new(), + }; + } + decode_input_packet(p).into_iter().collect() } fn decode_input_packet(p: &[u8]) -> Option { @@ -89,7 +106,7 @@ fn decode_input_packet(p: &[u8]) -> Option { modifiers | crate::inject::KEY_FLAG_SEMANTIC_VK, ) } - // UTF-8 text, gamepad, pen, touch, haptics — not yet injected. + // Gamepad, pen, touch, haptics — not yet injected. (UTF-8 text is handled in `decode`.) _ => return None, }) } @@ -144,6 +161,21 @@ mod tests { assert_eq!(ev[0].flags, 0x04 | crate::inject::KEY_FLAG_SEMANTIC_VK); } + #[test] + fn decodes_utf8_text_per_scalar() { + // "aß😀" — ASCII, Latin-1, and an astral scalar; one TextInput event per scalar. + let pt = wrap(MAGIC_UTF8, "aß😀".as_bytes()); + let ev = decode(&pt); + assert_eq!(ev.len(), 3); + assert!(ev.iter().all(|e| e.kind == InputKind::TextInput)); + assert_eq!(ev[0].code, 'a' as u32); + assert_eq!(ev[1].code, 'ß' as u32); + assert_eq!(ev[2].code, 0x1F600); + // Truncated / invalid UTF-8 decodes to nothing rather than mojibake. + let bad = wrap(MAGIC_UTF8, &[0xff, 0xfe]); + assert!(decode(&bad).is_empty()); + } + #[test] fn ignores_non_input_type() { let mut pt = vec![0x00, 0x02]; // type 0x0200 (keepalive) diff --git a/crates/punktfunk-host/src/native/handshake.rs b/crates/punktfunk-host/src/native/handshake.rs index f9d41128..4328e818 100644 --- a/crates/punktfunk-host/src/native/handshake.rs +++ b/crates/punktfunk-host/src/native/handshake.rs @@ -444,6 +444,15 @@ pub(super) async fn negotiate( punktfunk_core::quic::HOST_CAP_CLIPBOARD } else { 0 + } + // Committed-text injection (InputKind::TextInput): only where the session's inject + // backend can actually type text — Windows SendInput (KEYEVENTF_UNICODE) and the + // Linux wlroots virtual keyboard (dynamic Unicode keymap). Clients without the bit + // keep their VK-synthesis fallback for IME text. + | if crate::inject::text_input_supported() { + punktfunk_core::quic::HOST_CAP_TEXT_INPUT + } else { + 0 }, // The negotiated session AEAD (resolved above) + its 32-byte key toward a ChaCha // client; toward everyone else cipher 0 keeps the Welcome byte-identical to the diff --git a/include/punktfunk_core.h b/include/punktfunk_core.h index 5d98f390..76546eac 100644 --- a/include/punktfunk_core.h +++ b/include/punktfunk_core.h @@ -498,6 +498,18 @@ #define HOST_CAP_CLIPBOARD 2 #endif +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`Welcome::host_caps`] bit: the host's active inject backend can type **committed text** +// ([`InputKind::TextInput`](crate::input::InputKind::TextInput) — one Unicode scalar per event): +// Windows (`KEYEVENTF_UNICODE`) and Linux wlroots (dynamic Unicode keymap on a dedicated virtual +// keyboard); the KWin/libei/gamescope backends can only press layout keycodes, so those sessions +// don't set it. A capable client routes its IME's committed text (autocorrect, gesture typing, +// non-Latin scripts, emoji) through `TextInput` instead of lossy VK synthesis; absent the bit it +// keeps the VK fallback. Packs into the existing trailing `host_caps` byte — no wire-layout +// change; an older host ignores the unknown input tag anyway (input is lossy by design). +#define HOST_CAP_TEXT_INPUT 4 +#endif + #if defined(PUNKTFUNK_FEATURE_QUIC) // [`Hello::video_codecs`] bit: the client can decode H.264 / AVC. The GPU-less **software** // encode path (openh264) emits H.264, so a client that wants to stream from a software host MUST @@ -1094,6 +1106,17 @@ enum PunktfunkInputKind // [`HOST_CAP_GAMEPAD_STATE`](crate::quic::HOST_CAP_GAMEPAD_STATE); an older host ignores the // unknown tag (every pad then uses the session-default kind — the pre-existing behaviour). PUNKTFUNK_INPUT_KIND_GAMEPAD_ARRIVAL = 14, + // One Unicode scalar of **committed text** — `code` = the scalar value, everything else 0. + // + // The IME path: the layout-independent VK key events cannot express text an input method + // *commits* (autocorrect, gesture typing, non-Latin scripts, emoji), so a capable client + // sends the committed characters verbatim and the host injects them directly (Windows + // `KEYEVENTF_UNICODE`; Linux wlroots via a dynamically-grown Unicode keymap on a dedicated + // virtual keyboard). A multi-character commit is consecutive events in order. Sent only when + // the host advertised [`HOST_CAP_TEXT_INPUT`](crate::quic::HOST_CAP_TEXT_INPUT) — toward an + // older host (or one whose inject backend can't type text) clients keep the best-effort VK + // synthesis, and an older host ignores the unknown tag entirely. + PUNKTFUNK_INPUT_KIND_TEXT_INPUT = 15, }; #ifndef __cplusplus #if __STDC_VERSION__ >= 202311L