From 6db91cbf4079d14f6171c299ac24c615a0cbfa7f Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Tue, 14 Jul 2026 14:21:20 +0200 Subject: [PATCH] feat(client): 3-finger swipe toggles the on-screen keyboard mid-stream iOS + Android: a three-finger vertical swipe up/down summons/dismisses the device soft keyboard while streaming (trackpad + pointer modes). Mobile scroll is now exactly two fingers so it never collides with the 3+-finger gesture (3+ only fell into the old `>= 2` scroll path by accident). Android: a TYPE_NULL KeyCaptureView plus IME meta-shift wrapping feeds key events through. iOS: UIKeyInput plus a SoftKeyMap char->VK table with a GCKeyboard dup gate so a hardware keyboard and the soft keyboard don't double-emit. Co-Authored-By: Claude Fable 5 --- .../kotlin/io/unom/punktfunk/MainActivity.kt | 12 ++++ .../kotlin/io/unom/punktfunk/StreamScreen.kt | 64 ++++++++++++++++++- .../kotlin/io/unom/punktfunk/TouchInput.kt | 52 ++++++++++++++- .../Sources/PunktfunkKit/Input/KeyMaps.swift | 41 ++++++++++++ .../PunktfunkKit/Input/TouchMouse.swift | 61 ++++++++++++++++-- .../PunktfunkKit/Views/StreamViewIOS.swift | 59 +++++++++++++++++ crates/pf-presenter/src/touch.rs | 4 +- 7 files changed, 283 insertions(+), 10 deletions(-) diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt index 70b8b63a..7b25a8fc 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt @@ -3,6 +3,7 @@ package io.unom.punktfunk import android.os.Build import android.os.Bundle import android.view.InputDevice +import android.view.KeyCharacterMap import android.view.KeyEvent import android.view.MotionEvent import androidx.activity.ComponentActivity @@ -153,7 +154,18 @@ class MainActivity : ComponentActivity() { // physical-keyboard layout), keycode fallback — see Keymap docs. val vk = Keymap.toVk(event) if (vk != 0) { + // Soft-keyboard events (the IME's virtual device — the stream's + // KeyCaptureView path) carry Shift only as META state, where a real + // keyboard sends discrete Shift transitions — so mirror the meta bit as + // a VK_LSHIFT wrap or every IME capital/symbol lands unshifted on the + // host. Never applied to hardware events: their Shift already went over + // the wire, and a synthetic release here would un-hold a physical Shift + // the user is still pressing. + val imeShift = event.deviceId == KeyCharacterMap.VIRTUAL_KEYBOARD && + event.isShiftPressed && vk != 0xA0 && vk != 0xA1 + if (down && imeShift) NativeBridge.nativeSendKey(handle, 0xA0, true, 0) NativeBridge.nativeSendKey(handle, vk, down, 0) + if (!down && imeShift) NativeBridge.nativeSendKey(handle, 0xA0, false, 0) return true // consumed — don't let the system also act on it } } diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt index 3fbcf4a7..7e34de9f 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt @@ -6,15 +6,22 @@ import android.content.pm.ActivityInfo import android.content.pm.PackageManager import android.net.wifi.WifiManager import android.os.Build +import android.text.InputType import android.util.Log import android.view.SurfaceHolder import android.view.SurfaceView +import android.view.View import android.view.WindowManager +import android.view.inputmethod.BaseInputConnection +import android.view.inputmethod.EditorInfo +import android.view.inputmethod.InputConnection +import android.view.inputmethod.InputMethodManager import android.widget.Toast import androidx.activity.compose.BackHandler import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect @@ -166,6 +173,12 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { it.systemBarsBehavior = WindowInsetsControllerCompat.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE it.hide(WindowInsetsCompat.Type.systemBars()) } + // The soft keyboard (three-finger swipe up → KeyCaptureView below) must OVERLAY the + // stream, never pan/resize it — the video is a fixed-mode surface, not a document. + // Scoped to the stream; the app's other screens keep the default for their text fields. + val priorSoftInput = window?.attributes?.softInputMode + ?: WindowManager.LayoutParams.SOFT_INPUT_ADJUST_UNSPECIFIED + window?.setSoftInputMode(WindowManager.LayoutParams.SOFT_INPUT_ADJUST_NOTHING) // Lock to landscape while streaming — the host streams a landscape desktop, so pin the device // there (either landscape direction is fine) and stop it rotating to portrait mid-session. The // activity declares configChanges=orientation, so this re-lays out the surface in place without @@ -201,6 +214,8 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { activity?.streamHandle = 0L activity?.requestStreamExit = null activity?.setConsoleHighRefreshRate(true) // back to the console UI's max refresh + controller?.hide(WindowInsetsCompat.Type.ime()) // drop any keyboard left showing + window?.setSoftInputMode(priorSoftInput) controller?.show(WindowInsetsCompat.Type.systemBars()) window?.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) if (lowLatencyMode && Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { @@ -221,6 +236,9 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { // Back gesture = a deliberate exit → signal the quit so the host tears down now (no linger). BackHandler { NativeBridge.nativeDisconnectQuit(handle); onDisconnect() } + // Focus anchor the three-finger keyboard swipe summons the IME onto (see KeyCaptureView). + var keyCapture by remember { mutableStateOf(null) } + Box(modifier = Modifier.fillMaxSize()) { AndroidView( modifier = Modifier.fillMaxSize(), @@ -271,8 +289,16 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { StatsOverlay(it, statsVerbosity, decoderLabel, Modifier.align(Alignment.TopStart).padding(12.dp)) } } + // Invisible 1-px focus anchor for the host-typing soft keyboard (three-finger swipe + // up in the mouse modes) — it never draws or takes touches, it just owns IME focus. + AndroidView( + modifier = Modifier.size(1.dp), + factory = { ctx -> KeyCaptureView(ctx).also { keyCapture = it } }, + ) // Touch input per the Settings model: trackpad/direct-pointer mouse (the shared gesture - // vocabulary) or real multi-touch passthrough — see TouchInput.kt. + // vocabulary) or real multi-touch passthrough — see TouchInput.kt. Passthrough gets no + // keyboard gesture: its fingers belong to the host verbatim (a swipe there may BE a + // host-OS gesture), so intercepting three fingers would corrupt real multi-touch. Box( Modifier.fillMaxSize().pointerInput(handle, touchMode) { when (touchMode) { @@ -281,9 +307,45 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { handle, trackpad = touchMode == TouchMode.TRACKPAD, onCycleStats = { statsVerbosity = statsVerbosity.next() }, + onKeyboard = { show -> keyCapture?.setImeVisible(show) }, ) } }, ) } } + +/** + * Invisible focus anchor for typing on the host: the three-finger swipe summons the device IME + * onto this view. `TYPE_NULL` puts the IME in "dumb keyboard" mode — it delivers raw [KeyEvent]s + * (no composing text, no autocorrect), which flow through `MainActivity.dispatchKeyEvent` → + * `Keymap.toVk` → the host, the exact path a hardware keyboard takes. Text an IME insists on + * committing instead still arrives: the non-editable [BaseInputConnection] synthesizes KeyEvents + * for it via `KeyCharacterMap` (with Shift carried as meta state — see the IME-shift wrap in + * `MainActivity.dispatchKeyEvent`). + */ +private class KeyCaptureView(context: Context) : View(context) { + init { + isFocusable = true + isFocusableInTouchMode = true + } + + override fun onCheckIsTextEditor(): Boolean = true + + override fun onCreateInputConnection(outAttrs: EditorInfo): InputConnection { + outAttrs.inputType = InputType.TYPE_NULL + outAttrs.imeOptions = EditorInfo.IME_FLAG_NO_EXTRACT_UI or EditorInfo.IME_FLAG_NO_FULLSCREEN + return BaseInputConnection(this, false) + } + + fun setImeVisible(show: Boolean) { + val imm = context.getSystemService(Context.INPUT_METHOD_SERVICE) as? InputMethodManager + ?: return + if (show) { + requestFocus() + imm.showSoftInput(this, 0) + } else { + imm.hideSoftInputFromWindow(windowToken, 0) + } + } +} diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt index 81348165..16caa791 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt @@ -19,6 +19,10 @@ private const val TAP_SLOP = 12f private const val TAP_DRAG_MS = 250L private const val SCROLL_DIV = 4f +// Three-finger vertical swipe: the fraction of the view height the centroid must travel to +// summon (up) / dismiss (down) the local soft keyboard. +private const val KB_SWIPE_FRACTION = 0.10f + // Trackpad-mode pointer ballistics (relative one-finger motion). POINTER_SENS: base finger-px → // host-px gain (~1:1, never twitchy). The rest is mild acceleration so a flick crosses the screen // while a slow drag stays precise: above ACCEL_SPEED_FLOOR px/ms the gain ramps by ACCEL_GAIN per @@ -40,7 +44,9 @@ private const val ACCEL_MAX = 3.0f * * Both share the same gesture vocabulary: tap = left click; two-finger tap = right click; * two-finger drag = scroll; tap-then-press-and-drag = left-drag (text selection / moving - * windows); three-finger tap = [onCycleStats] (cycle the stats-HUD verbosity tier). + * windows); three-finger tap = [onCycleStats] (cycle the stats-HUD verbosity tier); + * three-finger swipe up/down = [onKeyboard] (summon/dismiss the local soft keyboard, for + * typing on the host). */ /** * Real multi-touch passthrough ([TouchMode.TOUCH]): every finger forwards as a host touchscreen @@ -94,6 +100,7 @@ internal suspend fun PointerInputScope.streamTouchInput( handle: Long, trackpad: Boolean, onCycleStats: () -> Unit, + onKeyboard: (show: Boolean) -> Unit, ) { var lastTapUp = 0L var lastTapX = 0f @@ -128,6 +135,12 @@ internal suspend fun PointerInputScope.streamTouchInput( var maxFingers = 1 var scrolling = false var scrollCount = 0 // pointer count the scroll centroid is anchored at + // Keyboard-swipe state: the 3+-finger centroid anchor (per finger count, like the + // scroll anchor) and a once-per-gesture latch. + var kbCount = 0 + var kbAnchorX = 0f + var kbAnchorY = 0f + var kbFired = false var prevCx = startX var prevCy = startY var upTime = down.uptimeMillis @@ -148,9 +161,12 @@ internal suspend fun PointerInputScope.streamTouchInput( break } if (pressed.size > maxFingers) maxFingers = pressed.size + // Dropping below three fingers forgets the keyboard-swipe anchor, so a 3→2→3 + // bounce re-anchors instead of reading the count change as swipe travel. + if (pressed.size < 3) kbCount = 0 - if (pressed.size >= 2) { - // Two+ fingers → scroll by the centroid delta; never move the cursor. + if (pressed.size == 2) { + // Two fingers → scroll by the centroid delta; never move the cursor. val cx = (pressed.sumOf { it.position.x.toDouble() } / pressed.size).toFloat() val cy = (pressed.sumOf { it.position.y.toDouble() } / pressed.size).toFloat() // (Re-)anchor whenever the finger COUNT changes, not just on scroll start: the @@ -177,6 +193,36 @@ internal suspend fun PointerInputScope.streamTouchInput( prevCx = cx moved = true } + } else if (pressed.size >= 3) { + // Three+ fingers → the keyboard swipe, never scroll (the documented + // vocabulary is TWO-finger scroll; 3+ only fell into the scroll path as an + // accident of its old `>= 2` bound). Anchor the centroid per finger count + // (same reasoning as the scroll anchor above) and fire once per gesture when + // the vertical travel crosses the threshold: up = show, down = hide. + val cx = (pressed.sumOf { it.position.x.toDouble() } / pressed.size).toFloat() + val cy = (pressed.sumOf { it.position.y.toDouble() } / pressed.size).toFloat() + if (pressed.size != kbCount) { + kbCount = pressed.size + kbAnchorX = cx + kbAnchorY = cy + } else { + val dy = cy - kbAnchorY + // Real centroid travel disqualifies the tap classification below (else a + // sub-threshold swipe would still fire the three-finger stats tap). + if (abs(dy) > TAP_SLOP || abs(cx - kbAnchorX) > TAP_SLOP) moved = true + if (!kbFired && abs(dy) >= size.height * KB_SWIPE_FRACTION) { + kbFired = true + onKeyboard(dy < 0) // finger up → show, finger down → hide + } + } + // Leaving the scroll state stale would read the 3→2 centroid jump as a wheel + // notch; clearing it makes a return to two fingers re-anchor fresh. Same for + // the trackpad's tracked finger: its prev position froze while 3+ fingers were + // down, so dropping straight back to one finger must re-anchor (zero delta), + // not replay the whole 3-finger phase as one cursor jump. + scrolling = false + scrollCount = 0 + trackId = PointerId(Long.MIN_VALUE) } else if (!scrolling) { // One finger (skipped once a gesture turned into a scroll, so dropping // back to one finger doesn't jerk the cursor). diff --git a/clients/apple/Sources/PunktfunkKit/Input/KeyMaps.swift b/clients/apple/Sources/PunktfunkKit/Input/KeyMaps.swift index 5103bfeb..3665135c 100644 --- a/clients/apple/Sources/PunktfunkKit/Input/KeyMaps.swift +++ b/clients/apple/Sources/PunktfunkKit/Input/KeyMaps.swift @@ -118,3 +118,44 @@ extension InputCapture { ] #endif } + +#if os(iOS) +/// US-layout character → Windows VK for the on-screen keyboard (`StreamLayerUIView`'s +/// UIKeyInput). Unlike every other key source, `insertText` delivers CHARACTERS, not key +/// positions, so this is the inverse of a US layout: `shift` means "wrap in VK_LSHIFT so the +/// host types the shifted symbol". Same contract as `hidToVK`: emit only VKs the host's +/// vk_to_evdev knows; anything unmapped is dropped by the caller. +enum SoftKeyMap { + static func vk(for ch: Character) -> (vk: UInt32, shift: Bool)? { + guard let ascii = ch.asciiValue else { return nil } + switch ascii { + case UInt8(ascii: "a")...UInt8(ascii: "z"): return (UInt32(ascii) - 0x20, false) + case UInt8(ascii: "A")...UInt8(ascii: "Z"): return (UInt32(ascii), true) + case UInt8(ascii: "0")...UInt8(ascii: "9"): return (UInt32(ascii), false) + case 0x0A, 0x0D: return (0x0D, false) // return + case 0x09: return (0x09, false) // tab + case 0x20: return (0x20, false) // space + default: return symbols[ch] + } + } + + /// US punctuation, plain and shifted, on the OEM VKs (mirrors `hidToVK`'s OEM block) plus + /// the shifted digit row. + private static let symbols: [Character: (vk: UInt32, shift: Bool)] = [ + "-": (0xBD, false), "_": (0xBD, true), + "=": (0xBB, false), "+": (0xBB, true), + "[": (0xDB, false), "{": (0xDB, true), + "]": (0xDD, false), "}": (0xDD, true), + "\\": (0xDC, false), "|": (0xDC, true), + ";": (0xBA, false), ":": (0xBA, true), + "'": (0xDE, false), "\"": (0xDE, true), + "`": (0xC0, false), "~": (0xC0, true), + ",": (0xBC, false), "<": (0xBC, true), + ".": (0xBE, false), ">": (0xBE, true), + "/": (0xBF, false), "?": (0xBF, true), + "!": (0x31, true), "@": (0x32, true), "#": (0x33, true), "$": (0x34, true), + "%": (0x35, true), "^": (0x36, true), "&": (0x37, true), "*": (0x38, true), + "(": (0x39, true), ")": (0x30, true), + ] +} +#endif diff --git a/clients/apple/Sources/PunktfunkKit/Input/TouchMouse.swift b/clients/apple/Sources/PunktfunkKit/Input/TouchMouse.swift index d6489853..7367a8ed 100644 --- a/clients/apple/Sources/PunktfunkKit/Input/TouchMouse.swift +++ b/clients/apple/Sources/PunktfunkKit/Input/TouchMouse.swift @@ -3,7 +3,8 @@ // identical. Two mouse modes share one gesture vocabulary — tap = left click · two-finger // tap = right click · two-finger drag = scroll · tap-then-press-and-drag = held left drag // (text selection / window moves) · three-finger tap = cycles the stats overlay tiers -// (off → compact → normal → detailed, matching Android): +// (off → compact → normal → detailed, matching Android) · three-finger swipe up/down = +// summon/dismiss the local soft keyboard for typing on the host (`onKeyboardGesture`): // // * trackpad (default): the cursor STAYS PUT on touch-down and moves by the finger's // relative delta with mild acceleration — swipe to nudge, lift and re-swipe to walk it @@ -61,6 +62,9 @@ final class TouchMouse { static let accelGain: CGFloat = 0.6 static let accelSpeedFloor: CGFloat = 0.3 static let accelMax: CGFloat = 3.0 + /// Three-finger vertical swipe: the fraction of the view height the centroid must + /// travel to summon (up) / dismiss (down) the local soft keyboard. + static let keyboardSwipeFraction: CGFloat = 0.10 /// Acceleration multiplier for a finger speed in physical px per ms. static func accel(forSpeed speed: CGFloat) -> CGFloat { @@ -72,6 +76,9 @@ final class TouchMouse { var send: ((PunktfunkInputEvent) -> Void)? /// View-space point → host-mode pixels through the letterbox (pointer mode's moves). var hostPoint: ((CGPoint) -> StreamLayerUIView.HostPoint?)? + /// Three-finger vertical swipe crossed the threshold: `true` = show the local soft + /// keyboard (swipe up), `false` = dismiss it (swipe down). Fires at most once per gesture. + var onKeyboardGesture: ((Bool) -> Void)? /// No gesture in flight (all fingers up) — the view uses this to release its mode latch. var isIdle: Bool { !sessionActive && lastPos.isEmpty } @@ -95,6 +102,11 @@ final class TouchMouse { private var carryY: CGFloat = 0 /// Scroll anchor (centroid) — re-anchored every time a notch fires. private var scrollAnchor = CGPoint.zero + // Keyboard-swipe state: the 3+-finger centroid anchor (per finger count, like the scroll + // anchor) and a once-per-gesture latch. + private var kbCount = 0 + private var kbAnchor = CGPoint.zero + private var kbFired = false // Tap-drag arming: a quick tap leaves a window in which the next nearby touch drags. private var lastTapUp: TimeInterval = 0 private var lastTapPoint = CGPoint.zero @@ -114,6 +126,8 @@ final class TouchMouse { maxFingers = 0 moved = false scrolling = false + kbCount = 0 + kbFired = false // A touch landing just after a quick tap nearby = tap-and-drag: hold the left // button for this whole gesture (laptop-trackpad convention). dragHeld = first.timestamp - lastTapUp < Tuning.tapDragWindow @@ -140,8 +154,13 @@ final class TouchMouse { for touch in touches where lastPos[ObjectIdentifier(touch)] != nil { lastPos[ObjectIdentifier(touch)] = touch.location(in: view) } - if lastPos.count >= 2 { + // Dropping below three fingers forgets the keyboard-swipe anchor, so a 3→2→3 bounce + // re-anchors instead of reading the count change as swipe travel. + if lastPos.count < 3 { kbCount = 0 } + if lastPos.count == 2 { scrollByCentroid() + } else if lastPos.count >= 3 { + keyboardSwipe(in: view) } else if !scrolling, let touch = touches.first(where: { lastPos[ObjectIdentifier($0)] != nil }) { @@ -208,9 +227,9 @@ final class TouchMouse { // MARK: - Per-event work - /// Two fingers (or more) → scroll by the centroid delta; never move the cursor. Fires a - /// notch per `scrollNotchPt` of pan and re-anchors on fire; finger up scrolls up, finger - /// right scrolls right (the host WHEEL(120) convention). + /// Two fingers → scroll by the centroid delta; never move the cursor. Fires a notch per + /// `scrollNotchPt` of pan and re-anchors on fire; finger up scrolls up, finger right + /// scrolls right (the host WHEEL(120) convention). private func scrollByCentroid() { let n = CGFloat(lastPos.count) let cx = lastPos.values.reduce(0) { $0 + $1.x } / n @@ -233,6 +252,38 @@ final class TouchMouse { } } + /// Three+ fingers → the keyboard swipe, never scroll (the documented vocabulary is + /// TWO-finger scroll; 3+ only fell into the scroll path as an accident of its old `>= 2` + /// bound). The centroid is anchored per finger count — real fingers never land or lift in + /// the same event, so a count change must re-anchor rather than read as travel — and the + /// gesture fires at most once, when the vertical travel crosses the threshold: up = show + /// the local soft keyboard, down = dismiss it. + private func keyboardSwipe(in view: UIView) { + let n = CGFloat(lastPos.count) + let cx = lastPos.values.reduce(0) { $0 + $1.x } / n + let cy = lastPos.values.reduce(0) { $0 + $1.y } / n + if lastPos.count != kbCount { + kbCount = lastPos.count + kbAnchor = CGPoint(x: cx, y: cy) + } else { + let dy = cy - kbAnchor.y + // Real centroid travel disqualifies the tap classification in `ended` (else a + // sub-threshold swipe would still fire the three-finger stats tap). + if abs(dy) > Tuning.tapSlop || abs(cx - kbAnchor.x) > Tuning.tapSlop { moved = true } + if !kbFired, abs(dy) >= view.bounds.height * Tuning.keyboardSwipeFraction { + kbFired = true + onKeyboardGesture?(dy < 0) // finger up → show, finger down → dismiss + } + } + // Leaving the scroll state stale would read the 3→2 centroid jump as a wheel notch; + // clearing it makes a return to two fingers re-anchor fresh. Same for the trackpad's + // tracked finger: its prev position froze while 3+ fingers were down, so dropping + // straight back to one finger must re-anchor (zero delta), not replay the whole + // 3-finger phase as one cursor jump. + scrolling = false + trackKey = nil + } + /// One finger (and the gesture never became a scroll — dropping back from two fingers to /// one must not jerk the cursor). private func singleFinger(_ touch: UITouch, in view: UIView) { diff --git a/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift b/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift index 36882ca0..5273fe37 100644 --- a/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift +++ b/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift @@ -698,6 +698,7 @@ final class StreamLayerUIView: UIView { let mouse = TouchMouse() mouse.send = { [weak self] event in self?.onTouchEvent?(event) } mouse.hostPoint = { [weak self] point in self?.hostPoint(from: point) } + mouse.onKeyboardGesture = { [weak self] show in self?.setSoftKeyboardVisible(show) } return mouse }() /// The finger route latched at gesture start — a Settings change mid-gesture applies to @@ -708,6 +709,22 @@ final class StreamLayerUIView: UIView { func resetTouchInput() { touchMouse.reset() fingerRoute = nil + setSoftKeyboardVisible(false) // a stream that's gone takes its keyboard with it + } + + /// The soft keyboard is keyed off first-responder status: the three-finger swipe + /// (TouchMouse) summons/dismisses it here, and the UIKeyInput conformance below turns + /// what it types into wire key events. Also the reason `canBecomeFirstResponder` is true + /// on iOS (tvOS anchors the responder chain on the CONTROLLER instead — see + /// StreamViewController.viewDidAppear). + override var canBecomeFirstResponder: Bool { true } + + func setSoftKeyboardVisible(_ visible: Bool) { + if visible { + becomeFirstResponder() + } else if isFirstResponder { + resignFirstResponder() + } } #endif @@ -879,4 +896,46 @@ final class StreamLayerUIView: UIView { } #endif } + +#if os(iOS) +// The soft keyboard's output → wire key events. UIKeyInput is deliberately minimal (no +// UITextInput): the stream needs keystrokes, not an editing buffer — insertions map through +// `SoftKeyMap` to US-positional VKs (with a VK_LSHIFT wrap for shifted characters) and +// characters outside the map (emoji, non-Latin scripts) are dropped, matching the wire's VK +// contract. Events ride the same `onTouchEvent` path as the touch-driven mouse, so they're +// gated on captureEnabled with everything else and can't leak past a trust prompt. +extension StreamLayerUIView: UIKeyInput { + // Keep the IME literal — no autocorrect/smart substitutions; a remote desktop is not prose, + // and the host does its own text handling. + var autocorrectionType: UITextAutocorrectionType { get { .no } set {} } + var autocapitalizationType: UITextAutocapitalizationType { get { .none } set {} } + var spellCheckingType: UITextSpellCheckingType { get { .no } set {} } + var smartQuotesType: UITextSmartQuotesType { get { .no } set {} } + var smartDashesType: UITextSmartDashesType { get { .no } set {} } + var smartInsertDeleteType: UITextSmartInsertDeleteType { get { .no } set {} } + var keyboardType: UIKeyboardType { get { .asciiCapable } set {} } + + var hasText: Bool { false } + + func insertText(_ text: String) { + // A hardware keyboard's presses reach the host through GCKeyboard AND arrive here as + // UIKeyInput insertions while we're first responder — forwarding both would double + // every character, so the HID path owns keys whenever a hardware keyboard is attached. + guard GCKeyboard.coalesced == nil else { return } + for ch in text { + guard let key = SoftKeyMap.vk(for: ch) else { continue } + if key.shift { onTouchEvent?(.key(0xA0, down: true)) } // VK_LSHIFT + onTouchEvent?(.key(key.vk, down: true)) + onTouchEvent?(.key(key.vk, down: false)) + if key.shift { onTouchEvent?(.key(0xA0, down: false)) } + } + } + + func deleteBackward() { + guard GCKeyboard.coalesced == nil else { return } // see insertText + onTouchEvent?(.key(0x08, down: true)) // VK_BACK + onTouchEvent?(.key(0x08, down: false)) + } +} +#endif #endif diff --git a/crates/pf-presenter/src/touch.rs b/crates/pf-presenter/src/touch.rs index d9dd978a..55a4d36f 100644 --- a/crates/pf-presenter/src/touch.rs +++ b/crates/pf-presenter/src/touch.rs @@ -13,7 +13,9 @@ //! //! Shared gestures: tap = left click · two-finger tap = right click · two-finger drag = //! scroll · tap-then-press-and-drag = held left drag · three-finger tap = cycle the stats -//! overlay tier. +//! overlay tier. (The Android/Apple twins additionally map a three-finger vertical SWIPE to +//! their local soft keyboard and gate scroll to exactly two fingers for it; SDL builds have +//! no soft keyboard to summon, so here 2+ fingers scroll.) //! //! Unlike the Android/Apple hosts (which hand the engine a whole event's worth of changed //! touches at once), SDL delivers ONE finger transition per event, so this is a strictly