feat(client): 3-finger swipe toggles the on-screen keyboard mid-stream

iOS + Android: a three-finger vertical swipe up/down summons/dismisses the
device soft keyboard while streaming (trackpad + pointer modes). Mobile scroll
is now exactly two fingers so it never collides with the 3+-finger gesture
(3+ only fell into the old `>= 2` scroll path by accident).

Android: a TYPE_NULL KeyCaptureView plus IME meta-shift wrapping feeds key
events through. iOS: UIKeyInput plus a SoftKeyMap char->VK table with a
GCKeyboard dup gate so a hardware keyboard and the soft keyboard don't
double-emit.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-14 14:21:20 +02:00
parent 60d4653083
commit 6db91cbf40
7 changed files with 283 additions and 10 deletions
@@ -118,3 +118,44 @@ extension InputCapture {
]
#endif
}
#if os(iOS)
/// US-layout character Windows VK for the on-screen keyboard (`StreamLayerUIView`'s
/// UIKeyInput). Unlike every other key source, `insertText` delivers CHARACTERS, not key
/// positions, so this is the inverse of a US layout: `shift` means "wrap in VK_LSHIFT so the
/// host types the shifted symbol". Same contract as `hidToVK`: emit only VKs the host's
/// vk_to_evdev knows; anything unmapped is dropped by the caller.
enum SoftKeyMap {
static func vk(for ch: Character) -> (vk: UInt32, shift: Bool)? {
guard let ascii = ch.asciiValue else { return nil }
switch ascii {
case UInt8(ascii: "a")...UInt8(ascii: "z"): return (UInt32(ascii) - 0x20, false)
case UInt8(ascii: "A")...UInt8(ascii: "Z"): return (UInt32(ascii), true)
case UInt8(ascii: "0")...UInt8(ascii: "9"): return (UInt32(ascii), false)
case 0x0A, 0x0D: return (0x0D, false) // return
case 0x09: return (0x09, false) // tab
case 0x20: return (0x20, false) // space
default: return symbols[ch]
}
}
/// US punctuation, plain and shifted, on the OEM VKs (mirrors `hidToVK`'s OEM block) plus
/// the shifted digit row.
private static let symbols: [Character: (vk: UInt32, shift: Bool)] = [
"-": (0xBD, false), "_": (0xBD, true),
"=": (0xBB, false), "+": (0xBB, true),
"[": (0xDB, false), "{": (0xDB, true),
"]": (0xDD, false), "}": (0xDD, true),
"\\": (0xDC, false), "|": (0xDC, true),
";": (0xBA, false), ":": (0xBA, true),
"'": (0xDE, false), "\"": (0xDE, true),
"`": (0xC0, false), "~": (0xC0, true),
",": (0xBC, false), "<": (0xBC, true),
".": (0xBE, false), ">": (0xBE, true),
"/": (0xBF, false), "?": (0xBF, true),
"!": (0x31, true), "@": (0x32, true), "#": (0x33, true), "$": (0x34, true),
"%": (0x35, true), "^": (0x36, true), "&": (0x37, true), "*": (0x38, true),
"(": (0x39, true), ")": (0x30, true),
]
}
#endif
@@ -3,7 +3,8 @@
// identical. Two mouse modes share one gesture vocabulary tap = left click · two-finger
// tap = right click · two-finger drag = scroll · tap-then-press-and-drag = held left drag
// (text selection / window moves) · three-finger tap = cycles the stats overlay tiers
// (off compact normal detailed, matching Android):
// (off compact normal detailed, matching Android) · three-finger swipe up/down =
// summon/dismiss the local soft keyboard for typing on the host (`onKeyboardGesture`):
//
// * trackpad (default): the cursor STAYS PUT on touch-down and moves by the finger's
// relative delta with mild acceleration swipe to nudge, lift and re-swipe to walk it
@@ -61,6 +62,9 @@ final class TouchMouse {
static let accelGain: CGFloat = 0.6
static let accelSpeedFloor: CGFloat = 0.3
static let accelMax: CGFloat = 3.0
/// Three-finger vertical swipe: the fraction of the view height the centroid must
/// travel to summon (up) / dismiss (down) the local soft keyboard.
static let keyboardSwipeFraction: CGFloat = 0.10
/// Acceleration multiplier for a finger speed in physical px per ms.
static func accel(forSpeed speed: CGFloat) -> CGFloat {
@@ -72,6 +76,9 @@ final class TouchMouse {
var send: ((PunktfunkInputEvent) -> Void)?
/// View-space point host-mode pixels through the letterbox (pointer mode's moves).
var hostPoint: ((CGPoint) -> StreamLayerUIView.HostPoint?)?
/// Three-finger vertical swipe crossed the threshold: `true` = show the local soft
/// keyboard (swipe up), `false` = dismiss it (swipe down). Fires at most once per gesture.
var onKeyboardGesture: ((Bool) -> Void)?
/// No gesture in flight (all fingers up) the view uses this to release its mode latch.
var isIdle: Bool { !sessionActive && lastPos.isEmpty }
@@ -95,6 +102,11 @@ final class TouchMouse {
private var carryY: CGFloat = 0
/// Scroll anchor (centroid) re-anchored every time a notch fires.
private var scrollAnchor = CGPoint.zero
// Keyboard-swipe state: the 3+-finger centroid anchor (per finger count, like the scroll
// anchor) and a once-per-gesture latch.
private var kbCount = 0
private var kbAnchor = CGPoint.zero
private var kbFired = false
// Tap-drag arming: a quick tap leaves a window in which the next nearby touch drags.
private var lastTapUp: TimeInterval = 0
private var lastTapPoint = CGPoint.zero
@@ -114,6 +126,8 @@ final class TouchMouse {
maxFingers = 0
moved = false
scrolling = false
kbCount = 0
kbFired = false
// A touch landing just after a quick tap nearby = tap-and-drag: hold the left
// button for this whole gesture (laptop-trackpad convention).
dragHeld = first.timestamp - lastTapUp < Tuning.tapDragWindow
@@ -140,8 +154,13 @@ final class TouchMouse {
for touch in touches where lastPos[ObjectIdentifier(touch)] != nil {
lastPos[ObjectIdentifier(touch)] = touch.location(in: view)
}
if lastPos.count >= 2 {
// Dropping below three fingers forgets the keyboard-swipe anchor, so a 323 bounce
// re-anchors instead of reading the count change as swipe travel.
if lastPos.count < 3 { kbCount = 0 }
if lastPos.count == 2 {
scrollByCentroid()
} else if lastPos.count >= 3 {
keyboardSwipe(in: view)
} else if !scrolling, let touch = touches.first(where: {
lastPos[ObjectIdentifier($0)] != nil
}) {
@@ -208,9 +227,9 @@ final class TouchMouse {
// MARK: - Per-event work
/// Two fingers (or more) scroll by the centroid delta; never move the cursor. Fires a
/// notch per `scrollNotchPt` of pan and re-anchors on fire; finger up scrolls up, finger
/// right scrolls right (the host WHEEL(120) convention).
/// Two fingers scroll by the centroid delta; never move the cursor. Fires a notch per
/// `scrollNotchPt` of pan and re-anchors on fire; finger up scrolls up, finger right
/// scrolls right (the host WHEEL(120) convention).
private func scrollByCentroid() {
let n = CGFloat(lastPos.count)
let cx = lastPos.values.reduce(0) { $0 + $1.x } / n
@@ -233,6 +252,38 @@ final class TouchMouse {
}
}
/// Three+ fingers the keyboard swipe, never scroll (the documented vocabulary is
/// TWO-finger scroll; 3+ only fell into the scroll path as an accident of its old `>= 2`
/// bound). The centroid is anchored per finger count real fingers never land or lift in
/// the same event, so a count change must re-anchor rather than read as travel and the
/// gesture fires at most once, when the vertical travel crosses the threshold: up = show
/// the local soft keyboard, down = dismiss it.
private func keyboardSwipe(in view: UIView) {
let n = CGFloat(lastPos.count)
let cx = lastPos.values.reduce(0) { $0 + $1.x } / n
let cy = lastPos.values.reduce(0) { $0 + $1.y } / n
if lastPos.count != kbCount {
kbCount = lastPos.count
kbAnchor = CGPoint(x: cx, y: cy)
} else {
let dy = cy - kbAnchor.y
// Real centroid travel disqualifies the tap classification in `ended` (else a
// sub-threshold swipe would still fire the three-finger stats tap).
if abs(dy) > Tuning.tapSlop || abs(cx - kbAnchor.x) > Tuning.tapSlop { moved = true }
if !kbFired, abs(dy) >= view.bounds.height * Tuning.keyboardSwipeFraction {
kbFired = true
onKeyboardGesture?(dy < 0) // finger up show, finger down dismiss
}
}
// Leaving the scroll state stale would read the 32 centroid jump as a wheel notch;
// clearing it makes a return to two fingers re-anchor fresh. Same for the trackpad's
// tracked finger: its prev position froze while 3+ fingers were down, so dropping
// straight back to one finger must re-anchor (zero delta), not replay the whole
// 3-finger phase as one cursor jump.
scrolling = false
trackKey = nil
}
/// One finger (and the gesture never became a scroll dropping back from two fingers to
/// one must not jerk the cursor).
private func singleFinger(_ touch: UITouch, in view: UIView) {