feat(apple,android): three-way touch input — trackpad cursor (default), direct pointer, real multi-touch passthrough

The two touch clients had exactly complementary gaps: iOS forwarded fingers
ONLY as raw wire touches (no way to drive the host cursor from the touch
screen), Android had the two mouse modes but no passthrough. Both now share
one three-way "Touch input" setting: Trackpad (default) / Direct pointer /
Touch passthrough.

iOS/iPadOS: Input/TouchMouse.swift ports the Android gesture engine 1:1
(same px-based acceleration curve; tap=click, two-finger tap=right-click,
two-finger drag=scroll, tap-then-drag=held drag, three-finger tap=stats
HUD via the shared hudEnabled default); direct-pointer mode maps through
the aspect-fit letterbox; the previous always-on behavior lives on as the
passthrough option. The mode latches per gesture (a Settings change never
splits one gesture across models), touchesCancelled releases held state
without synthesizing a click, and session stop flushes a mid-drag button.
Settings picker on iPhone + iPad next to the iPad-only pointer-capture
toggle. Deliberate default change: trackpad, not passthrough.

Android: new nativeSendTouch JNI shim → wire TouchDown/Move/Up (the host
already injects real touch on every backend — libei touchscreen, wlroots,
KWin fake-input, SendInput); streamTouchPassthrough forwards every finger
with stable ids and lifts still-held contacts on teardown; the trackpadMode
Boolean becomes the TouchMode enum (old pref migrated on load, never
rewritten) with a Settings dropdown.

Verified: macOS swift build + full suite (incl. new TouchMouseTests), iOS
Simulator Swift compile, cargo check/fmt/clippy on the native crate, Kotlin
app+kit compile + unit tests. On-glass feel of the iOS ballistics and
Android passthrough against a touch-aware app still pending.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift

@@ -201,25 +201,36 @@ extension SettingsView {
     }
 
     #if os(iOS)
-    /// iPad-only pointer-capture toggle: lock the mouse/trackpad for relative movement (games) vs
-    /// forward an absolute cursor position (desktop). Empty on iPhone (no hardware-pointer lock —
-    /// the mouse path there is always the absolute fallback).
+    /// Touch-input model (iPhone + iPad) plus the iPad-only pointer-capture toggle: lock the
+    /// mouse/trackpad for relative movement (games) vs forward an absolute cursor position.
     @ViewBuilder var pointerSection: some View {
-        if UIDevice.current.userInterfaceIdiom == .pad {
-            Section {
-                Toggle("Capture pointer for games", isOn: $pointerCapture)
-            } header: {
-                Text("Pointer")
-            } footer: {
-                Text("With a mouse or trackpad connected, lock the pointer and send relative "
-                    + "movement — the expected behavior for games (mouse-look). Turn this off for "
-                    + "desktop use to keep the pointer free and send its absolute position instead. "
-                    + "The lock needs the stream full-screen and frontmost; it falls back to the "
-                    + "absolute pointer automatically (Stage Manager, Slide Over). Finger touch is "
-                    + "unaffected. Applies from the next session.")
-                    .font(.geist(12, relativeTo: .caption))
-                    .foregroundStyle(.secondary)
+        let isPad = UIDevice.current.userInterfaceIdiom == .pad
+        Section {
+            Picker("Touch input", selection: $touchMode) {
+                Text("Trackpad").tag(TouchInputMode.trackpad.rawValue)
+                Text("Direct pointer").tag(TouchInputMode.pointer.rawValue)
+                Text("Touch passthrough").tag(TouchInputMode.touch.rawValue)
             }
+            if isPad {
+                Toggle("Capture pointer for games", isOn: $pointerCapture)
+            }
+        } header: {
+            Text("Touch & pointer")
+        } footer: {
+            Text("Trackpad: your finger nudges the host cursor like a laptop touchpad — tap to "
+                + "click, two-finger tap for a right click, two-finger drag to scroll, "
+                + "tap-then-drag to hold the button, three-finger tap for the stats overlay. "
+                + "Direct pointer: the cursor jumps to your finger. Touch passthrough: real "
+                + "multi-touch reaches the host, for apps that understand touch. Applies from "
+                + "the next touch."
+                + (isPad
+                    ? " Pointer capture locks a hardware mouse/trackpad for relative movement "
+                        + "(mouse-look); off keeps the pointer free and sends absolute positions. "
+                        + "The lock needs the stream full-screen and frontmost, and falls back "
+                        + "automatically (Stage Manager, Slide Over)."
+                    : ""))
+                .font(.geist(12, relativeTo: .caption))
+                .foregroundStyle(.secondary)
         }
     }
     #endif

clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift

@@ -43,6 +43,7 @@ struct SettingsView: View {
     #endif
     #if os(iOS)
     @AppStorage(DefaultsKey.pointerCapture) var pointerCapture = true
+    @AppStorage(DefaultsKey.touchMode) var touchMode = TouchInputMode.trackpad.rawValue
     // The sidebar selection drives the detail pane on iPad and the pushed sub-page on iPhone.
     // Width class decides the initial value: nil on iPhone (show the category list first),
     // General on iPad (a two-column layout should never open with an empty detail).

clients/apple/Sources/PunktfunkKit/Input/TouchMouse.swift

@@ -0,0 +1,285 @@
+// Finger touches → host mouse, for the touchscreen devices: a port of the Android client's
+// touch gesture model (clients/android .../TouchInput.kt) so the two touch clients feel
+// identical. Two mouse modes share one gesture vocabulary — tap = left click · two-finger
+// tap = right click · two-finger drag = scroll · tap-then-press-and-drag = held left drag
+// (text selection / window moves) · three-finger tap = stats-HUD toggle:
+//
+//  * trackpad (default): the cursor STAYS PUT on touch-down and moves by the finger's
+//    relative delta with mild acceleration — swipe to nudge, lift and re-swipe to walk it
+//    across, tap to click where it is. This is what makes the cursor reachable on a small
+//    screen.
+//  * pointer: the cursor jumps to the finger and follows it (absolute moves through the
+//    aspect-fit letterbox) — direct pointing for desktop-style use.
+//
+// The third `TouchInputMode` (`touch`) never reaches this type: `StreamLayerUIView` forwards
+// those fingers as REAL wire touches (multi-touch passthrough) instead.
+
+#if os(iOS)
+import Foundation
+import PunktfunkCore
+import UIKit
+
+/// How touchscreen fingers drive the host — persisted under `DefaultsKey.touchMode`, latched
+/// per gesture by `StreamLayerUIView` (a Settings change applies from the NEXT touch, and a
+/// gesture never splits across models). `trackpad` is the default: a cursor is the
+/// universally workable model; passthrough only helps hosts/apps that actually speak touch.
+public enum TouchInputMode: String, CaseIterable, Sendable {
+    case trackpad
+    case pointer
+    case touch
+
+    /// The persisted setting, defaulting to trackpad when unset/unknown.
+    public static var current: TouchInputMode {
+        TouchInputMode(
+            rawValue: UserDefaults.standard.string(forKey: DefaultsKey.touchMode) ?? ""
+        ) ?? .trackpad
+    }
+}
+
+/// The gesture state machine behind the two mouse modes. One instance per stream view, fed
+/// only the DIRECT touches (fingers/Pencil — indirect pointers have their own path). Runs
+/// entirely on the main thread (UIKit touch delivery). Touches are tracked by identity key
+/// with positions cached per event — `UITouch` objects are never retained.
+final class TouchMouse {
+    /// Gesture/ballistics tuning. Distances are in points where they gate gestures; the
+    /// relative ballistics work in PHYSICAL pixels (point deltas × screen scale) so the
+    /// acceleration curve matches the Android client's pixel-based constants 1:1.
+    enum Tuning {
+        /// Movement under this (pt) still counts as a tap, not a drag.
+        static let tapSlop: CGFloat = 8
+        /// A new touch this soon (s) after a tap, near it, starts a held left-button drag.
+        static let tapDragWindow: TimeInterval = 0.25
+        /// Two-finger pan distance (pt) per 120-unit wheel notch — matches the feel of the
+        /// indirect-trackpad scroll path in StreamViewIOS (~10 pt per notch).
+        static let scrollNotchPt: CGFloat = 10
+        /// Base finger-px → host-px gain (~1:1, never twitchy). The acceleration below lets a
+        /// flick cross the screen while a slow drag stays precise.
+        static let pointerSens: CGFloat = 1.3
+        /// Above `accelSpeedFloor` px/ms the gain ramps by `accelGain` per px/ms, capped at
+        /// `accelMax` (so a fast swipe can't fling the cursor uncontrollably).
+        static let accelGain: CGFloat = 0.6
+        static let accelSpeedFloor: CGFloat = 0.3
+        static let accelMax: CGFloat = 3.0
+
+        /// Acceleration multiplier for a finger speed in physical px per ms.
+        static func accel(forSpeed speed: CGFloat) -> CGFloat {
+            min(1 + accelGain * max(speed - accelSpeedFloor, 0), accelMax)
+        }
+    }
+
+    /// Wire events out (the owner gates them on its capture state).
+    var send: ((PunktfunkInputEvent) -> Void)?
+    /// View-space point → host-mode pixels through the letterbox (pointer mode's moves).
+    var hostPoint: ((CGPoint) -> StreamLayerUIView.HostPoint?)?
+
+    /// No gesture in flight (all fingers up) — the view uses this to release its mode latch.
+    var isIdle: Bool { !sessionActive && lastPos.isEmpty }
+
+    private var trackpad = true
+    /// Last known position per active finger (identity key) — kept because moved events only
+    /// carry the CHANGED touches while the scroll centroid needs every finger.
+    private var lastPos: [ObjectIdentifier: CGPoint] = [:]
+    private var sessionActive = false
+    private var startPoint = CGPoint.zero
+    private var maxFingers = 0
+    private var moved = false
+    private var scrolling = false
+    private var dragHeld = false
+    // Trackpad relative-motion state: the tracked finger, its last position/time, and the
+    // sub-pixel remainder so a slow drag isn't lost to integer truncation.
+    private var trackKey: ObjectIdentifier?
+    private var prevPoint = CGPoint.zero
+    private var prevTime: TimeInterval = 0
+    private var carryX: CGFloat = 0
+    private var carryY: CGFloat = 0
+    /// Scroll anchor (centroid) — re-anchored every time a notch fires.
+    private var scrollAnchor = CGPoint.zero
+    // Tap-drag arming: a quick tap leaves a window in which the next nearby touch drags.
+    private var lastTapUp: TimeInterval = 0
+    private var lastTapPoint = CGPoint.zero
+
+    /// GameStream mouse button ids.
+    private enum Button { static let left: UInt32 = 1; static let right: UInt32 = 3 }
+
+    func began(_ touches: Set<UITouch>, in view: UIView, trackpad: Bool) {
+        let starting = lastPos.isEmpty
+        for touch in touches {
+            lastPos[ObjectIdentifier(touch)] = touch.location(in: view)
+        }
+        if starting, let first = touches.first {
+            self.trackpad = trackpad
+            sessionActive = true
+            startPoint = first.location(in: view)
+            maxFingers = 0
+            moved = false
+            scrolling = false
+            // A touch landing just after a quick tap nearby = tap-and-drag: hold the left
+            // button for this whole gesture (laptop-trackpad convention).
+            dragHeld = first.timestamp - lastTapUp < Tuning.tapDragWindow
+                && abs(startPoint.x - lastTapPoint.x) < Tuning.tapSlop
+                && abs(startPoint.y - lastTapPoint.y) < Tuning.tapSlop
+            lastTapUp = 0 // consume the arming either way
+            // Pointer mode jumps the cursor to the finger; trackpad leaves it put (the whole
+            // point — you nudge it with swipes instead).
+            if !trackpad, let h = hostPoint?(startPoint) {
+                send?(.mouseMoveAbs(x: h.x, y: h.y, surfaceWidth: h.w, surfaceHeight: h.h))
+            }
+            if dragHeld { send?(.mouseButton(Button.left, down: true)) }
+            trackKey = ObjectIdentifier(first)
+            prevPoint = startPoint
+            prevTime = first.timestamp
+            carryX = 0
+            carryY = 0
+        }
+        maxFingers = max(maxFingers, lastPos.count)
+    }
+
+    func moved(_ touches: Set<UITouch>, in view: UIView) {
+        guard sessionActive else { return }
+        for touch in touches where lastPos[ObjectIdentifier(touch)] != nil {
+            lastPos[ObjectIdentifier(touch)] = touch.location(in: view)
+        }
+        if lastPos.count >= 2 {
+            scrollByCentroid()
+        } else if !scrolling, let touch = touches.first(where: {
+            lastPos[ObjectIdentifier($0)] != nil
+        }) {
+            singleFinger(touch, in: view)
+        }
+    }
+
+    func ended(_ touches: Set<UITouch>, in view: UIView) {
+        guard sessionActive || !lastPos.isEmpty else { return }
+        var upTime: TimeInterval = 0
+        for touch in touches {
+            lastPos.removeValue(forKey: ObjectIdentifier(touch))
+            if trackKey == ObjectIdentifier(touch) { trackKey = nil }
+            upTime = max(upTime, touch.timestamp)
+        }
+        guard lastPos.isEmpty, sessionActive else { return }
+        sessionActive = false
+        if dragHeld {
+            dragHeld = false
+            send?(.mouseButton(Button.left, down: false)) // end the drag
+        } else if !moved {
+            switch maxFingers {
+            case 3...:
+                Self.toggleHUD() // in-stream stats-overlay toggle, same as Android
+            case 2: // two-finger tap → right click
+                send?(.mouseButton(Button.right, down: true))
+                send?(.mouseButton(Button.right, down: false))
+            default: // tap → left click (at the cursor's current spot), arm tap-drag
+                send?(.mouseButton(Button.left, down: true))
+                send?(.mouseButton(Button.left, down: false))
+                lastTapUp = upTime
+                lastTapPoint = startPoint
+            }
+        }
+    }
+
+    /// System-cancelled touches (incoming call, gesture takeover): release anything held but
+    /// never synthesize a click out of a cancellation.
+    func cancelled(_ touches: Set<UITouch>) {
+        for touch in touches {
+            lastPos.removeValue(forKey: ObjectIdentifier(touch))
+            if trackKey == ObjectIdentifier(touch) { trackKey = nil }
+        }
+        if lastPos.isEmpty { abortSession() }
+    }
+
+    /// Session teardown: release anything held on the wire and forget all gesture state.
+    func reset() {
+        lastPos.removeAll()
+        trackKey = nil
+        abortSession()
+        lastTapUp = 0
+    }
+
+    private func abortSession() {
+        if dragHeld {
+            dragHeld = false
+            send?(.mouseButton(Button.left, down: false))
+        }
+        sessionActive = false
+        scrolling = false
+        moved = false
+    }
+
+    // MARK: - Per-event work
+
+    /// Two fingers (or more) → scroll by the centroid delta; never move the cursor. Fires a
+    /// notch per `scrollNotchPt` of pan and re-anchors on fire; finger up scrolls up, finger
+    /// right scrolls right (the host WHEEL(120) convention).
+    private func scrollByCentroid() {
+        let n = CGFloat(lastPos.count)
+        let cx = lastPos.values.reduce(0) { $0 + $1.x } / n
+        let cy = lastPos.values.reduce(0) { $0 + $1.y } / n
+        if !scrolling {
+            scrolling = true
+            scrollAnchor = CGPoint(x: cx, y: cy)
+        }
+        let notchesY = Int32((scrollAnchor.y - cy) / Tuning.scrollNotchPt)
+        let notchesX = Int32((cx - scrollAnchor.x) / Tuning.scrollNotchPt)
+        if notchesY != 0 {
+            send?(.scroll(notchesY * 120))
+            scrollAnchor.y = cy
+            moved = true
+        }
+        if notchesX != 0 {
+            send?(.scroll(notchesX * 120, horizontal: true))
+            scrollAnchor.x = cx
+            moved = true
+        }
+    }
+
+    /// One finger (and the gesture never became a scroll — dropping back from two fingers to
+    /// one must not jerk the cursor).
+    private func singleFinger(_ touch: UITouch, in view: UIView) {
+        let loc = touch.location(in: view)
+        if abs(loc.x - startPoint.x) > Tuning.tapSlop || abs(loc.y - startPoint.y) > Tuning.tapSlop {
+            moved = true
+        }
+        guard trackpad else {
+            if let h = hostPoint?(loc) { // pointer mode: the cursor follows the finger
+                send?(.mouseMoveAbs(x: h.x, y: h.y, surfaceWidth: h.w, surfaceHeight: h.h))
+            }
+            return
+        }
+        // Relative: move by the finger delta × (sensitivity × acceleration), carrying the
+        // sub-pixel remainder. Re-anchor (zero delta this frame) if the tracked finger
+        // changed, so lifting one of several fingers never jumps the cursor.
+        let key = ObjectIdentifier(touch)
+        if key != trackKey {
+            trackKey = key
+            prevPoint = loc
+            prevTime = touch.timestamp
+            return
+        }
+        // Ballistics in physical pixels so the curve matches the Android tuning exactly.
+        let scale = view.window?.screen.scale ?? view.traitCollection.displayScale
+        let dx = (loc.x - prevPoint.x) * scale
+        let dy = (loc.y - prevPoint.y) * scale
+        let dtMs = max((touch.timestamp - prevTime) * 1000, 1)
+        prevPoint = loc
+        prevTime = touch.timestamp
+        let gain = Tuning.pointerSens * Tuning.accel(forSpeed: hypot(dx, dy) / dtMs)
+        carryX += dx * gain
+        carryY += dy * gain
+        let outX = Int32(carryX) // truncates toward zero → remainder kept with its sign
+        let outY = Int32(carryY)
+        if outX != 0 || outY != 0 {
+            send?(.mouseMove(dx: outX, dy: outY))
+            carryX -= CGFloat(outX)
+            carryY -= CGFloat(outY)
+        }
+    }
+
+    /// Three-finger tap toggles the stats overlay — through the shared `hudEnabled` default,
+    /// which the app's HUD views observe via @AppStorage (so this needs no wiring to them).
+    private static func toggleHUD() {
+        let defaults = UserDefaults.standard
+        let on = defaults.object(forKey: DefaultsKey.hudEnabled) as? Bool ?? true
+        defaults.set(!on, forKey: DefaultsKey.hudEnabled)
+    }
+}
+#endif

clients/apple/Sources/PunktfunkKit/Support/DefaultsKeys.swift

@@ -41,6 +41,11 @@ public enum DefaultsKey {
     /// scene and silently falls back to the absolute pointer when it can't (Stage Manager / Slide
     /// Over). Read by `StreamViewController.prefersPointerLocked`.
     public static let pointerCapture = "punktfunk.pointerCapture"
+    /// iPhone/iPad: how touchscreen fingers drive the host — a `TouchInputMode` raw value:
+    /// "trackpad" (default: relative cursor with tap-click / two-finger-scroll gestures),
+    /// "pointer" (the cursor jumps to the finger), or "touch" (real multi-touch passthrough).
+    /// Read live per gesture by `StreamLayerUIView`.
+    public static let touchMode = "punktfunk.touchMode"
     /// Experimental: show the host's game library (browsed over the management API). Off by default.
     public static let libraryEnabled = "punktfunk.libraryEnabled"
     /// macOS: take the window fullscreen while streaming and restore it on the host list. On by default.

clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift

@@ -339,6 +339,9 @@ public final class StreamViewController: UIViewController {
         setCaptured(false)
         inputCapture?.stop()
         inputCapture = nil
+        // Release anything the touch-driven mouse still holds (a mid-drag session end) while
+        // onTouchEvent can still deliver the button-up.
+        streamView.resetTouchInput()
         streamView.onTouchEvent = nil
         streamView.onPointerMoveAbs = nil
         streamView.onPointerButton = nil
@@ -454,7 +457,8 @@ final class StreamLayerUIView: UIView {
 
     /// Reads the LIVE negotiated mode in pixels (the touch/pointer coordinate space).
     var currentHostMode: (() -> CGSize)?
-    /// Direct fingers / Pencil → wire touch events.
+    /// Direct fingers / Pencil → wire events: real touches in passthrough mode, or the
+    /// touch-driven mouse events (`TouchMouse`) in the trackpad/pointer modes.
     var onTouchEvent: ((PunktfunkInputEvent) -> Void)?
     /// Indirect pointer (mouse/trackpad with no lock) → absolute cursor moves.
     var onPointerMoveAbs: ((HostPoint) -> Void)?
@@ -468,6 +472,22 @@ final class StreamLayerUIView: UIView {
     /// GameStream button held per active indirect-pointer touch (one click/drag session);
     /// released when that touch ends.
     private var pointerButtons: [ObjectIdentifier: UInt32] = [:]
+    /// Touch-driven mouse for the trackpad/pointer `TouchInputMode`s (see TouchMouse.swift).
+    private lazy var touchMouse: TouchMouse = {
+        let mouse = TouchMouse()
+        mouse.send = { [weak self] event in self?.onTouchEvent?(event) }
+        mouse.hostPoint = { [weak self] point in self?.hostPoint(from: point) }
+        return mouse
+    }()
+    /// The finger route latched at gesture start — a Settings change mid-gesture applies to
+    /// the NEXT touch, so one gesture never splits across input models.
+    private var fingerRoute: TouchInputMode?
+
+    /// Release anything the touch-driven mouse holds and forget gesture state — session stop.
+    func resetTouchInput() {
+        touchMouse.reset()
+        fingerRoute = nil
+    }
     #endif
 
     override init(frame: CGRect) {
@@ -504,10 +524,10 @@ final class StreamLayerUIView: UIView {
         route(touches, event: event, kind: .up)
     }
     override func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent?) {
-        route(touches, event: event, kind: .up)
+        route(touches, event: event, kind: .cancel)
     }
 
-    private enum TouchKind { case down, move, up }
+    private enum TouchKind { case down, move, up, cancel }
 
     /// Split a touch batch by kind: an INDIRECT POINTER (mouse/trackpad with no lock) drives
     /// the host cursor as an absolute mouse; everything else (direct finger, Pencil) is a host
@@ -521,7 +541,28 @@ final class StreamLayerUIView: UIView {
                 fingers.insert(touch)
             }
         }
-        if !fingers.isEmpty { forwardTouches(fingers, kind: kind) }
+        if !fingers.isEmpty { forwardFingers(fingers, kind: kind) }
+    }
+
+    /// Route direct fingers by the touch-input model, latched for the whole gesture:
+    /// passthrough → real wire touches; trackpad/pointer → the TouchMouse gesture engine.
+    private func forwardFingers(_ touches: Set<UITouch>, kind: TouchKind) {
+        let mode = fingerRoute ?? TouchInputMode.current
+        fingerRoute = mode
+        switch mode {
+        case .touch:
+            // A cancellation lifts the wire touch like a normal up — the host just sees the
+            // contact end.
+            forwardTouches(touches, kind: kind == .cancel ? .up : kind)
+        case .trackpad, .pointer:
+            switch kind {
+            case .down: touchMouse.began(touches, in: self, trackpad: mode == .trackpad)
+            case .move: touchMouse.moved(touches, in: self)
+            case .up: touchMouse.ended(touches, in: self)
+            case .cancel: touchMouse.cancelled(touches)
+            }
+        }
+        if touchIDs.isEmpty, touchMouse.isIdle { fingerRoute = nil }
     }
 
     /// An indirect-pointer touch is a button-held click/drag session: forward its position as
@@ -537,7 +578,7 @@ final class StreamLayerUIView: UIView {
             onPointerButton?(button, true)
         case .move:
             if let host { onPointerMoveAbs?(host) }
-        case .up:
+        case .up, .cancel:
             if let host { onPointerMoveAbs?(host) }
             if let button = pointerButtons.removeValue(forKey: key) {
                 onPointerButton?(button, false)
@@ -554,7 +595,7 @@ final class StreamLayerUIView: UIView {
             case .down:
                 id = nextFreeID()
                 touchIDs[key] = id
-            case .move, .up:
+            case .move, .up, .cancel:
                 guard let known = touchIDs[key] else { continue }
                 id = known
             }