diff --git a/CLAUDE.md b/CLAUDE.md index 215fbe6..5ab54f5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -204,7 +204,13 @@ Low-latency desktop/game streaming stack, Linux-first, with a shared Rust protoc "always show scroll bars" overrides `.hidden`); launcher/settings/add-host/keyboard render-verified live on this Mac via `PUNKTFUNK_FORCE_GAMEPAD_UI=1` (dev hook, forces the mode without a pad). Controller-in-hand on-glass validation still pending on all - platforms. Tests: `swift test` in + platforms. **Touch input (iOS/iPadOS, 2026-07-02):** a 3-way model in Settings — + **Trackpad** (default; the Android client's gesture vocabulary ported 1:1 in + `Input/TouchMouse.swift`: tap=click · two-finger tap=right-click · two-finger drag=scroll · + tap-then-drag=held drag · three-finger tap=HUD toggle, relative ballistics with the same + px-based acceleration curve), **Direct pointer** (cursor jumps to the finger), **Touch + passthrough** (the previous always-on behavior — real wire touches). Latched per gesture + from `DefaultsKey.touchMode`; not yet on-glass validated. Tests: `swift test` in `clients/apple` (unit + real-codec round trip), `test-loopback.sh` (Swift client vs synthetic punktfunk1-hosts on loopback — runs on macOS; includes the pairing ceremony + `--require-pairing` gate), @@ -350,7 +356,11 @@ Low-latency desktop/game streaming stack, Linux-first, with a shared Rust protoc the `MulticastLock` + permission UX), SPAKE2 PIN pairing + TOFU (Keystore identity + known-host store), Compose UI (Connect/Settings/Stream) with D-pad/controller focus nav. Built for `arm64-v8a` + `x86_64`; published to Google Play (Internal Testing) via `android.yml` - (`ci/play-upload.py`). Next: real-device gamepad/HDR live-verify, presenter/latency polish. + (`ci/play-upload.py`). Touch input is the same 3-way model as iOS (2026-07-02): the existing + Trackpad/Direct mouse modes plus new **real multi-touch passthrough** + (`streamTouchPassthrough` → `nativeSendTouch` → wire TouchDown/Move/Up), a `TouchMode` + Settings dropdown replacing the old trackpad Boolean (migrated on load); not yet + on-device validated. Next: real-device gamepad/HDR live-verify, presenter/latency polish. 2. **Sub-frame pipelining**: overlap encode and transmit within a frame. Requires a direct NVENC SDK wrapper (libavcodec only emits whole AUs) — the next big latency lever (~2–4 ms at high res). diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt index 1499d16..368bfd1 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt @@ -33,13 +33,19 @@ data class Settings( /** Show the live stats overlay (FPS / throughput / latency) during a stream. */ val statsHudEnabled: Boolean = true, /** - * Touch input model. `true` (default) = trackpad: the cursor stays put on touch-down and moves - * by the finger's relative delta (swipe to nudge, lift and re-swipe to walk it across), tap to - * click where it is. `false` = direct pointing: the cursor jumps to the finger (the old behaviour). + * Touch input model — how touchscreen fingers drive the host. [TouchMode.TRACKPAD] (default): + * the cursor stays put on touch-down and moves by the finger's relative delta (swipe to nudge, + * lift and re-swipe to walk it across), tap to click where it is. [TouchMode.POINTER]: the + * cursor jumps to the finger (direct pointing). [TouchMode.TOUCH]: real multi-touch + * passthrough — every finger reaches the host as a touchscreen contact, for apps/games that + * understand touch. Mirrors the Apple client's TouchInputMode. */ - val trackpadMode: Boolean = true, + val touchMode: TouchMode = TouchMode.TRACKPAD, ) +/** [Settings.touchMode] values; persisted by name. */ +enum class TouchMode { TRACKPAD, POINTER, TOUCH } + /** Loads/saves [Settings] in the app-private `punktfunk_settings` prefs. */ class SettingsStore(context: Context) { private val prefs = @@ -57,7 +63,10 @@ class SettingsStore(context: Context) { codec = prefs.getString(K_CODEC, "auto") ?: "auto", micEnabled = prefs.getBoolean(K_MIC, false), statsHudEnabled = prefs.getBoolean(K_HUD, true), - trackpadMode = prefs.getBoolean(K_TRACKPAD, true), + touchMode = prefs.getString(K_TOUCH_MODE, null) + ?.let { name -> TouchMode.entries.firstOrNull { it.name == name } } + // Migration: the pre-enum Boolean "trackpad_mode" (true = trackpad, false = direct). + ?: if (prefs.getBoolean(K_TRACKPAD, true)) TouchMode.TRACKPAD else TouchMode.POINTER, ) fun save(s: Settings) { @@ -73,7 +82,7 @@ class SettingsStore(context: Context) { .putString(K_CODEC, s.codec) .putBoolean(K_MIC, s.micEnabled) .putBoolean(K_HUD, s.statsHudEnabled) - .putBoolean(K_TRACKPAD, s.trackpadMode) + .putString(K_TOUCH_MODE, s.touchMode.name) .apply() } @@ -89,6 +98,9 @@ class SettingsStore(context: Context) { const val K_CODEC = "codec" const val K_MIC = "mic_enabled" const val K_HUD = "stats_hud_enabled" + const val K_TOUCH_MODE = "touch_mode" + + /** Legacy Boolean the enum replaced — read once as the migration default, never written. */ const val K_TRACKPAD = "trackpad_mode" } } @@ -195,6 +207,13 @@ val COMPOSITOR_OPTIONS = listOf( "gamescope", ) +/** (mode, label) for the touch-input model. */ +val TOUCH_MODE_OPTIONS = listOf( + TouchMode.TRACKPAD to "Trackpad", + TouchMode.POINTER to "Direct pointer", + TouchMode.TOUCH to "Touch passthrough", +) + /** index = GamepadPref wire byte (0=Auto 1=Xbox360 2=DualSense 3=XboxOne 4=DualShock4). */ val GAMEPAD_OPTIONS = listOf( "Automatic", diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt index 6fbd079..489819b 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt @@ -165,13 +165,21 @@ fun SettingsScreen(initial: Settings, onChange: (Settings) -> Unit, onBack: () - ) } - SettingsGroup("Pointer") { - ToggleRow( - title = "Trackpad mode", - subtitle = "Relative cursor like a laptop touchpad — swipe to nudge, tap to click. " + - "Off = the cursor jumps to your finger.", - checked = s.trackpadMode, - onCheckedChange = { on -> update(s.copy(trackpadMode = on)) }, + SettingsGroup("Touch input") { + SettingDropdown( + label = "Touch input", + options = TOUCH_MODE_OPTIONS, + selected = s.touchMode, + onSelect = { mode -> update(s.copy(touchMode = mode)) }, + ) + Text( + "Trackpad: relative cursor like a laptop touchpad — tap to click, two-finger " + + "tap right-clicks, two fingers scroll, tap-then-drag holds the button. " + + "Direct pointer: the cursor jumps to your finger. Touch passthrough: real " + + "multi-touch reaches the host, for apps that understand touch.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(top = 6.dp), ) } diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt index f889ac3..ab14f25 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt @@ -57,7 +57,7 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { var stats by remember { mutableStateOf(null) } var showStats by remember { mutableStateOf(initialSettings.statsHudEnabled) } // Touch model is fixed per session (re-keys the gesture handler below if it ever changes). - val trackpad = initialSettings.trackpadMode + val touchMode = initialSettings.touchMode LaunchedEffect(handle, showStats) { NativeBridge.nativeSetVideoStatsEnabled(handle, showStats) if (showStats) { @@ -148,11 +148,18 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { if (showStats) { stats?.let { StatsOverlay(it, Modifier.align(Alignment.TopStart).padding(12.dp)) } } - // Touch → mouse (trackpad vs. direct pointing + the shared gesture vocabulary — see - // streamTouchInput in TouchInput.kt). + // Touch input per the Settings model: trackpad/direct-pointer mouse (the shared gesture + // vocabulary) or real multi-touch passthrough — see TouchInput.kt. Box( - Modifier.fillMaxSize().pointerInput(handle, trackpad) { - streamTouchInput(handle, trackpad, onToggleStats = { showStats = !showStats }) + Modifier.fillMaxSize().pointerInput(handle, touchMode) { + when (touchMode) { + TouchMode.TOUCH -> streamTouchPassthrough(handle) + else -> streamTouchInput( + handle, + trackpad = touchMode == TouchMode.TRACKPAD, + onToggleStats = { showStats = !showStats }, + ) + } }, ) } diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt index 715128b..7891a49 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt @@ -2,7 +2,11 @@ package io.unom.punktfunk import androidx.compose.foundation.gestures.awaitEachGesture import androidx.compose.foundation.gestures.awaitFirstDown +import androidx.compose.ui.input.pointer.PointerId import androidx.compose.ui.input.pointer.PointerInputScope +import androidx.compose.ui.input.pointer.changedToDownIgnoreConsumed +import androidx.compose.ui.input.pointer.changedToUpIgnoreConsumed +import androidx.compose.ui.input.pointer.positionChanged import io.unom.punktfunk.kit.NativeBridge import kotlin.math.abs import kotlin.math.hypot @@ -38,6 +42,54 @@ private const val ACCEL_MAX = 3.0f * two-finger drag = scroll; tap-then-press-and-drag = left-drag (text selection / moving * windows); three-finger tap = [onToggleStats] (the stats HUD). */ +/** + * Real multi-touch passthrough ([TouchMode.TOUCH]): every finger forwards as a host touchscreen + * contact (down/move/up with a stable per-finger id), with NO gesture interpretation — taps, + * drags and multi-finger input mean whatever the remote app decides. Coordinates are overlay + * pixels with the overlay size as the surface, exactly like the absolute-mouse path (the host + * normalizes and maps into the output). On teardown (stream leaves composition) every still-held + * contact is lifted so nothing stays stuck on the host. + */ +internal suspend fun PointerInputScope.streamTouchPassthrough(handle: Long) { + val ids = mutableMapOf() + fun alloc(p: PointerId): Int { + var id = 0 + while (ids.containsValue(id)) id++ + ids[p] = id + return id + } + try { + awaitPointerEventScope { + while (true) { + val ev = awaitPointerEvent() + val sw = size.width + val sh = size.height + if (sw <= 0 || sh <= 0) continue + for (c in ev.changes) { + val x = c.position.x.roundToInt().coerceIn(0, sw - 1) + val y = c.position.y.roundToInt().coerceIn(0, sh - 1) + when { + c.changedToDownIgnoreConsumed() -> + NativeBridge.nativeSendTouch(handle, alloc(c.id), 0, x, y, sw, sh) + c.changedToUpIgnoreConsumed() -> + ids.remove(c.id)?.let { + NativeBridge.nativeSendTouch(handle, it, 2, 0, 0, sw, sh) + } + c.positionChanged() -> + ids[c.id]?.let { + NativeBridge.nativeSendTouch(handle, it, 1, x, y, sw, sh) + } + } + c.consume() + } + } + } + } finally { + // Lift anything still down (composition/session teardown mid-touch). + ids.values.forEach { NativeBridge.nativeSendTouch(handle, it, 2, 0, 0, 1, 1) } + } +} + internal suspend fun PointerInputScope.streamTouchInput( handle: Long, trackpad: Boolean, diff --git a/clients/android/app/src/test/kotlin/io/unom/punktfunk/screenshots/ShotScenes.kt b/clients/android/app/src/test/kotlin/io/unom/punktfunk/screenshots/ShotScenes.kt index 2dfd79a..7cd3893 100644 --- a/clients/android/app/src/test/kotlin/io/unom/punktfunk/screenshots/ShotScenes.kt +++ b/clients/android/app/src/test/kotlin/io/unom/punktfunk/screenshots/ShotScenes.kt @@ -27,6 +27,7 @@ import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp import io.unom.punktfunk.BrandDark import io.unom.punktfunk.Settings +import io.unom.punktfunk.TouchMode import io.unom.punktfunk.SettingsScreen import io.unom.punktfunk.StatsOverlay import io.unom.punktfunk.components.HostCard @@ -109,7 +110,7 @@ internal fun SettingsScene() { gamepad = 2, micEnabled = true, statsHudEnabled = true, - trackpadMode = true, + touchMode = TouchMode.TRACKPAD, ), onChange = {}, onBack = {}, diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index f8e39d5..20c0c43 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -159,6 +159,22 @@ object NativeBridge { /** One scroll step. axis: 0=vertical 1=horizontal. delta: signed, 120-scaled, +=up/right. */ external fun nativeSendScroll(handle: Long, axis: Int, delta: Int) + /** + * One REAL touchscreen transition (the touch-passthrough input mode). [kind]: 0=down 1=move + * 2=up. [id] distinguishes fingers and is reusable after up; coordinates are pixels on the + * client's touch surface — the host rescales against [surfaceWidth]×[surfaceHeight] and + * injects a real touch contact. On up only [id] matters. + */ + external fun nativeSendTouch( + handle: Long, + id: Int, + kind: Int, + x: Int, + y: Int, + surfaceWidth: Int, + surfaceHeight: Int, + ) + /** One key transition. vk: Windows VK (0 = dropped by Rust). mods: VK modifier mask (0 for now). */ external fun nativeSendKey(handle: Long, vk: Int, down: Boolean, mods: Int) diff --git a/clients/android/native/src/session/input.rs b/clients/android/native/src/session/input.rs index b5be196..3628549 100644 --- a/clients/android/native/src/session/input.rs +++ b/clients/android/native/src/session/input.rs @@ -93,6 +93,34 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendScroll( send_event(handle, InputKind::MouseScroll, axis as u32, delta, 0, 0); } +/// `NativeBridge.nativeSendTouch(handle, id, kind, x, y, surfaceWidth, surfaceHeight)` — one REAL +/// touchscreen transition (`kind`: 0=down 1=move 2=up), for the touch-passthrough input mode. `id` +/// distinguishes fingers (reusable after up); coordinates are pixels on the client's touch +/// surface, whose size rides in `flags` so the host can rescale into the output (identical +/// packing to MouseMoveAbs). On up only the id matters. The host injects a real touch contact +/// (libei touchscreen / wlroots / SendInput). +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendTouch( + _env: JNIEnv, + _this: JObject, + handle: jlong, + id: jint, + kind: jint, + x: jint, + y: jint, + surface_width: jint, + surface_height: jint, +) { + let kind = match kind { + 0 => InputKind::TouchDown, + 1 => InputKind::TouchMove, + _ => InputKind::TouchUp, + }; + let w = (surface_width.max(0) as u32) & 0xffff; + let h = (surface_height.max(0) as u32) & 0xffff; + send_event(handle, kind, id as u32, x, y, (w << 16) | h); +} + /// `NativeBridge.nativeSendKey(handle, vk, down, mods)` — one key transition. `vk`: Windows /// Virtual-Key code (0 = unmapped → dropped). `down`: 1=press, 0=release. `mods`: VK modifier /// bitmask (0 for now — the host folds modifiers from the L/R modifier key events themselves). diff --git a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift index 218d77f..7a3d712 100644 --- a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift +++ b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift @@ -201,25 +201,36 @@ extension SettingsView { } #if os(iOS) - /// iPad-only pointer-capture toggle: lock the mouse/trackpad for relative movement (games) vs - /// forward an absolute cursor position (desktop). Empty on iPhone (no hardware-pointer lock — - /// the mouse path there is always the absolute fallback). + /// Touch-input model (iPhone + iPad) plus the iPad-only pointer-capture toggle: lock the + /// mouse/trackpad for relative movement (games) vs forward an absolute cursor position. @ViewBuilder var pointerSection: some View { - if UIDevice.current.userInterfaceIdiom == .pad { - Section { - Toggle("Capture pointer for games", isOn: $pointerCapture) - } header: { - Text("Pointer") - } footer: { - Text("With a mouse or trackpad connected, lock the pointer and send relative " - + "movement — the expected behavior for games (mouse-look). Turn this off for " - + "desktop use to keep the pointer free and send its absolute position instead. " - + "The lock needs the stream full-screen and frontmost; it falls back to the " - + "absolute pointer automatically (Stage Manager, Slide Over). Finger touch is " - + "unaffected. Applies from the next session.") - .font(.geist(12, relativeTo: .caption)) - .foregroundStyle(.secondary) + let isPad = UIDevice.current.userInterfaceIdiom == .pad + Section { + Picker("Touch input", selection: $touchMode) { + Text("Trackpad").tag(TouchInputMode.trackpad.rawValue) + Text("Direct pointer").tag(TouchInputMode.pointer.rawValue) + Text("Touch passthrough").tag(TouchInputMode.touch.rawValue) } + if isPad { + Toggle("Capture pointer for games", isOn: $pointerCapture) + } + } header: { + Text("Touch & pointer") + } footer: { + Text("Trackpad: your finger nudges the host cursor like a laptop touchpad — tap to " + + "click, two-finger tap for a right click, two-finger drag to scroll, " + + "tap-then-drag to hold the button, three-finger tap for the stats overlay. " + + "Direct pointer: the cursor jumps to your finger. Touch passthrough: real " + + "multi-touch reaches the host, for apps that understand touch. Applies from " + + "the next touch." + + (isPad + ? " Pointer capture locks a hardware mouse/trackpad for relative movement " + + "(mouse-look); off keeps the pointer free and sends absolute positions. " + + "The lock needs the stream full-screen and frontmost, and falls back " + + "automatically (Stage Manager, Slide Over)." + : "")) + .font(.geist(12, relativeTo: .caption)) + .foregroundStyle(.secondary) } } #endif diff --git a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift index 9f11794..1f34761 100644 --- a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift +++ b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift @@ -43,6 +43,7 @@ struct SettingsView: View { #endif #if os(iOS) @AppStorage(DefaultsKey.pointerCapture) var pointerCapture = true + @AppStorage(DefaultsKey.touchMode) var touchMode = TouchInputMode.trackpad.rawValue // The sidebar selection drives the detail pane on iPad and the pushed sub-page on iPhone. // Width class decides the initial value: nil on iPhone (show the category list first), // General on iPad (a two-column layout should never open with an empty detail). diff --git a/clients/apple/Sources/PunktfunkKit/Input/TouchMouse.swift b/clients/apple/Sources/PunktfunkKit/Input/TouchMouse.swift new file mode 100644 index 0000000..bffc495 --- /dev/null +++ b/clients/apple/Sources/PunktfunkKit/Input/TouchMouse.swift @@ -0,0 +1,285 @@ +// Finger touches → host mouse, for the touchscreen devices: a port of the Android client's +// touch gesture model (clients/android .../TouchInput.kt) so the two touch clients feel +// identical. Two mouse modes share one gesture vocabulary — tap = left click · two-finger +// tap = right click · two-finger drag = scroll · tap-then-press-and-drag = held left drag +// (text selection / window moves) · three-finger tap = stats-HUD toggle: +// +// * trackpad (default): the cursor STAYS PUT on touch-down and moves by the finger's +// relative delta with mild acceleration — swipe to nudge, lift and re-swipe to walk it +// across, tap to click where it is. This is what makes the cursor reachable on a small +// screen. +// * pointer: the cursor jumps to the finger and follows it (absolute moves through the +// aspect-fit letterbox) — direct pointing for desktop-style use. +// +// The third `TouchInputMode` (`touch`) never reaches this type: `StreamLayerUIView` forwards +// those fingers as REAL wire touches (multi-touch passthrough) instead. + +#if os(iOS) +import Foundation +import PunktfunkCore +import UIKit + +/// How touchscreen fingers drive the host — persisted under `DefaultsKey.touchMode`, latched +/// per gesture by `StreamLayerUIView` (a Settings change applies from the NEXT touch, and a +/// gesture never splits across models). `trackpad` is the default: a cursor is the +/// universally workable model; passthrough only helps hosts/apps that actually speak touch. +public enum TouchInputMode: String, CaseIterable, Sendable { + case trackpad + case pointer + case touch + + /// The persisted setting, defaulting to trackpad when unset/unknown. + public static var current: TouchInputMode { + TouchInputMode( + rawValue: UserDefaults.standard.string(forKey: DefaultsKey.touchMode) ?? "" + ) ?? .trackpad + } +} + +/// The gesture state machine behind the two mouse modes. One instance per stream view, fed +/// only the DIRECT touches (fingers/Pencil — indirect pointers have their own path). Runs +/// entirely on the main thread (UIKit touch delivery). Touches are tracked by identity key +/// with positions cached per event — `UITouch` objects are never retained. +final class TouchMouse { + /// Gesture/ballistics tuning. Distances are in points where they gate gestures; the + /// relative ballistics work in PHYSICAL pixels (point deltas × screen scale) so the + /// acceleration curve matches the Android client's pixel-based constants 1:1. + enum Tuning { + /// Movement under this (pt) still counts as a tap, not a drag. + static let tapSlop: CGFloat = 8 + /// A new touch this soon (s) after a tap, near it, starts a held left-button drag. + static let tapDragWindow: TimeInterval = 0.25 + /// Two-finger pan distance (pt) per 120-unit wheel notch — matches the feel of the + /// indirect-trackpad scroll path in StreamViewIOS (~10 pt per notch). + static let scrollNotchPt: CGFloat = 10 + /// Base finger-px → host-px gain (~1:1, never twitchy). The acceleration below lets a + /// flick cross the screen while a slow drag stays precise. + static let pointerSens: CGFloat = 1.3 + /// Above `accelSpeedFloor` px/ms the gain ramps by `accelGain` per px/ms, capped at + /// `accelMax` (so a fast swipe can't fling the cursor uncontrollably). + static let accelGain: CGFloat = 0.6 + static let accelSpeedFloor: CGFloat = 0.3 + static let accelMax: CGFloat = 3.0 + + /// Acceleration multiplier for a finger speed in physical px per ms. + static func accel(forSpeed speed: CGFloat) -> CGFloat { + min(1 + accelGain * max(speed - accelSpeedFloor, 0), accelMax) + } + } + + /// Wire events out (the owner gates them on its capture state). + var send: ((PunktfunkInputEvent) -> Void)? + /// View-space point → host-mode pixels through the letterbox (pointer mode's moves). + var hostPoint: ((CGPoint) -> StreamLayerUIView.HostPoint?)? + + /// No gesture in flight (all fingers up) — the view uses this to release its mode latch. + var isIdle: Bool { !sessionActive && lastPos.isEmpty } + + private var trackpad = true + /// Last known position per active finger (identity key) — kept because moved events only + /// carry the CHANGED touches while the scroll centroid needs every finger. + private var lastPos: [ObjectIdentifier: CGPoint] = [:] + private var sessionActive = false + private var startPoint = CGPoint.zero + private var maxFingers = 0 + private var moved = false + private var scrolling = false + private var dragHeld = false + // Trackpad relative-motion state: the tracked finger, its last position/time, and the + // sub-pixel remainder so a slow drag isn't lost to integer truncation. + private var trackKey: ObjectIdentifier? + private var prevPoint = CGPoint.zero + private var prevTime: TimeInterval = 0 + private var carryX: CGFloat = 0 + private var carryY: CGFloat = 0 + /// Scroll anchor (centroid) — re-anchored every time a notch fires. + private var scrollAnchor = CGPoint.zero + // Tap-drag arming: a quick tap leaves a window in which the next nearby touch drags. + private var lastTapUp: TimeInterval = 0 + private var lastTapPoint = CGPoint.zero + + /// GameStream mouse button ids. + private enum Button { static let left: UInt32 = 1; static let right: UInt32 = 3 } + + func began(_ touches: Set, in view: UIView, trackpad: Bool) { + let starting = lastPos.isEmpty + for touch in touches { + lastPos[ObjectIdentifier(touch)] = touch.location(in: view) + } + if starting, let first = touches.first { + self.trackpad = trackpad + sessionActive = true + startPoint = first.location(in: view) + maxFingers = 0 + moved = false + scrolling = false + // A touch landing just after a quick tap nearby = tap-and-drag: hold the left + // button for this whole gesture (laptop-trackpad convention). + dragHeld = first.timestamp - lastTapUp < Tuning.tapDragWindow + && abs(startPoint.x - lastTapPoint.x) < Tuning.tapSlop + && abs(startPoint.y - lastTapPoint.y) < Tuning.tapSlop + lastTapUp = 0 // consume the arming either way + // Pointer mode jumps the cursor to the finger; trackpad leaves it put (the whole + // point — you nudge it with swipes instead). + if !trackpad, let h = hostPoint?(startPoint) { + send?(.mouseMoveAbs(x: h.x, y: h.y, surfaceWidth: h.w, surfaceHeight: h.h)) + } + if dragHeld { send?(.mouseButton(Button.left, down: true)) } + trackKey = ObjectIdentifier(first) + prevPoint = startPoint + prevTime = first.timestamp + carryX = 0 + carryY = 0 + } + maxFingers = max(maxFingers, lastPos.count) + } + + func moved(_ touches: Set, in view: UIView) { + guard sessionActive else { return } + for touch in touches where lastPos[ObjectIdentifier(touch)] != nil { + lastPos[ObjectIdentifier(touch)] = touch.location(in: view) + } + if lastPos.count >= 2 { + scrollByCentroid() + } else if !scrolling, let touch = touches.first(where: { + lastPos[ObjectIdentifier($0)] != nil + }) { + singleFinger(touch, in: view) + } + } + + func ended(_ touches: Set, in view: UIView) { + guard sessionActive || !lastPos.isEmpty else { return } + var upTime: TimeInterval = 0 + for touch in touches { + lastPos.removeValue(forKey: ObjectIdentifier(touch)) + if trackKey == ObjectIdentifier(touch) { trackKey = nil } + upTime = max(upTime, touch.timestamp) + } + guard lastPos.isEmpty, sessionActive else { return } + sessionActive = false + if dragHeld { + dragHeld = false + send?(.mouseButton(Button.left, down: false)) // end the drag + } else if !moved { + switch maxFingers { + case 3...: + Self.toggleHUD() // in-stream stats-overlay toggle, same as Android + case 2: // two-finger tap → right click + send?(.mouseButton(Button.right, down: true)) + send?(.mouseButton(Button.right, down: false)) + default: // tap → left click (at the cursor's current spot), arm tap-drag + send?(.mouseButton(Button.left, down: true)) + send?(.mouseButton(Button.left, down: false)) + lastTapUp = upTime + lastTapPoint = startPoint + } + } + } + + /// System-cancelled touches (incoming call, gesture takeover): release anything held but + /// never synthesize a click out of a cancellation. + func cancelled(_ touches: Set) { + for touch in touches { + lastPos.removeValue(forKey: ObjectIdentifier(touch)) + if trackKey == ObjectIdentifier(touch) { trackKey = nil } + } + if lastPos.isEmpty { abortSession() } + } + + /// Session teardown: release anything held on the wire and forget all gesture state. + func reset() { + lastPos.removeAll() + trackKey = nil + abortSession() + lastTapUp = 0 + } + + private func abortSession() { + if dragHeld { + dragHeld = false + send?(.mouseButton(Button.left, down: false)) + } + sessionActive = false + scrolling = false + moved = false + } + + // MARK: - Per-event work + + /// Two fingers (or more) → scroll by the centroid delta; never move the cursor. Fires a + /// notch per `scrollNotchPt` of pan and re-anchors on fire; finger up scrolls up, finger + /// right scrolls right (the host WHEEL(120) convention). + private func scrollByCentroid() { + let n = CGFloat(lastPos.count) + let cx = lastPos.values.reduce(0) { $0 + $1.x } / n + let cy = lastPos.values.reduce(0) { $0 + $1.y } / n + if !scrolling { + scrolling = true + scrollAnchor = CGPoint(x: cx, y: cy) + } + let notchesY = Int32((scrollAnchor.y - cy) / Tuning.scrollNotchPt) + let notchesX = Int32((cx - scrollAnchor.x) / Tuning.scrollNotchPt) + if notchesY != 0 { + send?(.scroll(notchesY * 120)) + scrollAnchor.y = cy + moved = true + } + if notchesX != 0 { + send?(.scroll(notchesX * 120, horizontal: true)) + scrollAnchor.x = cx + moved = true + } + } + + /// One finger (and the gesture never became a scroll — dropping back from two fingers to + /// one must not jerk the cursor). + private func singleFinger(_ touch: UITouch, in view: UIView) { + let loc = touch.location(in: view) + if abs(loc.x - startPoint.x) > Tuning.tapSlop || abs(loc.y - startPoint.y) > Tuning.tapSlop { + moved = true + } + guard trackpad else { + if let h = hostPoint?(loc) { // pointer mode: the cursor follows the finger + send?(.mouseMoveAbs(x: h.x, y: h.y, surfaceWidth: h.w, surfaceHeight: h.h)) + } + return + } + // Relative: move by the finger delta × (sensitivity × acceleration), carrying the + // sub-pixel remainder. Re-anchor (zero delta this frame) if the tracked finger + // changed, so lifting one of several fingers never jumps the cursor. + let key = ObjectIdentifier(touch) + if key != trackKey { + trackKey = key + prevPoint = loc + prevTime = touch.timestamp + return + } + // Ballistics in physical pixels so the curve matches the Android tuning exactly. + let scale = view.window?.screen.scale ?? view.traitCollection.displayScale + let dx = (loc.x - prevPoint.x) * scale + let dy = (loc.y - prevPoint.y) * scale + let dtMs = max((touch.timestamp - prevTime) * 1000, 1) + prevPoint = loc + prevTime = touch.timestamp + let gain = Tuning.pointerSens * Tuning.accel(forSpeed: hypot(dx, dy) / dtMs) + carryX += dx * gain + carryY += dy * gain + let outX = Int32(carryX) // truncates toward zero → remainder kept with its sign + let outY = Int32(carryY) + if outX != 0 || outY != 0 { + send?(.mouseMove(dx: outX, dy: outY)) + carryX -= CGFloat(outX) + carryY -= CGFloat(outY) + } + } + + /// Three-finger tap toggles the stats overlay — through the shared `hudEnabled` default, + /// which the app's HUD views observe via @AppStorage (so this needs no wiring to them). + private static func toggleHUD() { + let defaults = UserDefaults.standard + let on = defaults.object(forKey: DefaultsKey.hudEnabled) as? Bool ?? true + defaults.set(!on, forKey: DefaultsKey.hudEnabled) + } +} +#endif diff --git a/clients/apple/Sources/PunktfunkKit/Support/DefaultsKeys.swift b/clients/apple/Sources/PunktfunkKit/Support/DefaultsKeys.swift index 59aff1d..7a6ee1c 100644 --- a/clients/apple/Sources/PunktfunkKit/Support/DefaultsKeys.swift +++ b/clients/apple/Sources/PunktfunkKit/Support/DefaultsKeys.swift @@ -41,6 +41,11 @@ public enum DefaultsKey { /// scene and silently falls back to the absolute pointer when it can't (Stage Manager / Slide /// Over). Read by `StreamViewController.prefersPointerLocked`. public static let pointerCapture = "punktfunk.pointerCapture" + /// iPhone/iPad: how touchscreen fingers drive the host — a `TouchInputMode` raw value: + /// "trackpad" (default: relative cursor with tap-click / two-finger-scroll gestures), + /// "pointer" (the cursor jumps to the finger), or "touch" (real multi-touch passthrough). + /// Read live per gesture by `StreamLayerUIView`. + public static let touchMode = "punktfunk.touchMode" /// Experimental: show the host's game library (browsed over the management API). Off by default. public static let libraryEnabled = "punktfunk.libraryEnabled" /// macOS: take the window fullscreen while streaming and restore it on the host list. On by default. diff --git a/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift b/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift index fc57b67..03e1af2 100644 --- a/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift +++ b/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift @@ -339,6 +339,9 @@ public final class StreamViewController: UIViewController { setCaptured(false) inputCapture?.stop() inputCapture = nil + // Release anything the touch-driven mouse still holds (a mid-drag session end) while + // onTouchEvent can still deliver the button-up. + streamView.resetTouchInput() streamView.onTouchEvent = nil streamView.onPointerMoveAbs = nil streamView.onPointerButton = nil @@ -454,7 +457,8 @@ final class StreamLayerUIView: UIView { /// Reads the LIVE negotiated mode in pixels (the touch/pointer coordinate space). var currentHostMode: (() -> CGSize)? - /// Direct fingers / Pencil → wire touch events. + /// Direct fingers / Pencil → wire events: real touches in passthrough mode, or the + /// touch-driven mouse events (`TouchMouse`) in the trackpad/pointer modes. var onTouchEvent: ((PunktfunkInputEvent) -> Void)? /// Indirect pointer (mouse/trackpad with no lock) → absolute cursor moves. var onPointerMoveAbs: ((HostPoint) -> Void)? @@ -468,6 +472,22 @@ final class StreamLayerUIView: UIView { /// GameStream button held per active indirect-pointer touch (one click/drag session); /// released when that touch ends. private var pointerButtons: [ObjectIdentifier: UInt32] = [:] + /// Touch-driven mouse for the trackpad/pointer `TouchInputMode`s (see TouchMouse.swift). + private lazy var touchMouse: TouchMouse = { + let mouse = TouchMouse() + mouse.send = { [weak self] event in self?.onTouchEvent?(event) } + mouse.hostPoint = { [weak self] point in self?.hostPoint(from: point) } + return mouse + }() + /// The finger route latched at gesture start — a Settings change mid-gesture applies to + /// the NEXT touch, so one gesture never splits across input models. + private var fingerRoute: TouchInputMode? + + /// Release anything the touch-driven mouse holds and forget gesture state — session stop. + func resetTouchInput() { + touchMouse.reset() + fingerRoute = nil + } #endif override init(frame: CGRect) { @@ -504,10 +524,10 @@ final class StreamLayerUIView: UIView { route(touches, event: event, kind: .up) } override func touchesCancelled(_ touches: Set, with event: UIEvent?) { - route(touches, event: event, kind: .up) + route(touches, event: event, kind: .cancel) } - private enum TouchKind { case down, move, up } + private enum TouchKind { case down, move, up, cancel } /// Split a touch batch by kind: an INDIRECT POINTER (mouse/trackpad with no lock) drives /// the host cursor as an absolute mouse; everything else (direct finger, Pencil) is a host @@ -521,7 +541,28 @@ final class StreamLayerUIView: UIView { fingers.insert(touch) } } - if !fingers.isEmpty { forwardTouches(fingers, kind: kind) } + if !fingers.isEmpty { forwardFingers(fingers, kind: kind) } + } + + /// Route direct fingers by the touch-input model, latched for the whole gesture: + /// passthrough → real wire touches; trackpad/pointer → the TouchMouse gesture engine. + private func forwardFingers(_ touches: Set, kind: TouchKind) { + let mode = fingerRoute ?? TouchInputMode.current + fingerRoute = mode + switch mode { + case .touch: + // A cancellation lifts the wire touch like a normal up — the host just sees the + // contact end. + forwardTouches(touches, kind: kind == .cancel ? .up : kind) + case .trackpad, .pointer: + switch kind { + case .down: touchMouse.began(touches, in: self, trackpad: mode == .trackpad) + case .move: touchMouse.moved(touches, in: self) + case .up: touchMouse.ended(touches, in: self) + case .cancel: touchMouse.cancelled(touches) + } + } + if touchIDs.isEmpty, touchMouse.isIdle { fingerRoute = nil } } /// An indirect-pointer touch is a button-held click/drag session: forward its position as @@ -537,7 +578,7 @@ final class StreamLayerUIView: UIView { onPointerButton?(button, true) case .move: if let host { onPointerMoveAbs?(host) } - case .up: + case .up, .cancel: if let host { onPointerMoveAbs?(host) } if let button = pointerButtons.removeValue(forKey: key) { onPointerButton?(button, false) @@ -554,7 +595,7 @@ final class StreamLayerUIView: UIView { case .down: id = nextFreeID() touchIDs[key] = id - case .move, .up: + case .move, .up, .cancel: guard let known = touchIDs[key] else { continue } id = known } diff --git a/clients/apple/Tests/PunktfunkKitTests/TouchMouseTests.swift b/clients/apple/Tests/PunktfunkKitTests/TouchMouseTests.swift new file mode 100644 index 0000000..fb23d63 --- /dev/null +++ b/clients/apple/Tests/PunktfunkKitTests/TouchMouseTests.swift @@ -0,0 +1,42 @@ +#if os(iOS) +import XCTest + +@testable import PunktfunkKit + +/// Pins the touch-mouse tuning contract (ported 1:1 from the Android client's TouchInput.kt +/// so the two touch clients feel identical) and the mode parsing. The gesture state machine +/// itself needs UITouch instances and is validated on-glass. +final class TouchMouseTests: XCTestCase { + func testModeParsingDefaultsToTrackpad() { + XCTAssertEqual(TouchInputMode(rawValue: "trackpad"), .trackpad) + XCTAssertEqual(TouchInputMode(rawValue: "pointer"), .pointer) + XCTAssertEqual(TouchInputMode(rawValue: "touch"), .touch) + // Unknown/unset values must fall back to trackpad — never crash or go touch-silent. + XCTAssertNil(TouchInputMode(rawValue: "bogus")) + } + + func testAccelerationCurve() { + // At or below the speed floor: no acceleration — slow drags stay precise. + XCTAssertEqual(TouchMouse.Tuning.accel(forSpeed: 0), 1) + XCTAssertEqual(TouchMouse.Tuning.accel(forSpeed: TouchMouse.Tuning.accelSpeedFloor), 1) + // Above the floor the gain ramps... + let mid = TouchMouse.Tuning.accel(forSpeed: 1.0) + XCTAssertGreaterThan(mid, 1) + XCTAssertLessThan(mid, TouchMouse.Tuning.accelMax) + // ...and a flick is capped so it can't fling the cursor uncontrollably. + XCTAssertEqual(TouchMouse.Tuning.accel(forSpeed: 100), TouchMouse.Tuning.accelMax) + // Monotonic in between. + XCTAssertLessThanOrEqual( + TouchMouse.Tuning.accel(forSpeed: 0.5), TouchMouse.Tuning.accel(forSpeed: 1.5)) + } + + func testTuningRelations() { + // The tap-drag window must be long enough to hit but short enough not to turn every + // second tap into a drag. + XCTAssertGreaterThan(TouchMouse.Tuning.tapDragWindow, 0.1) + XCTAssertLessThan(TouchMouse.Tuning.tapDragWindow, 0.5) + // A wheel notch per ~10 pt of two-finger pan (the indirect-trackpad path's feel). + XCTAssertGreaterThan(TouchMouse.Tuning.scrollNotchPt, 0) + } +} +#endif