diff --git a/clients/apple/Sources/PunktfunkKit/InputCapture.swift b/clients/apple/Sources/PunktfunkKit/InputCapture.swift index 1b6e663..995dd47 100644 --- a/clients/apple/Sources/PunktfunkKit/InputCapture.swift +++ b/clients/apple/Sources/PunktfunkKit/InputCapture.swift @@ -1,10 +1,18 @@ -// Input capture → punktfunk/1 datagrams, via the GameController framework. +// Input capture → punktfunk/1 datagrams. // -// GCMouse delivers RAW deltas (not the accelerated cursor) — exactly what the host-side -// injector expects for relative motion. GCKeyboard gives HID keycodes which we map to the -// Windows VK space the host's vk_to_evdev table consumes (same space Moonlight uses). -// Gamepads (GCController) come later — the host's uinput pads already speak the -// GamepadButton/GamepadAxis event kinds, but m3's injector path doesn't route them yet. +// Mouse MOTION and BUTTONS take different paths per platform. On macOS GCMouse's +// mouseMovedHandler/pressedChangedHandler proved unreliable in the field (delivered +// nothing on a live Mac while GCKeyboard worked — a documented GameController quirk), so +// macOS drives motion + buttons from NSEvent under cursor disassociation instead (fed by +// StreamLayerView, the same channel that already carries scroll), and the GCMouse motion/ +// button handlers are not installed there. NSEvent deltas under +// CGAssociateMouseAndMouseCursorPosition(false) are the relative motion — OS-acceleration- +// applied (not raw HID), which is exactly what Moonlight's macOS client ships and is fine. +// iOS keeps the GCMouse path (raw deltas under pointer lock). GCKeyboard (both platforms) +// gives HID keycodes which we map to the Windows VK space the host's vk_to_evdev table +// consumes (same space Moonlight uses). Gamepads (GCController) come later — the host's +// uinput pads already speak the GamepadButton/GamepadAxis event kinds, but m3's injector +// path doesn't route them yet. // // The wire carries integer deltas; GC hands us Floats. We accumulate the fractional // remainder per axis so slow, sub-pixel motion isn't truncated away. @@ -32,6 +40,14 @@ import UIKit import Foundation import GameController import PunktfunkCore +import os + +/// Diagnostic logging for the input path. Off by default (input is high-rate); set +/// PUNKTFUNK_INPUT_DEBUG=1 in the environment to surface whether relative motion + buttons +/// are actually being SENT to the host without needing host-side logs. Motion is throttled +/// to once per second (see `motionDebugTick`); buttons log every transition. +private let inputLog = Logger(subsystem: "io.unom.punktfunk", category: "input") +private let inputDebug = ProcessInfo.processInfo.environment["PUNKTFUNK_INPUT_DEBUG"] == "1" public final class InputCapture { private static weak var activeCapture: InputCapture? @@ -55,6 +71,11 @@ public final class InputCapture { /// it at the HID layer regardless, so its press AND release are dropped here. private var suppressedButton: UInt32? + /// Throttle for the PUNKTFUNK_INPUT_DEBUG motion counter (motion is high-rate — we log + /// a rolling count + the last delta once per second, never per event). Main-queue only. + private var motionDebugCount = 0 + private var motionDebugTick = Date.distantPast + /// One-shot twin of `suppressedButton` for the ⌘⎋ toggle: the physical Esc also /// reaches GCKeyboard, racing the NSEvent monitor — latched here so it can't type /// an Escape into the host in either toggle direction. @@ -122,6 +143,16 @@ public final class InputCapture { ) { [weak self] n in if let m = n.object as? GCMouse { self?.attach(mouse: m) } }) + #if os(iOS) + // The mouse can become the *current* one after it connected (and after our start() + // already ran) — re-attach on that too so a launch-time race doesn't leave the iOS + // GCMouse path without handlers. attach() is idempotent (dedupes by identity). + observers.append(NotificationCenter.default.addObserver( + forName: .GCMouseDidBecomeCurrent, object: nil, queue: .main + ) { [weak self] n in + if let m = n.object as? GCMouse { self?.attach(mouse: m) } + }) + #endif observers.append(NotificationCenter.default.addObserver( forName: .GCKeyboardDidConnect, object: nil, queue: .main ) { [weak self] n in @@ -215,6 +246,10 @@ public final class InputCapture { guard forwarding else { return } if button == suppressedButton { if !pressed { suppressedButton = nil } // capture click over — stop suppressing + if inputDebug { + inputLog.debug( + "button \(button, privacy: .public) \(pressed ? "down" : "up", privacy: .public) SUPPRESSED (engage click)") + } return } if pressed { @@ -222,14 +257,31 @@ public final class InputCapture { } else { pressedButtons.remove(button) } + if inputDebug { + inputLog.debug( + "button \(button, privacy: .public) \(pressed ? "down" : "up", privacy: .public) sent") + } connection.send(.mouseButton(button, down: pressed)) } + /// NSEvent button path (macOS): StreamLayerView's local mouse monitor routes physical + /// button transitions here so they go through the same `suppressedButton` engage-click + /// latch and `pressedButtons` release-on-blur set as the (iOS) GCMouse path. Wire ids: + /// 1=left 2=middle 3=right 4=X1 5=X2. + public func sendMouseButton(_ button: UInt32, pressed: Bool) { + sendButton(button, pressed: pressed) + } + private func attach(mouse: GCMouse) { guard let input = mouse.mouseInput, !mice.contains(where: { $0 === mouse }) // re-delivered on wake — attach once else { return } mice.append(mouse) + // macOS drives motion + buttons from NSEvent (StreamLayerView's local monitor → + // sendMotion/sendMouseButton) because GCMouse's handlers proved unreliable there; + // installing them too would double-send. iOS keeps GCMouse (raw deltas under + // pointer lock). See the file header. + #if !os(macOS) input.mouseMovedHandler = { [weak self] _, dx, dy in guard let self, self.forwarding else { return } // GC gives +y up; the host expects screen-space (+y down). @@ -241,6 +293,16 @@ public final class InputCapture { self.residualY = fy - iy if ix != 0 || iy != 0 { self.connection.send(.mouseMove(dx: Int32(ix), dy: Int32(iy))) + if inputDebug { + self.motionDebugCount += 1 + let now = Date() + if now.timeIntervalSince(self.motionDebugTick) >= 1 { + inputLog.debug( + "motion forwarded: \(self.motionDebugCount, privacy: .public) events, last dx \(Int(ix), privacy: .public) dy \(Int(iy), privacy: .public)") + self.motionDebugCount = 0 + self.motionDebugTick = now + } + } } } input.leftButton.pressedChangedHandler = { [weak self] _, _, pressed in @@ -260,12 +322,42 @@ public final class InputCapture { } } } + #endif // NOTE: no scroll handler here. GCMouse's scroll dpad only fires for plain HID // wheel deltas — trackpad/Magic Mouse scrolling is gesture-based and never // reaches GameController. Scroll arrives via the stream view's scrollWheel // override (NSEvent covers wheels too) → sendScroll(). } + /// Forward relative mouse motion (macOS). Fed by StreamLayerView's NSEvent monitor — + /// while captured the cursor is disassociated (CGAssociateMouseAndMouseCursorPosition + /// (false)), so mouseMoved/dragged deltaX/deltaY ARE the relative motion, the same + /// channel sendScroll already uses. Unlike the (iOS) GCMouse path this is NOT y-negated: + /// NSEvent deltaY is already screen-space (+y down), which is what the host expects. + /// Fractional remainders accumulate so slow, sub-pixel motion isn't truncated away. + public func sendMotion(dx: Float, dy: Float) { + guard forwarding else { return } + let fx = dx + residualX + let fy = dy + residualY + let ix = fx.rounded(.towardZero) + let iy = fy.rounded(.towardZero) + residualX = fx - ix + residualY = fy - iy + guard ix != 0 || iy != 0 else { return } + connection.send(.mouseMove(dx: Int32(ix), dy: Int32(iy))) + if inputDebug { + // High-rate — log a rolling count + the last delta once per second, not per event. + motionDebugCount += 1 + let now = Date() + if now.timeIntervalSince(motionDebugTick) >= 1 { + inputLog.debug( + "motion forwarded: \(self.motionDebugCount, privacy: .public) events, last dx \(Int(ix), privacy: .public) dy \(Int(iy), privacy: .public)") + motionDebugCount = 0 + motionDebugTick = now + } + } + } + /// Forward a scroll gesture, WHEEL_DELTA(120)-scaled (positive = up / right, /// Moonlight's convention). Fed by StreamLayerView.scrollWheel — the only delivery /// path that covers trackpad/Magic Mouse gestures (GCMouse never reports them). diff --git a/clients/apple/Sources/PunktfunkKit/StreamView.swift b/clients/apple/Sources/PunktfunkKit/StreamView.swift index 6ef1324..58c5c1e 100644 --- a/clients/apple/Sources/PunktfunkKit/StreamView.swift +++ b/clients/apple/Sources/PunktfunkKit/StreamView.swift @@ -19,13 +19,22 @@ import AppKit import AVFoundation import SwiftUI +import os + +/// Same diagnostic switch as InputCapture: PUNKTFUNK_INPUT_DEBUG=1 logs when the macOS +/// NSEvent mouse monitor (relative motion + buttons) is installed/removed, so the user can +/// confirm the new motion path is actually live for a session. +private let streamInputLog = Logger(subsystem: "io.unom.punktfunk", category: "input") +private let streamInputDebug = + ProcessInfo.processInfo.environment["PUNKTFUNK_INPUT_DEBUG"] == "1" /// Hides the LOCAL cursor while captured. The host renders its own cursor, and the local -/// one both diverges from it (the host applies acceleration/clamping to our raw deltas) -/// and can wander out of the window — a click there would focus another app. So while -/// captured we do what Moonlight does: warp the cursor into the view, freeze it -/// (`CGAssociateMouseAndMouseCursorPosition(false)` — GCMouse still delivers raw HID -/// deltas), and hide it. hide/unhide and associate are balanced via `captured`. +/// one both diverges from it (the host applies acceleration/clamping to our deltas) and +/// can wander out of the window — a click there would focus another app. So while captured +/// we do what Moonlight does: warp the cursor into the view, freeze it +/// (`CGAssociateMouseAndMouseCursorPosition(false)` — under which NSEvent mouseMoved/ +/// dragged deltas become the relative motion StreamLayerView forwards), and hide it. +/// hide/unhide and associate are balanced via `captured`. private final class CursorCapture { private var captured = false @@ -106,6 +115,10 @@ public final class StreamLayerView: NSView { private var inputCapture: InputCapture? private var appObservers: [NSObjectProtocol] = [] private var windowObservers: [NSObjectProtocol] = [] + /// Local NSEvent monitor carrying relative mouse MOTION + BUTTONS to the host while + /// captured (GCMouse's own delivery proved unreliable on macOS — see InputCapture). + /// Installed on engage, removed on release; nil while not captured. + private var mouseEventMonitor: Any? /// Whether input capture is currently engaged (cursor hidden+frozen, mouse/keyboard /// forwarded). Main-thread only. @@ -249,6 +262,10 @@ public final class StreamLayerView: NSView { else { return } cursorCapture.capture(in: self) inputCapture?.setForwarding(true, suppressClick: fromClick) + // Install AFTER the warp + setForwarding: the engage warp generates no forwarded + // delta (the monitor isn't up yet), and the engage click's suppression latch is + // already armed, so the monitor only ever sees genuine post-engage input. + installMouseMonitor() captured = true window?.makeFirstResponder(self) notifyCaptureChange(true) @@ -256,12 +273,66 @@ public final class StreamLayerView: NSView { private func releaseCapture() { guard captured else { return } + removeMouseMonitor() cursorCapture.release() inputCapture?.setForwarding(false) captured = false notifyCaptureChange(false) } + /// A single local monitor for motion + buttons, installed only while captured. A local + /// monitor is more robust than view overrides for relative motion: it sidesteps the + /// `window.acceptsMouseMovedEvents`/tracking-area/responder-chain requirements, and + /// since the cursor is frozen mid-view while captured every such event belongs here. + /// ALL four motion types are covered so motion keeps flowing during a button-held drag, + /// not just `.mouseMoved`. NSEvent deltas under disassociation are OS-pointer- + /// acceleration-applied (not raw HID) — what Moonlight's macOS client ships; if the + /// host re-accelerates there's mild double-acceleration, acceptable and fixable later + /// via IOHID. Events are returned (not swallowed): the cursor is frozen, so they're + /// inert locally. + private func installMouseMonitor() { + guard mouseEventMonitor == nil else { return } + mouseEventMonitor = NSEvent.addLocalMonitorForEvents(matching: [ + .mouseMoved, .leftMouseDragged, .rightMouseDragged, .otherMouseDragged, + .leftMouseDown, .leftMouseUp, .rightMouseDown, .rightMouseUp, + .otherMouseDown, .otherMouseUp, + ]) { [weak self] event in + guard let self, self.captured, let ic = self.inputCapture else { return event } + switch event.type { + case .mouseMoved, .leftMouseDragged, .rightMouseDragged, .otherMouseDragged: + ic.sendMotion(dx: Float(event.deltaX), dy: Float(event.deltaY)) // no y-negation + case .leftMouseDown: ic.sendMouseButton(1, pressed: true) + case .leftMouseUp: ic.sendMouseButton(1, pressed: false) + case .rightMouseDown: ic.sendMouseButton(3, pressed: true) + case .rightMouseUp: ic.sendMouseButton(3, pressed: false) + case .otherMouseDown: ic.sendMouseButton(self.wireButton(for: event), pressed: true) + case .otherMouseUp: ic.sendMouseButton(self.wireButton(for: event), pressed: false) + default: break + } + return event + } + if streamInputDebug { streamInputLog.debug("mouse NSEvent monitor installed (capture engaged)") } + } + + private func removeMouseMonitor() { + if let monitor = mouseEventMonitor { + NSEvent.removeMonitor(monitor) + mouseEventMonitor = nil + if streamInputDebug { streamInputLog.debug("mouse NSEvent monitor removed (capture released)") } + } + } + + /// NSEvent `buttonNumber` → GameStream wire id for the "other" buttons: 2 = middle, + /// 3 = first side (X1), 4 = second side (X2). Unknown extras fall back to middle. + private func wireButton(for event: NSEvent) -> UInt32 { + switch event.buttonNumber { + case 2: return 2 // middle + case 3: return 4 // X1 + case 4: return 5 // X2 + default: return 2 + } + } + /// Engage/release can run inside a SwiftUI update pass (captureEnabled flips in /// updateNSView; release in dismantleNSView) — publishing model state synchronously /// there is undefined behavior, so the callback is deferred a runloop turn. @@ -315,6 +386,7 @@ public final class StreamLayerView: NSView { /// whoever owns it (PunktfunkConnection.close() is safe alongside a draining pump). public func stop() { releaseCapture() + removeMouseMonitor() // belt-and-suspenders: releaseCapture no-ops if not captured inputCapture?.stop() inputCapture = nil pump?.stop() @@ -323,6 +395,7 @@ public final class StreamLayerView: NSView { } deinit { + removeMouseMonitor() appObservers.forEach(NotificationCenter.default.removeObserver(_:)) windowObservers.forEach(NotificationCenter.default.removeObserver(_:)) pump?.stop() diff --git a/clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift b/clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift index 6188506..6db5b6c 100644 --- a/clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift +++ b/clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift @@ -21,6 +21,14 @@ import GameController import PunktfunkCore import SwiftUI import UIKit +import os + +/// Same diagnostic switch as InputCapture (PUNKTFUNK_INPUT_DEBUG=1): on iOS we log the +/// resolved pointer-lock state each time capture engages, so the user can see whether the +/// scene actually locked (GCMouse only delivers deltas while it did) or whether we're on +/// the touch fallback. +private let iosInputLog = Logger(subsystem: "io.unom.punktfunk", category: "input") +private let iosInputDebug = ProcessInfo.processInfo.environment["PUNKTFUNK_INPUT_DEBUG"] == "1" public struct StreamView: UIViewControllerRepresentable { private let connection: PunktfunkConnection @@ -76,6 +84,17 @@ public final class StreamViewController: UIViewController { private var pointerInteraction: UIPointerInteraction? #endif + /// Reads whether the scene's pointer is actually locked right now; nil = state + /// unavailable (no scene yet, or pre-availability). Only while this is true does GCMouse + /// deliver relative deltas — otherwise the touch path carries input. + private func pointerLockEngaged() -> Bool? { + #if os(iOS) + return view.window?.windowScene?.pointerLockState?.isLocked + #else + return nil + #endif + } + var onCaptureChange: ((Bool) -> Void)? var captureEnabled = true { @@ -96,9 +115,9 @@ public final class StreamViewController: UIViewController { view = StreamLayerUIView() #if os(iOS) // Hide the iPadOS cursor while it hovers the video: the host renders its own - // cursor from our raw deltas, so the local one only diverges from it. (True - // pointer LOCK — prefersPointerLocked — isn't consulted through - // UIHostingController; this hides the pointer without locking it.) + // cursor from our deltas, so the local one only diverges from it. This hides the + // pointer; true pointer LOCK (below) is what makes GCMouse deliver relative deltas + // — and the system only grants it on a full-screen, frontmost iPad scene. let interaction = UIPointerInteraction(delegate: self) view.addInteraction(interaction) pointerInteraction = interaction @@ -106,8 +125,19 @@ public final class StreamViewController: UIViewController { } #if os(iOS) - public override var prefersPointerLocked: Bool { captured } + // Pointer lock is only meaningful on iPad (iPhone has no hardware-pointer lock) and + // only when capture is engaged. The system additionally requires full-screen + frontmost + // and may drop it (Slide Over/Stage Manager/backgrounding) — verified in setCaptured(). + public override var prefersPointerLocked: Bool { + captured && UIDevice.current.userInterfaceIdiom == .pad + } public override var prefersHomeIndicatorAutoHidden: Bool { true } + + // If SwiftUI's UIHostingController reparents us, a plain container parent that forwards + // its pointer-lock decision to its children will then reach this VC. (UIHostingController + // itself does not consult children, which is why GCMouse deltas can never arrive there — + // the touch path, always forwarded, is the unconditional fallback.) + public override var childViewControllerForPointerLock: UIViewController? { self } #endif func start( @@ -157,6 +187,16 @@ public final class StreamViewController: UIViewController { ) { [weak self] _ in self?.setCaptured(false) }) + // The system can drop the lock without us asking (Slide Over, Stage Manager, leaving + // foregroundActive). Surface it so the user sees, in PUNKTFUNK_INPUT_DEBUG, when + // GCMouse delivery has silently stopped and we've fallen back to touch. + observers.append(NotificationCenter.default.addObserver( + forName: UIPointerLockState.didChangeNotification, object: nil, queue: .main + ) { [weak self] _ in + guard let self, iosInputDebug else { return } + let locked = self.pointerLockEngaged().map(String.init(describing:)) ?? "unavailable" + iosInputLog.debug("pointer lock changed: isLocked=\(locked, privacy: .public)") + }) if captureEnabled { setCaptured(true) // entering a session is the deliberate "capture me" moment @@ -194,7 +234,16 @@ public final class StreamViewController: UIViewController { pointerInteraction?.invalidate() // re-resolve the hidden/visible pointer style let onCaptureChange = onCaptureChange let captured = captured - DispatchQueue.main.async { onCaptureChange?(captured) } + DispatchQueue.main.async { [weak self] in + onCaptureChange?(captured) + // The lock request is async — read the resolved state next turn. If it didn't + // engage, GCMouse won't deliver and the always-on touch path carries input. + if iosInputDebug, let self { + let locked = self.pointerLockEngaged().map(String.init(describing:)) ?? "unavailable" + iosInputLog.debug( + "setCaptured(\(captured, privacy: .public)) → pointer lock isLocked=\(locked, privacy: .public)") + } + } } #endif