fix(apple): drive macOS mouse motion/buttons from NSEvent; fix iPad pointer lock
Host-side logs proved the macOS client sent keyboard + scroll but ZERO relative mouse-motion and ZERO button events for an entire session — the user was moving the mouse the whole time. Root cause is client-side: GCMouse's mouseMovedHandler/pressedChangedHandler silently never fired on the live Mac (a documented GameController quirk) while GCKeyboard worked and scroll already rode NSEvent. So motion/buttons were the only input on a GCMouse-only path, and that path was dead. macOS: stop relying on GCMouse for motion/buttons (compiled out with #if !os(macOS)); drive them from a local NSEvent monitor installed only while captured — the same channel scrollWheel already uses successfully. Under CGAssociateMouseAndMouseCursorPosition(false) the mouseMoved/dragged deltaX/deltaY ARE the relative motion (OS-acceleration-applied, exactly what Moonlight's macOS client ships). All four motion event types are covered so motion keeps flowing during a button-held drag; buttons map left/right/middle/X1/X2 through the existing engage-click-suppression + release-on-blur logic. NSEvent deltaY is already screen-space (+y down) so, unlike the GCMouse path, it is NOT negated. iPad: the input failure there was a different cause — GCMouse only delivers relative deltas while the scene holds a true pointer LOCK, which the system grants only to a full-screen, frontmost iPad scene and which UIHostingController doesn't consult for children. Gate prefersPointerLocked to iPad + captured, add childViewControllerForPointerLock so a reparenting container forwards the lock decision to this VC, and log the resolved lock state. Touch remains the unconditional fallback. Adds a PUNKTFUNK_INPUT_DEBUG=1 switch (os.Logger, throttled) so motion/buttons being SENT is verifiable on-device without host-side logs. iOS GCMouse path otherwise unchanged; GCKeyboard unchanged on both. Researched + adversarially reviewed; Swift builds only on a Mac, so this is unverified-compiled here. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -19,13 +19,22 @@
|
||||
import AppKit
|
||||
import AVFoundation
|
||||
import SwiftUI
|
||||
import os
|
||||
|
||||
/// Same diagnostic switch as InputCapture: PUNKTFUNK_INPUT_DEBUG=1 logs when the macOS
|
||||
/// NSEvent mouse monitor (relative motion + buttons) is installed/removed, so the user can
|
||||
/// confirm the new motion path is actually live for a session.
|
||||
private let streamInputLog = Logger(subsystem: "io.unom.punktfunk", category: "input")
|
||||
private let streamInputDebug =
|
||||
ProcessInfo.processInfo.environment["PUNKTFUNK_INPUT_DEBUG"] == "1"
|
||||
|
||||
/// Hides the LOCAL cursor while captured. The host renders its own cursor, and the local
|
||||
/// one both diverges from it (the host applies acceleration/clamping to our raw deltas)
|
||||
/// and can wander out of the window — a click there would focus another app. So while
|
||||
/// captured we do what Moonlight does: warp the cursor into the view, freeze it
|
||||
/// (`CGAssociateMouseAndMouseCursorPosition(false)` — GCMouse still delivers raw HID
|
||||
/// deltas), and hide it. hide/unhide and associate are balanced via `captured`.
|
||||
/// one both diverges from it (the host applies acceleration/clamping to our deltas) and
|
||||
/// can wander out of the window — a click there would focus another app. So while captured
|
||||
/// we do what Moonlight does: warp the cursor into the view, freeze it
|
||||
/// (`CGAssociateMouseAndMouseCursorPosition(false)` — under which NSEvent mouseMoved/
|
||||
/// dragged deltas become the relative motion StreamLayerView forwards), and hide it.
|
||||
/// hide/unhide and associate are balanced via `captured`.
|
||||
private final class CursorCapture {
|
||||
private var captured = false
|
||||
|
||||
@@ -106,6 +115,10 @@ public final class StreamLayerView: NSView {
|
||||
private var inputCapture: InputCapture?
|
||||
private var appObservers: [NSObjectProtocol] = []
|
||||
private var windowObservers: [NSObjectProtocol] = []
|
||||
/// Local NSEvent monitor carrying relative mouse MOTION + BUTTONS to the host while
|
||||
/// captured (GCMouse's own delivery proved unreliable on macOS — see InputCapture).
|
||||
/// Installed on engage, removed on release; nil while not captured.
|
||||
private var mouseEventMonitor: Any?
|
||||
|
||||
/// Whether input capture is currently engaged (cursor hidden+frozen, mouse/keyboard
|
||||
/// forwarded). Main-thread only.
|
||||
@@ -249,6 +262,10 @@ public final class StreamLayerView: NSView {
|
||||
else { return }
|
||||
cursorCapture.capture(in: self)
|
||||
inputCapture?.setForwarding(true, suppressClick: fromClick)
|
||||
// Install AFTER the warp + setForwarding: the engage warp generates no forwarded
|
||||
// delta (the monitor isn't up yet), and the engage click's suppression latch is
|
||||
// already armed, so the monitor only ever sees genuine post-engage input.
|
||||
installMouseMonitor()
|
||||
captured = true
|
||||
window?.makeFirstResponder(self)
|
||||
notifyCaptureChange(true)
|
||||
@@ -256,12 +273,66 @@ public final class StreamLayerView: NSView {
|
||||
|
||||
private func releaseCapture() {
|
||||
guard captured else { return }
|
||||
removeMouseMonitor()
|
||||
cursorCapture.release()
|
||||
inputCapture?.setForwarding(false)
|
||||
captured = false
|
||||
notifyCaptureChange(false)
|
||||
}
|
||||
|
||||
/// A single local monitor for motion + buttons, installed only while captured. A local
|
||||
/// monitor is more robust than view overrides for relative motion: it sidesteps the
|
||||
/// `window.acceptsMouseMovedEvents`/tracking-area/responder-chain requirements, and
|
||||
/// since the cursor is frozen mid-view while captured every such event belongs here.
|
||||
/// ALL four motion types are covered so motion keeps flowing during a button-held drag,
|
||||
/// not just `.mouseMoved`. NSEvent deltas under disassociation are OS-pointer-
|
||||
/// acceleration-applied (not raw HID) — what Moonlight's macOS client ships; if the
|
||||
/// host re-accelerates there's mild double-acceleration, acceptable and fixable later
|
||||
/// via IOHID. Events are returned (not swallowed): the cursor is frozen, so they're
|
||||
/// inert locally.
|
||||
private func installMouseMonitor() {
|
||||
guard mouseEventMonitor == nil else { return }
|
||||
mouseEventMonitor = NSEvent.addLocalMonitorForEvents(matching: [
|
||||
.mouseMoved, .leftMouseDragged, .rightMouseDragged, .otherMouseDragged,
|
||||
.leftMouseDown, .leftMouseUp, .rightMouseDown, .rightMouseUp,
|
||||
.otherMouseDown, .otherMouseUp,
|
||||
]) { [weak self] event in
|
||||
guard let self, self.captured, let ic = self.inputCapture else { return event }
|
||||
switch event.type {
|
||||
case .mouseMoved, .leftMouseDragged, .rightMouseDragged, .otherMouseDragged:
|
||||
ic.sendMotion(dx: Float(event.deltaX), dy: Float(event.deltaY)) // no y-negation
|
||||
case .leftMouseDown: ic.sendMouseButton(1, pressed: true)
|
||||
case .leftMouseUp: ic.sendMouseButton(1, pressed: false)
|
||||
case .rightMouseDown: ic.sendMouseButton(3, pressed: true)
|
||||
case .rightMouseUp: ic.sendMouseButton(3, pressed: false)
|
||||
case .otherMouseDown: ic.sendMouseButton(self.wireButton(for: event), pressed: true)
|
||||
case .otherMouseUp: ic.sendMouseButton(self.wireButton(for: event), pressed: false)
|
||||
default: break
|
||||
}
|
||||
return event
|
||||
}
|
||||
if streamInputDebug { streamInputLog.debug("mouse NSEvent monitor installed (capture engaged)") }
|
||||
}
|
||||
|
||||
private func removeMouseMonitor() {
|
||||
if let monitor = mouseEventMonitor {
|
||||
NSEvent.removeMonitor(monitor)
|
||||
mouseEventMonitor = nil
|
||||
if streamInputDebug { streamInputLog.debug("mouse NSEvent monitor removed (capture released)") }
|
||||
}
|
||||
}
|
||||
|
||||
/// NSEvent `buttonNumber` → GameStream wire id for the "other" buttons: 2 = middle,
|
||||
/// 3 = first side (X1), 4 = second side (X2). Unknown extras fall back to middle.
|
||||
private func wireButton(for event: NSEvent) -> UInt32 {
|
||||
switch event.buttonNumber {
|
||||
case 2: return 2 // middle
|
||||
case 3: return 4 // X1
|
||||
case 4: return 5 // X2
|
||||
default: return 2
|
||||
}
|
||||
}
|
||||
|
||||
/// Engage/release can run inside a SwiftUI update pass (captureEnabled flips in
|
||||
/// updateNSView; release in dismantleNSView) — publishing model state synchronously
|
||||
/// there is undefined behavior, so the callback is deferred a runloop turn.
|
||||
@@ -315,6 +386,7 @@ public final class StreamLayerView: NSView {
|
||||
/// whoever owns it (PunktfunkConnection.close() is safe alongside a draining pump).
|
||||
public func stop() {
|
||||
releaseCapture()
|
||||
removeMouseMonitor() // belt-and-suspenders: releaseCapture no-ops if not captured
|
||||
inputCapture?.stop()
|
||||
inputCapture = nil
|
||||
pump?.stop()
|
||||
@@ -323,6 +395,7 @@ public final class StreamLayerView: NSView {
|
||||
}
|
||||
|
||||
deinit {
|
||||
removeMouseMonitor()
|
||||
appObservers.forEach(NotificationCenter.default.removeObserver(_:))
|
||||
windowObservers.forEach(NotificationCenter.default.removeObserver(_:))
|
||||
pump?.stop()
|
||||
|
||||
Reference in New Issue
Block a user