fix(apple): drive macOS mouse motion/buttons from NSEvent; fix iPad pointer lock
Host-side logs proved the macOS client sent keyboard + scroll but ZERO relative mouse-motion and ZERO button events for an entire session — the user was moving the mouse the whole time. Root cause is client-side: GCMouse's mouseMovedHandler/pressedChangedHandler silently never fired on the live Mac (a documented GameController quirk) while GCKeyboard worked and scroll already rode NSEvent. So motion/buttons were the only input on a GCMouse-only path, and that path was dead. macOS: stop relying on GCMouse for motion/buttons (compiled out with #if !os(macOS)); drive them from a local NSEvent monitor installed only while captured — the same channel scrollWheel already uses successfully. Under CGAssociateMouseAndMouseCursorPosition(false) the mouseMoved/dragged deltaX/deltaY ARE the relative motion (OS-acceleration-applied, exactly what Moonlight's macOS client ships). All four motion event types are covered so motion keeps flowing during a button-held drag; buttons map left/right/middle/X1/X2 through the existing engage-click-suppression + release-on-blur logic. NSEvent deltaY is already screen-space (+y down) so, unlike the GCMouse path, it is NOT negated. iPad: the input failure there was a different cause — GCMouse only delivers relative deltas while the scene holds a true pointer LOCK, which the system grants only to a full-screen, frontmost iPad scene and which UIHostingController doesn't consult for children. Gate prefersPointerLocked to iPad + captured, add childViewControllerForPointerLock so a reparenting container forwards the lock decision to this VC, and log the resolved lock state. Touch remains the unconditional fallback. Adds a PUNKTFUNK_INPUT_DEBUG=1 switch (os.Logger, throttled) so motion/buttons being SENT is verifiable on-device without host-side logs. iOS GCMouse path otherwise unchanged; GCKeyboard unchanged on both. Researched + adversarially reviewed; Swift builds only on a Mac, so this is unverified-compiled here. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -21,6 +21,14 @@ import GameController
|
||||
import PunktfunkCore
|
||||
import SwiftUI
|
||||
import UIKit
|
||||
import os
|
||||
|
||||
/// Same diagnostic switch as InputCapture (PUNKTFUNK_INPUT_DEBUG=1): on iOS we log the
|
||||
/// resolved pointer-lock state each time capture engages, so the user can see whether the
|
||||
/// scene actually locked (GCMouse only delivers deltas while it did) or whether we're on
|
||||
/// the touch fallback.
|
||||
private let iosInputLog = Logger(subsystem: "io.unom.punktfunk", category: "input")
|
||||
private let iosInputDebug = ProcessInfo.processInfo.environment["PUNKTFUNK_INPUT_DEBUG"] == "1"
|
||||
|
||||
public struct StreamView: UIViewControllerRepresentable {
|
||||
private let connection: PunktfunkConnection
|
||||
@@ -76,6 +84,17 @@ public final class StreamViewController: UIViewController {
|
||||
private var pointerInteraction: UIPointerInteraction?
|
||||
#endif
|
||||
|
||||
/// Reads whether the scene's pointer is actually locked right now; nil = state
|
||||
/// unavailable (no scene yet, or pre-availability). Only while this is true does GCMouse
|
||||
/// deliver relative deltas — otherwise the touch path carries input.
|
||||
private func pointerLockEngaged() -> Bool? {
|
||||
#if os(iOS)
|
||||
return view.window?.windowScene?.pointerLockState?.isLocked
|
||||
#else
|
||||
return nil
|
||||
#endif
|
||||
}
|
||||
|
||||
var onCaptureChange: ((Bool) -> Void)?
|
||||
|
||||
var captureEnabled = true {
|
||||
@@ -96,9 +115,9 @@ public final class StreamViewController: UIViewController {
|
||||
view = StreamLayerUIView()
|
||||
#if os(iOS)
|
||||
// Hide the iPadOS cursor while it hovers the video: the host renders its own
|
||||
// cursor from our raw deltas, so the local one only diverges from it. (True
|
||||
// pointer LOCK — prefersPointerLocked — isn't consulted through
|
||||
// UIHostingController; this hides the pointer without locking it.)
|
||||
// cursor from our deltas, so the local one only diverges from it. This hides the
|
||||
// pointer; true pointer LOCK (below) is what makes GCMouse deliver relative deltas
|
||||
// — and the system only grants it on a full-screen, frontmost iPad scene.
|
||||
let interaction = UIPointerInteraction(delegate: self)
|
||||
view.addInteraction(interaction)
|
||||
pointerInteraction = interaction
|
||||
@@ -106,8 +125,19 @@ public final class StreamViewController: UIViewController {
|
||||
}
|
||||
|
||||
#if os(iOS)
|
||||
public override var prefersPointerLocked: Bool { captured }
|
||||
// Pointer lock is only meaningful on iPad (iPhone has no hardware-pointer lock) and
|
||||
// only when capture is engaged. The system additionally requires full-screen + frontmost
|
||||
// and may drop it (Slide Over/Stage Manager/backgrounding) — verified in setCaptured().
|
||||
public override var prefersPointerLocked: Bool {
|
||||
captured && UIDevice.current.userInterfaceIdiom == .pad
|
||||
}
|
||||
public override var prefersHomeIndicatorAutoHidden: Bool { true }
|
||||
|
||||
// If SwiftUI's UIHostingController reparents us, a plain container parent that forwards
|
||||
// its pointer-lock decision to its children will then reach this VC. (UIHostingController
|
||||
// itself does not consult children, which is why GCMouse deltas can never arrive there —
|
||||
// the touch path, always forwarded, is the unconditional fallback.)
|
||||
public override var childViewControllerForPointerLock: UIViewController? { self }
|
||||
#endif
|
||||
|
||||
func start(
|
||||
@@ -157,6 +187,16 @@ public final class StreamViewController: UIViewController {
|
||||
) { [weak self] _ in
|
||||
self?.setCaptured(false)
|
||||
})
|
||||
// The system can drop the lock without us asking (Slide Over, Stage Manager, leaving
|
||||
// foregroundActive). Surface it so the user sees, in PUNKTFUNK_INPUT_DEBUG, when
|
||||
// GCMouse delivery has silently stopped and we've fallen back to touch.
|
||||
observers.append(NotificationCenter.default.addObserver(
|
||||
forName: UIPointerLockState.didChangeNotification, object: nil, queue: .main
|
||||
) { [weak self] _ in
|
||||
guard let self, iosInputDebug else { return }
|
||||
let locked = self.pointerLockEngaged().map(String.init(describing:)) ?? "unavailable"
|
||||
iosInputLog.debug("pointer lock changed: isLocked=\(locked, privacy: .public)")
|
||||
})
|
||||
|
||||
if captureEnabled {
|
||||
setCaptured(true) // entering a session is the deliberate "capture me" moment
|
||||
@@ -194,7 +234,16 @@ public final class StreamViewController: UIViewController {
|
||||
pointerInteraction?.invalidate() // re-resolve the hidden/visible pointer style
|
||||
let onCaptureChange = onCaptureChange
|
||||
let captured = captured
|
||||
DispatchQueue.main.async { onCaptureChange?(captured) }
|
||||
DispatchQueue.main.async { [weak self] in
|
||||
onCaptureChange?(captured)
|
||||
// The lock request is async — read the resolved state next turn. If it didn't
|
||||
// engage, GCMouse won't deliver and the always-on touch path carries input.
|
||||
if iosInputDebug, let self {
|
||||
let locked = self.pointerLockEngaged().map(String.init(describing:)) ?? "unavailable"
|
||||
iosInputLog.debug(
|
||||
"setCaptured(\(captured, privacy: .public)) → pointer lock isLocked=\(locked, privacy: .public)")
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
Reference in New Issue
Block a user