fix(apple/iOS): capture all attached mice; gate UIKit pointer path under lock
apple / swift (push) Successful in 1m6s
ci / web (push) Has been cancelled
android / android (push) Has been cancelled
apple / screenshots (push) Has been cancelled
ci / rust (push) Has been cancelled
deb / build-publish (push) Has been cancelled
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 7s
ci / docs-site (push) Has been cancelled
ci / bench (push) Has been cancelled
decky / build-publish (push) Has been cancelled
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 5s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 4s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 5s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 4s
release / apple (push) Successful in 7m30s
rpm / build-publish (fedora-44, punktfunk-fedora44-rpm) (push) Successful in 9m39s
rpm / build-publish (bazzite, punktfunk-fedora-rpm) (push) Successful in 9m43s
docker / deploy-docs (push) Successful in 18s
apple / swift (push) Successful in 1m6s
ci / web (push) Has been cancelled
android / android (push) Has been cancelled
apple / screenshots (push) Has been cancelled
ci / rust (push) Has been cancelled
deb / build-publish (push) Has been cancelled
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 7s
ci / docs-site (push) Has been cancelled
ci / bench (push) Has been cancelled
decky / build-publish (push) Has been cancelled
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 5s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 4s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 5s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 4s
release / apple (push) Successful in 7m30s
rpm / build-publish (fedora-44, punktfunk-fedora44-rpm) (push) Successful in 9m39s
rpm / build-publish (bazzite, punktfunk-fedora-rpm) (push) Successful in 9m43s
docker / deploy-docs (push) Successful in 18s
The iPad pointer lock engaged but a Magic Keyboard trackpad went dead the moment a second pointer (a Universal Control "V-UC Automouse") was connected — on-device PUNKTFUNK_INPUT_DEBUG logs showed only ONE GCMouse attached (whichever was GCMouse.current), so the other device's motion handler was never installed. InputCapture.start() now attaches a handler to EVERY GCMouse.mice(), not just GCMouse.current, so a trackpad and a second mouse both drive (each GCMouse delivers its own deltas through its own handler). New arrivals still come via the GCMouseDidConnect observer. Also gate the WHOLE UIKit indirect-pointer path (motion, buttons AND scroll) on !gcMouseForwarding, not just motion+scroll: under pointer lock GCMouse owns buttons too, and the trackpad/mouse also emit UIKit indirect-pointer events pinned at the locked position — without the gate a click double-sent (GCMouse + UIKit). The two paths are now exact mirrors on `gcMouseForwarding` (== locked). Removes the investigation-only diagnostics (attachedMiceSummary/hasGCMouse, the per-event UIKit pointer/scroll logs, the GCMouse attach/became-current logs); the pre-existing `pointer lock isLocked=… captured=…` debug line is restored. iOS compiles against the SDK; macOS swift build + test green (49 tests). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -107,23 +107,6 @@ public final class InputCapture {
|
|||||||
/// macOS (no GCMouse handlers installed; `sendMouseAbs` is never called there). Main-queue.
|
/// macOS (no GCMouse handlers installed; `sendMouseAbs` is never called there). Main-queue.
|
||||||
public var gcMouseForwarding = false
|
public var gcMouseForwarding = false
|
||||||
|
|
||||||
#if os(iOS)
|
|
||||||
/// Whether any device is attached as a `GCMouse` right now. The Magic Keyboard TRACKPAD does
|
|
||||||
/// not always register as a GCMouse on iPadOS (only a standalone mouse does) — when no GCMouse
|
|
||||||
/// is present the relative GCMouse path can't carry pointer motion. Main-queue.
|
|
||||||
public var hasGCMouse: Bool { !mice.isEmpty }
|
|
||||||
|
|
||||||
/// Diagnostic: a one-line description of every attached GCMouse (count + GCDevice identity), so
|
|
||||||
/// PUNKTFUNK_INPUT_DEBUG can reveal whether the trackpad showed up as a mouse at all.
|
|
||||||
public var attachedMiceSummary: String {
|
|
||||||
guard !mice.isEmpty else { return "0 mice" }
|
|
||||||
let parts = mice.map { mouse -> String in
|
|
||||||
"\(mouse.productCategory)/\(mouse.vendorName ?? "?")"
|
|
||||||
}
|
|
||||||
return "\(mice.count) mice: \(parts.joined(separator: ", "))"
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/// Fired on ⌘⎋ (the capture toggle — detected here so it works in both states; the
|
/// Fired on ⌘⎋ (the capture toggle — detected here so it works in both states; the
|
||||||
/// event itself is swallowed). Main queue.
|
/// event itself is swallowed). Main queue.
|
||||||
public var onToggleCapture: (() -> Void)?
|
public var onToggleCapture: (() -> Void)?
|
||||||
@@ -177,7 +160,13 @@ public final class InputCapture {
|
|||||||
previous.onPreempted?()
|
previous.onPreempted?()
|
||||||
}
|
}
|
||||||
Self.activeCapture = self
|
Self.activeCapture = self
|
||||||
if let mouse = GCMouse.current { attach(mouse: mouse) }
|
// Attach EVERY connected mouse, not just GCMouse.current. With two pointing devices (e.g.
|
||||||
|
// the iPad's own Magic Keyboard trackpad AND a Universal Control "V-UC Automouse"), only one
|
||||||
|
// is `current` at a time; attaching just that one left the OTHER device's motion handler
|
||||||
|
// uninstalled, so moving it did nothing. Each GCMouse delivers its own deltas through its own
|
||||||
|
// handler, so handling all of them lets either device drive. New arrivals are caught by the
|
||||||
|
// GCMouseDidConnect observer below.
|
||||||
|
for mouse in GCMouse.mice() { attach(mouse: mouse) }
|
||||||
if let keyboard = GCKeyboard.coalesced { attach(keyboard: keyboard) }
|
if let keyboard = GCKeyboard.coalesced { attach(keyboard: keyboard) }
|
||||||
observers.append(NotificationCenter.default.addObserver(
|
observers.append(NotificationCenter.default.addObserver(
|
||||||
forName: .GCMouseDidConnect, object: nil, queue: .main
|
forName: .GCMouseDidConnect, object: nil, queue: .main
|
||||||
@@ -411,12 +400,6 @@ public final class InputCapture {
|
|||||||
!mice.contains(where: { $0 === mouse }) // re-delivered on wake — attach once
|
!mice.contains(where: { $0 === mouse }) // re-delivered on wake — attach once
|
||||||
else { return }
|
else { return }
|
||||||
mice.append(mouse)
|
mice.append(mouse)
|
||||||
#if os(iOS)
|
|
||||||
if inputDebug {
|
|
||||||
inputLog.debug(
|
|
||||||
"GCMouse attached: \(mouse.productCategory, privacy: .public)/\(mouse.vendorName ?? "?", privacy: .public) — now \(self.attachedMiceSummary, privacy: .public)")
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
// macOS drives motion + buttons from NSEvent (StreamLayerView's local monitor →
|
// macOS drives motion + buttons from NSEvent (StreamLayerView's local monitor →
|
||||||
// sendMotion/sendMouseButton) because GCMouse's handlers proved unreliable there;
|
// sendMotion/sendMouseButton) because GCMouse's handlers proved unreliable there;
|
||||||
// installing them too would double-send. iOS keeps GCMouse (raw deltas under
|
// installing them too would double-send. iOS keeps GCMouse (raw deltas under
|
||||||
|
|||||||
@@ -11,13 +11,18 @@
|
|||||||
// host mode, so the host's rescale is the identity).
|
// host mode, so the host's rescale is the identity).
|
||||||
//
|
//
|
||||||
// A hardware mouse/trackpad is a pointer, not a finger. When the scene is pointer-LOCKED
|
// A hardware mouse/trackpad is a pointer, not a finger. When the scene is pointer-LOCKED
|
||||||
// (full-screen + frontmost iPad) GCMouse delivers raw relative deltas and the system hides
|
// (full-screen + frontmost iPad, and the user hasn't disabled pointer capture in Settings —
|
||||||
// the cursor — the gaming-grade path. When it CAN'T lock (Stage Manager, not frontmost,
|
// see PointerLockChain, which steers the lock request through SwiftUI's hosting controllers)
|
||||||
// iPhone) the system shows its own cursor and routes the mouse through UIKit's pointer path:
|
// GCMouse delivers raw relative deltas and the system hides the cursor — the gaming-grade path.
|
||||||
// hover + indirect-pointer touches, which we forward as ABSOLUTE cursor position (+ buttons)
|
// InputCapture handles EVERY connected mouse (GCMouse.mice), not just the current one, so a
|
||||||
// so the host cursor tracks the visible local one. We never forward an indirect pointer as a
|
// trackpad + a second pointer (e.g. a Universal Control mouse) both drive. When the scene CAN'T
|
||||||
// touch — doing so hid the cursor and made the host see taps instead of a moving mouse.
|
// lock (Stage Manager, not frontmost, iPhone, capture disabled) the system shows its own cursor
|
||||||
// GCMouse is gated off whenever the lock isn't held so the two paths can't double-send.
|
// and routes the mouse through UIKit's pointer path: hover + indirect-pointer touches, which we
|
||||||
|
// forward as ABSOLUTE cursor position (+ buttons) so the host cursor tracks the visible local one.
|
||||||
|
// We never forward an indirect pointer as a touch — doing so hid the cursor and made the host see
|
||||||
|
// taps instead of a moving mouse. The two paths are mutually exclusive on `gcMouseForwarding`
|
||||||
|
// (== locked): GCMouse forwards only WHILE locked, the UIKit indirect path (motion, buttons AND
|
||||||
|
// scroll) only while NOT locked — so a pointer that emits both channels under lock can't double-send.
|
||||||
// Hardware keyboard forwarding shares InputCapture with macOS — auto-engaged when streaming
|
// Hardware keyboard forwarding shares InputCapture with macOS — auto-engaged when streaming
|
||||||
// starts, ⌘⎋ toggles (detected from the HID stream; there is no NSEvent monitor here).
|
// starts, ⌘⎋ toggles (detected from the HID stream; there is no NSEvent monitor here).
|
||||||
//
|
//
|
||||||
@@ -236,32 +241,24 @@ public final class StreamViewController: UIViewController {
|
|||||||
guard self?.captureEnabled == true else { return }
|
guard self?.captureEnabled == true else { return }
|
||||||
connection?.send(event)
|
connection?.send(event)
|
||||||
}
|
}
|
||||||
// Indirect pointer (mouse/trackpad with no lock) → absolute cursor + buttons, routed
|
// Indirect pointer (mouse/trackpad) WITHOUT a lock → absolute cursor + buttons + scroll.
|
||||||
// through InputCapture so the forwarding gate and release-on-blur apply uniformly.
|
// While the scene is pointer-LOCKED the GCMouse path owns motion AND buttons AND scroll, so
|
||||||
|
// the whole UIKit indirect path is gated off here (`gcMouseForwarding`). The trackpad and a
|
||||||
|
// mouse BOTH report through GCMouse under lock and ALSO emit UIKit indirect-pointer events
|
||||||
|
// (pinned at the locked position) — without this gate a click double-sends (GCMouse + UIKit)
|
||||||
|
// and a second pointer (e.g. a Universal Control mouse) competes with the trackpad. The gate
|
||||||
|
// is the exact mirror of the GCMouse handlers, which fire only while locked.
|
||||||
streamView.onPointerMoveAbs = { [weak self] p in
|
streamView.onPointerMoveAbs = { [weak self] p in
|
||||||
guard let self else { return }
|
guard let self, self.inputCapture?.gcMouseForwarding == false else { return }
|
||||||
if iosInputDebug {
|
|
||||||
// Whether ANY UIKit pointer movement reaches us while the scene is LOCKED tells us
|
|
||||||
// if the trackpad (which may not be a GCMouse) can still be captured via UIKit.
|
|
||||||
iosInputLog.debug(
|
|
||||||
"UIKit pointer move x=\(p.x, privacy: .public) y=\(p.y, privacy: .public) locked=\(self.pointerLockEngaged() == true, privacy: .public) gcFwd=\(self.inputCapture?.gcMouseForwarding == true, privacy: .public)")
|
|
||||||
}
|
|
||||||
self.inputCapture?.sendMouseAbs(
|
self.inputCapture?.sendMouseAbs(
|
||||||
x: p.x, y: p.y, surfaceWidth: p.w, surfaceHeight: p.h)
|
x: p.x, y: p.y, surfaceWidth: p.w, surfaceHeight: p.h)
|
||||||
}
|
}
|
||||||
streamView.onPointerButton = { [weak self] button, down in
|
streamView.onPointerButton = { [weak self] button, down in
|
||||||
self?.inputCapture?.sendMouseButton(button, pressed: down)
|
guard let self, self.inputCapture?.gcMouseForwarding == false else { return }
|
||||||
|
self.inputCapture?.sendMouseButton(button, pressed: down)
|
||||||
}
|
}
|
||||||
// Trackpad two-finger / wheel scroll → host scroll. The pan recognizer is the
|
|
||||||
// UNLOCKED regime; while locked, GCMouse's scroll handler owns it — mirror the
|
|
||||||
// sendMouseAbs !gcMouseForwarding gate so the two can't double-send.
|
|
||||||
streamView.onScroll = { [weak self] dx, dy in
|
streamView.onScroll = { [weak self] dx, dy in
|
||||||
guard let self else { return }
|
guard let self, self.inputCapture?.gcMouseForwarding == false else { return }
|
||||||
if iosInputDebug {
|
|
||||||
iosInputLog.debug(
|
|
||||||
"UIKit scroll dx=\(dx, privacy: .public) dy=\(dy, privacy: .public) locked=\(self.pointerLockEngaged() == true, privacy: .public)")
|
|
||||||
}
|
|
||||||
guard self.inputCapture?.gcMouseForwarding == false else { return }
|
|
||||||
self.inputCapture?.sendScroll(dx: dx, dy: dy)
|
self.inputCapture?.sendScroll(dx: dx, dy: dy)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -472,7 +469,7 @@ public final class StreamViewController: UIViewController {
|
|||||||
pointerInteraction?.invalidate() // re-resolve the hidden/visible cursor for the state
|
pointerInteraction?.invalidate() // re-resolve the hidden/visible cursor for the state
|
||||||
if iosInputDebug {
|
if iosInputDebug {
|
||||||
iosInputLog.debug(
|
iosInputLog.debug(
|
||||||
"pointer lock isLocked=\(locked, privacy: .public) captured=\(self.captured, privacy: .public) useGCMouse=\(useGCMouse, privacy: .public) [\(self.inputCapture?.attachedMiceSummary ?? "n/a", privacy: .public)]")
|
"pointer lock isLocked=\(locked, privacy: .public) captured=\(self.captured, privacy: .public)")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
Reference in New Issue
Block a user