From 12df1388debcdcd3a8e0e2808b7818ae586930a9 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Wed, 22 Jul 2026 00:03:19 +0200 Subject: [PATCH] =?UTF-8?q?feat(apple/input):=20desktop=20(absolute)=20mou?= =?UTF-8?q?se=20mode=20on=20macOS=20=E2=80=94=20remote-desktop=20sweep=20M?= =?UTF-8?q?1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Folds the parked client-side-cursor machinery (cursorMode auto/always/ never, hidden while disabled) into the cross-client mouse model: MouseInputMode capture|desktop under DefaultsKey.mouseMode, picked in Settings ▸ Keyboard & mouse (macOS), resolved at session start and gated off on gamescope hosts (relative-only EIS). Desktop model = the un-neutered absolute path with the SDL cursor policy: pointer never disassociated (enters/leaves the stream freely), monitor forwards letterboxed absolute positions, and the local cursor hides only while over the view via an invisible-cursor rect (the host's composited cursor is the one you see; AppKit manages the rect, so no hide/unhide balancing). ⌘⇧C becomes ⌃⌥⇧M — the same chord as the SDL clients — flipping the model live with an atomic release/re-engage. Verified: swift build + full test suites green (macOS arm64). Co-Authored-By: Claude Fable 5 --- .../Settings/SettingsView+Sections.swift | 22 ++++ .../Settings/SettingsView.swift | 1 + .../PunktfunkKit/Input/InputCapture.swift | 25 ++-- .../PunktfunkKit/Input/MouseInputMode.swift | 12 ++ .../PunktfunkKit/Views/StreamView.swift | 110 ++++++++++++------ .../PunktfunkShared/DefaultsKeys.swift | 7 +- 6 files changed, 128 insertions(+), 49 deletions(-) create mode 100644 clients/apple/Sources/PunktfunkKit/Input/MouseInputMode.swift diff --git a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift index eef4e79e..471a21ff 100644 --- a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift +++ b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift @@ -447,6 +447,14 @@ extension SettingsView { /// (always on macOS; an attached keyboard/mouse on iPad). Absent on tvOS (no such input path). @ViewBuilder var inputSection: some View { Section("Keyboard & mouse") { + #if os(macOS) + described(mouseModeDescription) { + Picker("Mouse input", selection: $mouseMode) { + Text("Capture (games)").tag(MouseInputMode.capture.rawValue) + Text("Desktop (absolute)").tag(MouseInputMode.desktop.rawValue) + } + } + #endif described((ModifierLayout(rawValue: modifierLayout) ?? .mac).detail) { Picker("Modifier keys", selection: $modifierLayout) { ForEach(ModifierLayout.allCases, id: \.self) { layout in @@ -459,6 +467,20 @@ extension SettingsView { } } } + + #if os(macOS) + /// The SELECTED mouse model explained — dynamic, like the touch-mode caption. + private var mouseModeDescription: String { + switch MouseInputMode(rawValue: mouseMode) ?? .capture { + case .capture: + return "The pointer locks to the stream and sends relative motion — best for " + + "games. ⌃⌥⇧M switches live; applies from the next capture otherwise." + case .desktop: + return "The pointer moves freely in and out of the stream and sends absolute " + + "positions — best for remote desktop work. Unavailable on gamescope hosts." + } + } + #endif #endif // MARK: - Audio diff --git a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift index e4e73dea..56174796 100644 --- a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift +++ b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift @@ -89,6 +89,7 @@ struct SettingsView: View { @State var customMode = false #endif #if os(macOS) + @AppStorage(DefaultsKey.mouseMode) var mouseMode = MouseInputMode.capture.rawValue @AppStorage(DefaultsKey.speakerUID) var speakerUID = "" @AppStorage(DefaultsKey.micUID) var micUID = "" @AppStorage(DefaultsKey.micChannel) var micChannel = 0 diff --git a/clients/apple/Sources/PunktfunkKit/Input/InputCapture.swift b/clients/apple/Sources/PunktfunkKit/Input/InputCapture.swift index 3a00bd9d..546c5b3f 100644 --- a/clients/apple/Sources/PunktfunkKit/Input/InputCapture.swift +++ b/clients/apple/Sources/PunktfunkKit/Input/InputCapture.swift @@ -110,11 +110,11 @@ public final class InputCapture { /// event itself is swallowed). Main queue. public var onToggleCapture: (() -> Void)? - /// Fired on ⌘⇧C (the client-side-cursor toggle — flips between the captured/disassociated - /// relative path and the visible-cursor absolute path; detected here, like ⌘⎋, so it works - /// regardless of the current capture state and the event itself is swallowed). macOS only; - /// the absolute-vs-relative forwarding lives entirely in StreamLayerView. Main queue. - public var onToggleCursor: (() -> Void)? + /// Fired on ⌃⌥⇧M (the mouse-model flip, capture ⇄ desktop — cross-client parity with the + /// SDL clients' Ctrl+Alt+Shift+M; detected here, like ⌘⎋, so it works regardless of the + /// current capture state and the event itself is swallowed). macOS only; the + /// absolute-vs-relative forwarding lives entirely in StreamLayerView. Main queue. + public var onToggleMouseMode: (() -> Void)? /// The cross-client combos (Windows/Linux parity: Ctrl+Alt+Shift+Q/D/S), fired from the macOS /// keyDown monitor only WHILE FORWARDING — that's the state in which the app's menu (which @@ -245,13 +245,14 @@ public final class InputCapture { self.onToggleCapture?() return nil } - // ⌘⇧C toggles the client-side cursor (visible-cursor absolute path vs the - // captured relative path). keyCode 8 = kVK_ANSI_C; layout-independent so it - // fires the same on any keyboard. Suppress the C (latched like ⌘⎋'s Esc) so it - // doesn't type into the host, and swallow the event so it doesn't beep. - if event.keyCode == 8 /* C */, flags == [.command, .shift] { - self.suppressedVK = 0x43 // VK_C — the same physical C is en route via GC - self.onToggleCursor?() + // ⌃⌥⇧M flips the mouse model (capture ⇄ desktop — the SDL clients' identical + // chord). Detected in both capture states, like ⌘⎋, so the model can be set + // before engaging. keyCode 46 = kVK_ANSI_M; layout-independent. Suppress the M + // (latched like ⌘⎋'s Esc) so it doesn't type into the host, and swallow the + // event so it doesn't beep. + if event.keyCode == 46 /* M */, flags == [.control, .option, .shift] { + self.suppressedVK = 0x4D // VK_M — the same physical M is en route via GC + self.onToggleMouseMode?() return nil } // The cross-client combos (Ctrl+Alt+Shift+Q/D/S — the same set every other diff --git a/clients/apple/Sources/PunktfunkKit/Input/MouseInputMode.swift b/clients/apple/Sources/PunktfunkKit/Input/MouseInputMode.swift new file mode 100644 index 00000000..c3a9c97c --- /dev/null +++ b/clients/apple/Sources/PunktfunkKit/Input/MouseInputMode.swift @@ -0,0 +1,12 @@ +/// How a physical mouse drives the host — the cross-client mouse model (the SDL clients' +/// `MouseMode` / `Settings::mouse_mode`, design/remote-desktop-sweep.md M1). Stored stringly +/// under `DefaultsKey.mouseMode`. +public enum MouseInputMode: String, CaseIterable, Sendable { + /// Pointer capture (disassociated, hidden cursor, relative deltas) — the game model, + /// and the default: the only cursor you see is the host's. + case capture + /// Absolute pointer, uncaptured: the cursor enters and leaves the stream freely and + /// motion is forwarded as absolute positions through the letterbox. The remote desktop + /// model. Requires a host injector with absolute support (not gamescope). + case desktop +} diff --git a/clients/apple/Sources/PunktfunkKit/Views/StreamView.swift b/clients/apple/Sources/PunktfunkKit/Views/StreamView.swift index b288a09c..c48b0d1a 100644 --- a/clients/apple/Sources/PunktfunkKit/Views/StreamView.swift +++ b/clients/apple/Sources/PunktfunkKit/Views/StreamView.swift @@ -38,10 +38,11 @@ private let streamInputDebug = /// dragged deltas become the relative motion StreamLayerView forwards), and hide it. /// hide/unhide and associate are balanced via `captured`. /// -/// In CLIENT-SIDE-CURSOR mode (gamescope, whose capture carries no host cursor) this is a -/// no-op: the local cursor stays visible and free, and StreamLayerView forwards ABSOLUTE -/// positions instead — the visible system cursor IS the on-screen cursor. `disassociate` -/// selects between the two; `release()` only undoes what `capture` actually did. +/// In the DESKTOP mouse model (absolute pointer, remote-desktop-sweep M1) this is a no-op: +/// the pointer stays free (entering and leaving the stream at will) and StreamLayerView +/// forwards ABSOLUTE positions instead; the local cursor is hidden only while over the view +/// (cursor rects). `disassociate` selects between the two; `release()` only undoes what +/// `capture` actually did. private final class CursorCapture { private var captured = false /// Whether the engaged capture actually disassociated+hid (false in cursor-visible mode), @@ -207,14 +208,17 @@ public final class StreamLayerView: NSView { /// forwarded). Main-thread only. public private(set) var captured = false - /// Client-side-cursor mode: when true the local system cursor stays VISIBLE over the - /// stream and the mouse monitor forwards ABSOLUTE positions (the visible cursor is the - /// on-screen cursor — gamescope draws none, so no double cursor); when false the existing - /// captured/disassociated relative path runs unchanged. Initialized at session start from - /// the `cursorMode` setting + the host's resolved compositor, toggled live by ⌘⇧C. A live - /// flip re-engages capture in the new mode so disassociation + the abs/rel choice swap - /// atomically. Main-thread only. - private var cursorVisible = false + /// Desktop (absolute) mouse model — remote-desktop-sweep M1: when true the pointer is + /// never disassociated (it enters and leaves the stream freely) and the mouse monitor + /// forwards ABSOLUTE positions through the letterbox; the local cursor is hidden only + /// while over this view (cursor rects — the host's composited cursor, tracking our + /// sends, is the one you see) and reappears the moment it leaves. When false the + /// captured/disassociated relative path runs unchanged. Initialized at session start + /// from the `mouseMode` setting gated by the host's resolved compositor (gamescope's + /// EIS is relative-only — absolute sends would be dropped, so it pins to capture); + /// flipped live by ⌃⌥⇧M. A live flip re-engages capture in the new model so + /// disassociation + the abs/rel choice swap atomically. Main-thread only. + private var desktopMouse = false /// One-shot auto-engage request (stream start, trust confirmed) — attempted as soon /// as the view is in a window with real bounds, then dropped, so it can never fire /// surprisingly later (e.g. on a resize). @@ -440,9 +444,9 @@ public final class StreamLayerView: NSView { // If the cursor grab is refused (e.g. the reactivating click arrives before the app is // frontmost), stay released so the NEXT click retries — never latch captured=true over // a free cursor, which would make mouseDown's `!captured` guard reject every later click. - // In client-side-cursor mode there is no grab (the cursor stays visible) — capture + // In the desktop mouse model there is no grab (the pointer stays free) — capture // always engages and the monitor forwards absolute positions instead. - guard cursorCapture.capture(in: self, disassociate: !cursorVisible) else { return } + guard cursorCapture.capture(in: self, disassociate: !desktopMouse) else { return } inputCapture?.setForwarding(true, suppressClick: fromClick) // Install AFTER the warp + setForwarding: the engage warp generates no forwarded // delta (the monitor isn't up yet), and the engage click's suppression latch is @@ -450,6 +454,7 @@ public final class StreamLayerView: NSView { installMouseMonitor() captured = true window?.makeFirstResponder(self) + window?.invalidateCursorRects(for: self) // desktop model: hide-over-view engages notifyCaptureChange(true) } @@ -459,9 +464,28 @@ public final class StreamLayerView: NSView { cursorCapture.release() inputCapture?.setForwarding(false) captured = false + window?.invalidateCursorRects(for: self) notifyCaptureChange(false) } + /// A fully transparent cursor for the desktop mouse model's hide-over-view rect — + /// an empty 1×1 image draws nothing. + private static let invisibleCursor = NSCursor( + image: NSImage(size: NSSize(width: 1, height: 1)), hotSpot: .zero) + + /// Desktop mouse model: the local cursor is hidden while over the stream (the host's + /// composited cursor, tracking our absolute sends, is the one you see) and reappears + /// the moment it leaves the view — AppKit applies/removes the rect's cursor for us, + /// so there is no hide/unhide balancing to get wrong. Capture model instead hides + /// globally via `CursorCapture` (the pointer can't leave the view there). + override public func resetCursorRects() { + if captured && desktopMouse { + addCursorRect(bounds, cursor: Self.invisibleCursor) + } else { + super.resetCursorRects() + } + } + /// A single local monitor for motion + buttons, installed only while captured. A local /// monitor is more robust than view overrides for relative motion: it sidesteps the /// `window.acceptsMouseMovedEvents`/tracking-area/responder-chain requirements, and @@ -473,12 +497,12 @@ public final class StreamLayerView: NSView { /// via IOHID. Events are returned (not swallowed): the cursor is frozen, so they're /// inert locally. /// - /// In client-side-cursor mode the cursor is NOT frozen, so bare `.mouseMoved` events are + /// In the desktop mouse model the cursor is NOT frozen, so bare `.mouseMoved` events are /// only generated while `window.acceptsMouseMovedEvents` is true — we enable it here and /// restore it on removal so absolute hover-motion keeps flowing without a click held. private func installMouseMonitor() { guard mouseEventMonitor == nil else { return } - if cursorVisible { + if desktopMouse { savedAcceptsMouseMoved = window?.acceptsMouseMovedEvents window?.acceptsMouseMovedEvents = true } @@ -490,8 +514,8 @@ public final class StreamLayerView: NSView { guard let self, self.captured, let ic = self.inputCapture else { return event } switch event.type { case .mouseMoved, .leftMouseDragged, .rightMouseDragged, .otherMouseDragged: - if self.cursorVisible { - // Client-side cursor: forward the ABSOLUTE position (mapped through the + if self.desktopMouse { + // Desktop mouse model: forward the ABSOLUTE position (mapped through the // aspect-fit letterbox into host pixels), the same path the iPad pointer // fallback uses. Events in the letterbox bars are dropped (nil host point). if let p = self.hostPoint(from: event) { @@ -609,14 +633,27 @@ public final class StreamLayerView: NSView { // be a cursor trap with dead input. self?.releaseCapture() } - // ⌘⇧C flips the client-side cursor live. Only the key window's stream owns it (same - // guard as the ⌘⎋ capture toggle). Re-engage capture in the new mode so disassociation - // and the absolute/relative forwarding choice swap atomically — releaseCapture restores - // the old mode's grab (if any), engageCapture installs the new one. - // ⌘⇧C would flip the client-side cursor live — NEUTERED while the feature is disabled - // (see the cursorVisible resolution below): toggling it on under gamescope's relative-only - // input traps the pointer. Restore this body when absolute/synthetic-cursor support lands. - capture.onToggleCursor = {} + // ⌃⌥⇧M flips the mouse model (capture ⇄ desktop) live — the SDL clients' identical + // chord. Only the key window's stream owns it (same guard as the ⌘⎋ capture toggle). + // Re-engage capture in the new model so disassociation and the absolute/relative + // forwarding choice swap atomically — releaseCapture restores the old model's grab + // (if any), engageCapture installs the new one. On a gamescope host the chord is a + // no-op: its EIS grants only a relative pointer, so the desktop model's absolute + // sends would be silently dropped (pointer stuck = "all input dead"). + capture.onToggleMouseMode = { [weak self] in + guard let self, self.window?.isKeyWindow == true, + let conn = self.connection else { return } + guard conn.resolvedCompositor != .gamescope else { + streamInputLog.info("mouse-mode chord ignored: gamescope host is relative-only") + return + } + let wasCaptured = self.captured + if wasCaptured { self.releaseCapture() } + self.desktopMouse.toggle() + if wasCaptured { self.engageCapture(fromClick: false) } + self.window?.invalidateCursorRects(for: self) + streamInputLog.info("mouse mode: \(self.desktopMouse ? "desktop" : "capture", privacy: .public)") + } // The cross-client combos (⌃⌥⇧Q/D/S — Ctrl+Alt+Shift on the other clients), delivered by // the monitor only while captured; the same key-window ownership rule as ⌘⎋ throughout. capture.onReleaseCapture = { [weak self] in @@ -643,15 +680,18 @@ public final class StreamLayerView: NSView { capture.start() inputCapture = capture - // Client-side cursor is TEMPORARILY DISABLED. It positions the host cursor with ABSOLUTE - // events, but gamescope's input socket (EIS) grants only a relative pointer, so those are - // silently dropped — the pointer never moves and clicks/scroll land on the stuck position - // (looks like "all input dead"). gamescope is exactly the compositor Auto enabled it for. - // Forced off until per-compositor gating (KWin/GNOME/Sway have absolute) or a synthetic- - // cursor-over-relative path lands; the resolution logic below is kept for that. See the - // ⌘⇧C handler (also neutered) and the cursorMode setting (hidden). - cursorVisible = false - _ = connection.resolvedCompositor // (was: Auto → gamescope; kept to document intent) + // Desktop (absolute) mouse model — resolved at session start from the mouseMode + // setting, gated by the host's compositor: gamescope's input socket (EIS) grants + // only a relative pointer, so absolute sends would be silently dropped there + // (pointer stuck = "all input dead") — pinned to capture. ⌃⌥⇧M flips it live. + let mode = MouseInputMode( + rawValue: UserDefaults.standard.string(forKey: DefaultsKey.mouseMode) ?? "" + ) ?? .capture + let absOK = connection.resolvedCompositor != .gamescope + desktopMouse = mode == .desktop && absOK + if mode == .desktop && !absOK { + streamInputLog.info("desktop mouse mode unavailable on a gamescope host (relative-only) — using capture") + } // Presenter choice + lifecycle live in SessionPresenter (shared with iOS/tvOS): stage-2 // (explicit VTDecompressionSession decode + a CAMetalLayer/display-link present) by diff --git a/clients/apple/Sources/PunktfunkShared/DefaultsKeys.swift b/clients/apple/Sources/PunktfunkShared/DefaultsKeys.swift index 750d5b65..95211585 100644 --- a/clients/apple/Sources/PunktfunkShared/DefaultsKeys.swift +++ b/clients/apple/Sources/PunktfunkShared/DefaultsKeys.swift @@ -94,8 +94,11 @@ public enum DefaultsKey { /// stays 4:2:0). Sharper text/UI at the cost of more bandwidth. public static let enable444 = "punktfunk.enable444" public static let hosts = "punktfunk.hosts" - /// Client-side cursor mode: "auto" (shown only in gamescope sessions), "always", "never". - public static let cursorMode = "punktfunk.cursorMode" + /// Physical-mouse model (macOS): "capture" (pointer lock + relative, the default) or + /// "desktop" (uncaptured absolute pointer) — the cross-client `mouse_mode`. Replaces the + /// never-shipped "punktfunk.cursorMode" (auto/always/never client-side-cursor setting, + /// which was hidden while disabled and had no readers). + public static let mouseMode = "punktfunk.mouseMode" /// Invert the scroll-wheel / two-finger-scroll direction sent to the host (both axes). Off by /// default: the local (natural-scrolling) sign passes through untouched. When on, the sign is /// negated at the single scroll sink (`InputCapture.sendScroll`), so it flips consistently across