diff --git a/clients/apple/Sources/PunktfunkClient/SettingsView.swift b/clients/apple/Sources/PunktfunkClient/SettingsView.swift index 1da94c7..3f9c9a7 100644 --- a/clients/apple/Sources/PunktfunkClient/SettingsView.swift +++ b/clients/apple/Sources/PunktfunkClient/SettingsView.swift @@ -18,6 +18,7 @@ struct SettingsView: View { @AppStorage(DefaultsKey.gamepadType) private var gamepadType = 0 @AppStorage(DefaultsKey.bitrateKbps) private var bitrateKbps = 0 @AppStorage(DefaultsKey.presenter) private var presenter = "stage1" + @AppStorage(DefaultsKey.cursorMode) private var cursorMode = "auto" @AppStorage(DefaultsKey.micEnabled) private var micEnabled = true @ObservedObject private var gamepads = GamepadManager.shared #if os(macOS) @@ -371,6 +372,24 @@ struct SettingsView: View { .font(.caption) .foregroundStyle(.secondary) } + #if os(macOS) + Section { + Picker("Cursor in stream", selection: $cursorMode) { + Text("Auto (gamescope)").tag("auto") + Text("Always").tag("always") + Text("Never").tag("never") + } + } header: { + Text("Cursor") + } footer: { + Text("Show the local system cursor over the stream instead of capturing it. " + + "gamescope's capture carries no cursor, so the client draws its own — " + + "Auto turns this on only for gamescope sessions. ⌘⇧C toggles it live " + + "during a session.") + .font(.caption) + .foregroundStyle(.secondary) + } + #endif Section { Picker("Presenter", selection: $presenter) { Text("Stage 1 (default)").tag("stage1") diff --git a/clients/apple/Sources/PunktfunkClient/StreamHUDView.swift b/clients/apple/Sources/PunktfunkClient/StreamHUDView.swift index 012f56f..ef7541b 100644 --- a/clients/apple/Sources/PunktfunkClient/StreamHUDView.swift +++ b/clients/apple/Sources/PunktfunkClient/StreamHUDView.swift @@ -39,6 +39,11 @@ struct StreamHUDView: View { : "Click the stream to capture input") .font(.caption2) .foregroundStyle(.secondary) + // The client-side cursor (⌘⇧C) draws the local cursor over the stream instead of + // capturing it — the only accurate cursor for gamescope, whose capture has none. + Text("⌘⇧C toggles the on-screen cursor") + .font(.caption2) + .foregroundStyle(.secondary) #elseif os(iOS) // Touch always plays directly; ⌘⎋ (hardware keyboard) toggles kb/mouse. Text(model.mouseCaptured diff --git a/clients/apple/Sources/PunktfunkKit/DefaultsKeys.swift b/clients/apple/Sources/PunktfunkKit/DefaultsKeys.swift index 1fdb039..d4443ab 100644 --- a/clients/apple/Sources/PunktfunkKit/DefaultsKeys.swift +++ b/clients/apple/Sources/PunktfunkKit/DefaultsKeys.swift @@ -20,4 +20,6 @@ public enum DefaultsKey { public static let micUID = "punktfunk.micUID" public static let presenter = "punktfunk.presenter" public static let hosts = "punktfunk.hosts" + /// Client-side cursor mode: "auto" (shown only in gamescope sessions), "always", "never". + public static let cursorMode = "punktfunk.cursorMode" } diff --git a/clients/apple/Sources/PunktfunkKit/InputCapture.swift b/clients/apple/Sources/PunktfunkKit/InputCapture.swift index 2d50a41..68da205 100644 --- a/clients/apple/Sources/PunktfunkKit/InputCapture.swift +++ b/clients/apple/Sources/PunktfunkKit/InputCapture.swift @@ -111,6 +111,12 @@ public final class InputCapture { /// event itself is swallowed). Main queue. public var onToggleCapture: (() -> Void)? + /// Fired on ⌘⇧C (the client-side-cursor toggle — flips between the captured/disassociated + /// relative path and the visible-cursor absolute path; detected here, like ⌘⎋, so it works + /// regardless of the current capture state and the event itself is swallowed). macOS only; + /// the absolute-vs-relative forwarding lives entirely in StreamLayerView. Main queue. + public var onToggleCursor: (() -> Void)? + /// Fired when a newer InputCapture takes the process-global GC handler slots (the /// singletons hold ONE handler each): the preempted owner must drop its capture /// state — its handlers are gone, so it would otherwise sit "captured" with dead @@ -203,6 +209,15 @@ public final class InputCapture { self.onToggleCapture?() return nil } + // ⌘⇧C toggles the client-side cursor (visible-cursor absolute path vs the + // captured relative path). keyCode 8 = kVK_ANSI_C; layout-independent so it + // fires the same on any keyboard. Suppress the C (latched like ⌘⎋'s Esc) so it + // doesn't type into the host, and swallow the event so it doesn't beep. + if event.keyCode == 8 /* C */, flags == [.command, .shift] { + self.suppressedVK = 0x43 // VK_C — the same physical C is en route via GC + self.onToggleCursor?() + return nil + } return event } #endif diff --git a/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift b/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift index ccb0265..d22d6e3 100644 --- a/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift +++ b/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift @@ -195,6 +195,13 @@ public final class PunktfunkConnection { /// DualSense feedback. public private(set) var resolvedGamepad: GamepadType = .auto + /// The compositor the host actually resolved for this session's virtual output (the + /// Welcome's echo of the requested `compositor`, with `.auto` resolved to a concrete + /// backend). `.auto` = an older host that didn't say. Clients use it to decide + /// client-side cursor behavior: `.gamescope`'s PipeWire capture carries no cursor, so + /// the client draws its own (a visible system cursor over the stream). + public private(set) var resolvedCompositor: Compositor = .auto + /// Host clock minus client clock (nanoseconds), from the connect-time wall-clock skew handshake /// (`punktfunk_connection_clock_offset_ns`). Add it to a local `CLOCK_REALTIME` instant to /// express that instant in the host's capture clock — the clock each `AccessUnit.ptsNs` is @@ -268,6 +275,9 @@ public final class PunktfunkConnection { var gp: UInt32 = 0 _ = punktfunk_connection_gamepad(handle, &gp) resolvedGamepad = GamepadType(rawValue: gp) ?? .auto + var comp: UInt32 = 0 + _ = punktfunk_connection_compositor(handle, &comp) + resolvedCompositor = Compositor(rawValue: comp) ?? .auto var offset: Int64 = 0 _ = punktfunk_connection_clock_offset_ns(handle, &offset) clockOffsetNs = offset diff --git a/clients/apple/Sources/PunktfunkKit/StreamView.swift b/clients/apple/Sources/PunktfunkKit/StreamView.swift index a72792e..fd73c37 100644 --- a/clients/apple/Sources/PunktfunkKit/StreamView.swift +++ b/clients/apple/Sources/PunktfunkKit/StreamView.swift @@ -35,31 +35,47 @@ private let streamInputDebug = /// (`CGAssociateMouseAndMouseCursorPosition(false)` — under which NSEvent mouseMoved/ /// dragged deltas become the relative motion StreamLayerView forwards), and hide it. /// hide/unhide and associate are balanced via `captured`. +/// +/// In CLIENT-SIDE-CURSOR mode (gamescope, whose capture carries no host cursor) this is a +/// no-op: the local cursor stays visible and free, and StreamLayerView forwards ABSOLUTE +/// positions instead — the visible system cursor IS the on-screen cursor. `disassociate` +/// selects between the two; `release()` only undoes what `capture` actually did. private final class CursorCapture { private var captured = false + /// Whether the engaged capture actually disassociated+hid (false in cursor-visible mode), + /// so `release()` only restores when it must. + private var disassociated = false /// Returns whether capture actually engaged. It can fail mid app-activation — the click /// that reactivates the app delivers `mouseDown` before the app is frontmost, and /// `CGAssociateMouseAndMouseCursorPosition` is refused then — so the caller must stay - /// released and let the NEXT click retry, never latching a half-captured state. - func capture(in view: NSView) -> Bool { + /// released and let the NEXT click retry, never latching a half-captured state. With + /// `disassociate: false` (cursor-visible mode) it always engages — there is no grab to + /// be refused, the cursor stays free and visible. + func capture(in view: NSView, disassociate: Bool) -> Bool { guard !captured, let window = view.window, view.bounds.width > 0 else { return false } - // Park the cursor mid-view so a click can't land in (and activate) another app. - let rectOnScreen = window.convertToScreen(view.convert(view.bounds, to: nil)) - let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 - CGWarpMouseCursorPosition( - CGPoint(x: rectOnScreen.midX, y: primaryHeight - rectOnScreen.midY)) - guard CGAssociateMouseAndMouseCursorPosition(0) == .success else { return false } - NSCursor.hide() + if disassociate { + // Park the cursor mid-view so a click can't land in (and activate) another app. + let rectOnScreen = window.convertToScreen(view.convert(view.bounds, to: nil)) + let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 + CGWarpMouseCursorPosition( + CGPoint(x: rectOnScreen.midX, y: primaryHeight - rectOnScreen.midY)) + guard CGAssociateMouseAndMouseCursorPosition(0) == .success else { return false } + NSCursor.hide() + } captured = true + disassociated = disassociate return true } func release() { guard captured else { return } - CGAssociateMouseAndMouseCursorPosition(1) - NSCursor.unhide() + if disassociated { + CGAssociateMouseAndMouseCursorPosition(1) + NSCursor.unhide() + } captured = false + disassociated = false } } @@ -136,10 +152,22 @@ public final class StreamLayerView: NSView { /// captured (GCMouse's own delivery proved unreliable on macOS — see InputCapture). /// Installed on engage, removed on release; nil while not captured. private var mouseEventMonitor: Any? + /// The window's `acceptsMouseMovedEvents` value before client-side-cursor capture raised + /// it (nil = not raised by us); restored on release so we leave the window as we found it. + private var savedAcceptsMouseMoved: Bool? /// Whether input capture is currently engaged (cursor hidden+frozen, mouse/keyboard /// forwarded). Main-thread only. public private(set) var captured = false + + /// Client-side-cursor mode: when true the local system cursor stays VISIBLE over the + /// stream and the mouse monitor forwards ABSOLUTE positions (the visible cursor is the + /// on-screen cursor — gamescope draws none, so no double cursor); when false the existing + /// captured/disassociated relative path runs unchanged. Initialized at session start from + /// the `cursorMode` setting + the host's resolved compositor, toggled live by ⌘⇧C. A live + /// flip re-engages capture in the new mode so disassociation + the abs/rel choice swap + /// atomically. Main-thread only. + private var cursorVisible = false /// One-shot auto-engage request (stream start, trust confirmed) — attempted as soon /// as the view is in a window with real bounds, then dropped, so it can never fire /// surprisingly later (e.g. on a resize). @@ -333,7 +361,9 @@ public final class StreamLayerView: NSView { // If the cursor grab is refused (e.g. the reactivating click arrives before the app is // frontmost), stay released so the NEXT click retries — never latch captured=true over // a free cursor, which would make mouseDown's `!captured` guard reject every later click. - guard cursorCapture.capture(in: self) else { return } + // In client-side-cursor mode there is no grab (the cursor stays visible) — capture + // always engages and the monitor forwards absolute positions instead. + guard cursorCapture.capture(in: self, disassociate: !cursorVisible) else { return } inputCapture?.setForwarding(true, suppressClick: fromClick) // Install AFTER the warp + setForwarding: the engage warp generates no forwarded // delta (the monitor isn't up yet), and the engage click's suppression latch is @@ -363,8 +393,16 @@ public final class StreamLayerView: NSView { /// host re-accelerates there's mild double-acceleration, acceptable and fixable later /// via IOHID. Events are returned (not swallowed): the cursor is frozen, so they're /// inert locally. + /// + /// In client-side-cursor mode the cursor is NOT frozen, so bare `.mouseMoved` events are + /// only generated while `window.acceptsMouseMovedEvents` is true — we enable it here and + /// restore it on removal so absolute hover-motion keeps flowing without a click held. private func installMouseMonitor() { guard mouseEventMonitor == nil else { return } + if cursorVisible { + savedAcceptsMouseMoved = window?.acceptsMouseMovedEvents + window?.acceptsMouseMovedEvents = true + } mouseEventMonitor = NSEvent.addLocalMonitorForEvents(matching: [ .mouseMoved, .leftMouseDragged, .rightMouseDragged, .otherMouseDragged, .leftMouseDown, .leftMouseUp, .rightMouseDown, .rightMouseUp, @@ -373,7 +411,16 @@ public final class StreamLayerView: NSView { guard let self, self.captured, let ic = self.inputCapture else { return event } switch event.type { case .mouseMoved, .leftMouseDragged, .rightMouseDragged, .otherMouseDragged: - ic.sendMotion(dx: Float(event.deltaX), dy: Float(event.deltaY)) // no y-negation + if self.cursorVisible { + // Client-side cursor: forward the ABSOLUTE position (mapped through the + // aspect-fit letterbox into host pixels), the same path the iPad pointer + // fallback uses. Events in the letterbox bars are dropped (nil host point). + if let p = self.hostPoint(from: event) { + ic.sendMouseAbs(x: p.x, y: p.y, surfaceWidth: p.w, surfaceHeight: p.h) + } + } else { + ic.sendMotion(dx: Float(event.deltaX), dy: Float(event.deltaY)) // no y-negation + } case .leftMouseDown: ic.sendMouseButton(1, pressed: true) case .leftMouseUp: ic.sendMouseButton(1, pressed: false) case .rightMouseDown: ic.sendMouseButton(3, pressed: true) @@ -393,6 +440,43 @@ public final class StreamLayerView: NSView { mouseEventMonitor = nil if streamInputDebug { streamInputLog.debug("mouse NSEvent monitor removed (capture released)") } } + // Restore the window's prior mouse-moved-events setting if we raised it (cursor mode). + if let saved = savedAcceptsMouseMoved { + window?.acceptsMouseMovedEvents = saved + savedAcceptsMouseMoved = nil + } + } + + /// One host-pixel point on the negotiated output, with the surface dimensions the host + /// rescales against (surface == host mode, so the host applies no extra scaling). + private struct HostPoint { let x: Int32; let y: Int32; let w: UInt32; let h: UInt32 } + + /// Map an NSEvent's cursor location into host-mode pixels for the client-side-cursor + /// (absolute) path. NSEvent.locationInWindow is window space, origin BOTTOM-left (+y up); + /// we convert to this view's space, FLIP y to the host's top-left (+y down) convention, + /// then aspect-fit-letterbox into the host mode exactly like the iOS touch/pointer path. + /// Returns nil for events in the letterbox bars (outside the video rect) so the host's + /// cursor isn't dragged onto a black edge, and until a mode is negotiated. + private func hostPoint(from event: NSEvent) -> HostPoint? { + guard let connection else { return nil } + let mode = connection.currentMode() + guard mode.width > 0, mode.height > 0 else { return nil } + // Window → view coords (non-flipped: origin bottom-left), then flip y into view-top-left. + let inView = convert(event.locationInWindow, from: nil) + let p = CGPoint(x: inView.x, y: bounds.height - inView.y) + // The video occupies the aspect-fit rect inside the (non-flipped) bounds; AVMakeRect's + // origin is bottom-left, so flip its minY too to match p's top-left space. + let fit = AVMakeRect( + aspectRatio: CGSize(width: Int(mode.width), height: Int(mode.height)), + insideRect: bounds) + guard fit.width > 0, fit.height > 0 else { return nil } + let videoMinYTop = bounds.height - fit.maxY + let u = (p.x - fit.minX) / fit.width + let v = (p.y - videoMinYTop) / fit.height + guard u >= 0, u <= 1, v >= 0, v <= 1 else { return nil } // letterbox bars + let hx = Int32((u * CGFloat(mode.width)).rounded().clamped(0, CGFloat(mode.width - 1))) + let hy = Int32((v * CGFloat(mode.height)).rounded().clamped(0, CGFloat(mode.height - 1))) + return HostPoint(x: hx, y: hy, w: mode.width, h: mode.height) } /// NSEvent `buttonNumber` → GameStream wire id for the "other" buttons: 2 = middle, @@ -444,9 +528,28 @@ public final class StreamLayerView: NSView { // be a cursor trap with dead input. self?.releaseCapture() } + // ⌘⇧C flips the client-side cursor live. Only the key window's stream owns it (same + // guard as the ⌘⎋ capture toggle). Re-engage capture in the new mode so disassociation + // and the absolute/relative forwarding choice swap atomically — releaseCapture restores + // the old mode's grab (if any), engageCapture installs the new one. + capture.onToggleCursor = { [weak self] in + guard let self, self.window?.isKeyWindow == true else { return } + self.cursorVisible.toggle() + let wasCaptured = self.captured + self.releaseCapture() + if wasCaptured { self.engageCapture(fromClick: false) } + } capture.start() inputCapture = capture + // Resolve the client-side-cursor mode for this session: Auto → on iff the host + // resolved gamescope (whose capture carries no cursor); Always → on; Never → off. + switch UserDefaults.standard.string(forKey: DefaultsKey.cursorMode) ?? "auto" { + case "always": cursorVisible = true + case "never": cursorVisible = false + default: cursorVisible = connection.resolvedCompositor == .gamescope + } + // Presenter choice — default stage-1 (the known-good AVSampleBufferDisplayLayer). Stage-2 // (`punktfunk.presenter == "stage2"`) takes explicit VTDecompressionSession decode + a // CAMetalLayer/display-link present; it falls back here if Metal can't be set up. @@ -547,4 +650,12 @@ public final class StreamLayerView: NSView { teardownStage2() // invalidate the display link + stop the pipeline if stop() was missed } } + +extension CGFloat { + /// Clamp into a [lo, hi] range — keeps the absolute-cursor mapping inside the host's + /// pixel bounds even if a stray event reports a point a hair past the video rect. + fileprivate func clamped(_ lo: CGFloat, _ hi: CGFloat) -> CGFloat { + Swift.min(Swift.max(self, lo), hi) + } +} #endif