enricobuehler
a730ca8557
ci / rust (push) Has been cancelled
Scroll was wired to GCMouse's scroll dpad, which only fires for plain HID wheel deltas — trackpad and Magic Mouse scrolling are gesture events that never reach GameController, so scrolling was dead on the default Mac setups. The stream view now overrides scrollWheel (while captured the cursor is parked mid-view, so it receives every scroll event) and feeds InputCapture.sendScroll: precise gesture deltas are pixels (~0.1 notch/px, SDL's factor → ×12 for WHEEL_DELTA(120)), classic wheels are lines (×120), fractional remainders accumulate, and the GC scroll handler is gone so wheel mice can't double-deliver. Signs pass through as-is, preserving the local (natural-)scrolling preference. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
382 lines
16 KiB
Swift
382 lines
16 KiB
Swift
// SwiftUI presentation: AVSampleBufferDisplayLayer fed straight from the punktfunk/1 connection.
|
||
//
|
||
// Stage-1 presenter (see README): the layer accepts *compressed* HEVC sample buffers and
|
||
// does hardware decode + display itself — fastest path to pixels, IOSurface-backed
|
||
// zero-copy on Apple silicon. Stage 2 (explicit VTDecompressionSession + CAMetalLayer)
|
||
// replaces this when we start tuning frame pacing / measuring glass-to-glass.
|
||
//
|
||
// The view also owns the input-capture state machine (Moonlight-style): capture is a
|
||
// deliberate, reversible state — engaged when the stream starts and when the user clicks
|
||
// into the video, released by ⌘⎋ or focus loss, and NEVER engaged by mere app
|
||
// activation (the click that activates the window may be a title-bar drag or a resize —
|
||
// warping the cursor there is exactly the intrusiveness this design removes). While
|
||
// released, nothing is forwarded to the host and the local cursor is free.
|
||
//
|
||
// macOS-first (NSViewRepresentable); the iOS variant is the same layer under
|
||
// UIViewRepresentable.
|
||
|
||
#if os(macOS)
|
||
import AppKit
|
||
import AVFoundation
|
||
import SwiftUI
|
||
|
||
/// Hides the LOCAL cursor while captured. The host renders its own cursor, and the local
|
||
/// one both diverges from it (the host applies acceleration/clamping to our raw deltas)
|
||
/// and can wander out of the window — a click there would focus another app. So while
|
||
/// captured we do what Moonlight does: warp the cursor into the view, freeze it
|
||
/// (`CGAssociateMouseAndMouseCursorPosition(false)` — GCMouse still delivers raw HID
|
||
/// deltas), and hide it. hide/unhide and associate are balanced via `captured`.
|
||
private final class CursorCapture {
|
||
private var captured = false
|
||
|
||
func capture(in view: NSView) {
|
||
guard !captured, let window = view.window, view.bounds.width > 0 else { return }
|
||
// Park the cursor mid-view so a click can't land in (and activate) another app.
|
||
let rectOnScreen = window.convertToScreen(view.convert(view.bounds, to: nil))
|
||
let primaryHeight = NSScreen.screens.first?.frame.height ?? 0
|
||
CGWarpMouseCursorPosition(
|
||
CGPoint(x: rectOnScreen.midX, y: primaryHeight - rectOnScreen.midY))
|
||
CGAssociateMouseAndMouseCursorPosition(0)
|
||
NSCursor.hide()
|
||
captured = true
|
||
}
|
||
|
||
func release() {
|
||
guard captured else { return }
|
||
CGAssociateMouseAndMouseCursorPosition(1)
|
||
NSCursor.unhide()
|
||
captured = false
|
||
}
|
||
}
|
||
|
||
public struct StreamView: NSViewRepresentable {
|
||
private let connection: PunktfunkConnection
|
||
private let captureEnabled: Bool
|
||
private let onCaptureChange: ((Bool) -> Void)?
|
||
private let onFrame: (@Sendable (AccessUnit) -> Void)?
|
||
private let onSessionEnd: (@Sendable () -> Void)?
|
||
|
||
/// `onFrame`/`onSessionEnd` fire on the pump thread — hop to the main actor for UI.
|
||
/// `captureEnabled: false` disables input capture entirely while UI (e.g. a trust
|
||
/// prompt) is layered over the stream; flipping it to true auto-engages capture
|
||
/// once. `onCaptureChange` (main thread) reports engage/release — drive the HUD's
|
||
/// "click to capture" / "⌘⎋ releases" hint with it.
|
||
public init(
|
||
connection: PunktfunkConnection,
|
||
captureEnabled: Bool = true,
|
||
onCaptureChange: ((Bool) -> Void)? = nil,
|
||
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
|
||
onSessionEnd: (@Sendable () -> Void)? = nil
|
||
) {
|
||
self.connection = connection
|
||
self.captureEnabled = captureEnabled
|
||
self.onCaptureChange = onCaptureChange
|
||
self.onFrame = onFrame
|
||
self.onSessionEnd = onSessionEnd
|
||
}
|
||
|
||
public func makeNSView(context: Context) -> StreamLayerView {
|
||
let view = StreamLayerView()
|
||
view.onCaptureChange = onCaptureChange
|
||
view.captureEnabled = captureEnabled
|
||
view.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
|
||
return view
|
||
}
|
||
|
||
public func updateNSView(_ view: StreamLayerView, context: Context) {
|
||
view.onCaptureChange = onCaptureChange
|
||
view.captureEnabled = captureEnabled
|
||
// SwiftUI reuses the NSView across state changes — repoint the pump only when the
|
||
// connection identity actually changed.
|
||
if view.connection !== connection {
|
||
view.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
|
||
}
|
||
}
|
||
|
||
public static func dismantleNSView(_ view: StreamLayerView, coordinator: ()) {
|
||
view.stop()
|
||
}
|
||
}
|
||
|
||
public final class StreamLayerView: NSView {
|
||
/// Cancellation handle owned by exactly one pump thread — a restart hands the old pump
|
||
/// its own token, so it can never be revived by a newer start().
|
||
private final class PumpToken: @unchecked Sendable {
|
||
private let lock = NSLock()
|
||
private var live = true
|
||
var isLive: Bool {
|
||
lock.lock()
|
||
defer { lock.unlock() }
|
||
return live
|
||
}
|
||
func cancel() {
|
||
lock.lock()
|
||
live = false
|
||
lock.unlock()
|
||
}
|
||
}
|
||
|
||
private let displayLayer = AVSampleBufferDisplayLayer()
|
||
private var token: PumpToken?
|
||
public private(set) var connection: PunktfunkConnection?
|
||
private let cursorCapture = CursorCapture()
|
||
private var inputCapture: InputCapture?
|
||
private var appObservers: [NSObjectProtocol] = []
|
||
private var windowObservers: [NSObjectProtocol] = []
|
||
|
||
/// Whether input capture is currently engaged (cursor hidden+frozen, mouse/keyboard
|
||
/// forwarded). Main-thread only.
|
||
public private(set) var captured = false
|
||
/// One-shot auto-engage request (stream start, trust confirmed) — attempted as soon
|
||
/// as the view is in a window with real bounds, then dropped, so it can never fire
|
||
/// surprisingly later (e.g. on a resize).
|
||
private var pendingAutoCapture = false
|
||
|
||
/// Reports engage/release on the main thread.
|
||
public var onCaptureChange: ((Bool) -> Void)?
|
||
|
||
/// Main-thread only. False = input capture disabled outright (UI layered over the
|
||
/// stream); flipping to true auto-engages once.
|
||
public var captureEnabled = true {
|
||
didSet {
|
||
guard captureEnabled != oldValue else { return }
|
||
if captureEnabled {
|
||
requestAutoCapture()
|
||
} else {
|
||
releaseCapture()
|
||
}
|
||
}
|
||
}
|
||
|
||
public override init(frame: NSRect) {
|
||
super.init(frame: frame)
|
||
displayLayer.videoGravity = .resizeAspect
|
||
layer = displayLayer // layer-hosting: assign before wantsLayer
|
||
wantsLayer = true
|
||
// Focus loss releases capture. Becoming active does NOT re-engage: the click
|
||
// that activates the window may be on the title bar (a drag) or a resize edge —
|
||
// the user clicks into the video (or hits ⌘⎋) when they want capture back.
|
||
appObservers.append(NotificationCenter.default.addObserver(
|
||
forName: NSApplication.didResignActiveNotification, object: nil, queue: .main
|
||
) { [weak self] _ in
|
||
self?.releaseCapture()
|
||
})
|
||
}
|
||
|
||
public required init?(coder: NSCoder) { fatalError("not used") }
|
||
|
||
public override func viewDidMoveToWindow() {
|
||
super.viewDidMoveToWindow()
|
||
windowObservers.forEach(NotificationCenter.default.removeObserver(_:))
|
||
windowObservers.removeAll()
|
||
guard let window else {
|
||
releaseCapture()
|
||
return
|
||
}
|
||
// ⌘-key-equivalents stay live while captured, so Settings (⌘,), a new window
|
||
// (⌘N), or Minimize (⌘M) can take key status without the APP resigning active —
|
||
// capture must release then too, or the new window inherits a hidden, frozen
|
||
// cursor and its local typing is double-delivered to the host.
|
||
for name in [NSWindow.didResignKeyNotification, NSWindow.didMiniaturizeNotification] {
|
||
windowObservers.append(NotificationCenter.default.addObserver(
|
||
forName: name, object: window, queue: .main
|
||
) { [weak self] _ in
|
||
self?.releaseCapture()
|
||
})
|
||
}
|
||
attemptPendingCapture()
|
||
}
|
||
|
||
public override func layout() {
|
||
super.layout()
|
||
attemptPendingCapture() // bounds become real here on first presentation
|
||
}
|
||
|
||
// MARK: - Capture state machine
|
||
|
||
/// Clicking into the video engages capture; that click is local (engagement), so
|
||
/// InputCapture suppresses its press/release toward the host. Clicks while captured
|
||
/// are the host's (GC forwards them) — nothing to do here.
|
||
public override func mouseDown(with event: NSEvent) {
|
||
if captureEnabled, !captured {
|
||
engageCapture(fromClick: true)
|
||
return
|
||
}
|
||
super.mouseDown(with: event)
|
||
}
|
||
|
||
/// A click from another app counts (one click into the video captures, not two).
|
||
public override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true }
|
||
|
||
/// The engage click is complete — drop its suppression latch (see InputCapture;
|
||
/// guards against GC delivering both halves of the click before our mouseDown).
|
||
public override func mouseUp(with event: NSEvent) {
|
||
inputCapture?.endClickSuppression()
|
||
super.mouseUp(with: event)
|
||
}
|
||
|
||
/// Scroll is forwarded from here, not from GCMouse: trackpad/Magic Mouse gestures
|
||
/// never reach GameController's scroll dpad. While captured the cursor is parked
|
||
/// mid-view, so this view receives every scroll event. Precise (gesture) deltas are
|
||
/// pixels — ~0.1 wheel notch per pixel (SDL's factor) → ×12 for WHEEL_DELTA(120);
|
||
/// classic wheels report lines, one notch = ±1 → ×120. Signs pass through as-is,
|
||
/// preserving the user's local (natural-)scrolling preference.
|
||
public override func scrollWheel(with event: NSEvent) {
|
||
guard captured, let inputCapture else {
|
||
super.scrollWheel(with: event)
|
||
return
|
||
}
|
||
let scale: Float = event.hasPreciseScrollingDeltas ? 12 : 120
|
||
inputCapture.sendScroll(
|
||
dx: Float(event.scrollingDeltaX) * scale,
|
||
dy: Float(event.scrollingDeltaY) * scale)
|
||
}
|
||
|
||
// While captured, the view is first responder and consumes key events — GC delivers
|
||
// them to the host independently, and consuming here stops the responder chain's
|
||
// "unhandled keyDown" beep without touching the event stream GC may rely on.
|
||
// ⌘-combos arrive via performKeyEquivalent instead and stay fully functional (⌘D).
|
||
public override var acceptsFirstResponder: Bool { true }
|
||
public override func keyDown(with event: NSEvent) {
|
||
if captured { return }
|
||
super.keyDown(with: event)
|
||
}
|
||
public override func keyUp(with event: NSEvent) {
|
||
if captured { return }
|
||
super.keyUp(with: event)
|
||
}
|
||
|
||
private func requestAutoCapture() {
|
||
pendingAutoCapture = true
|
||
attemptPendingCapture()
|
||
}
|
||
|
||
private func attemptPendingCapture() {
|
||
guard pendingAutoCapture, window != nil, bounds.width > 0 else { return }
|
||
pendingAutoCapture = false // one shot, even if the engage below is refused
|
||
engageCapture(fromClick: false)
|
||
}
|
||
|
||
private func engageCapture(fromClick: Bool) {
|
||
// A click is explicit intent AND may arrive mid-activation (acceptsFirstMouse:
|
||
// NSApp.isActive / isKeyWindow are still false for the click coming in from
|
||
// another app) — only the auto-engage paths require already-held key status.
|
||
guard captureEnabled, !captured, token != nil, window != nil,
|
||
fromClick || (NSApp.isActive && window?.isKeyWindow == true)
|
||
else { return }
|
||
cursorCapture.capture(in: self)
|
||
inputCapture?.setForwarding(true, suppressClick: fromClick)
|
||
captured = true
|
||
window?.makeFirstResponder(self)
|
||
notifyCaptureChange(true)
|
||
}
|
||
|
||
private func releaseCapture() {
|
||
guard captured else { return }
|
||
cursorCapture.release()
|
||
inputCapture?.setForwarding(false)
|
||
captured = false
|
||
notifyCaptureChange(false)
|
||
}
|
||
|
||
/// Engage/release can run inside a SwiftUI update pass (captureEnabled flips in
|
||
/// updateNSView; release in dismantleNSView) — publishing model state synchronously
|
||
/// there is undefined behavior, so the callback is deferred a runloop turn.
|
||
private func notifyCaptureChange(_ captured: Bool) {
|
||
guard let onCaptureChange else { return }
|
||
DispatchQueue.main.async { onCaptureChange(captured) }
|
||
}
|
||
|
||
// MARK: - Pump
|
||
|
||
/// Pump thread: pull AUs from the connection, wrap, enqueue. The first IDR yields the
|
||
/// format description; non-IDR AUs before it are dropped (the host opens with an IDR).
|
||
public func start(
|
||
connection: PunktfunkConnection,
|
||
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
|
||
onSessionEnd: (@Sendable () -> Void)? = nil
|
||
) {
|
||
stop()
|
||
let token = PumpToken()
|
||
self.token = token
|
||
self.connection = connection
|
||
let layer = displayLayer
|
||
layer.flush() // drop any frames a previous connection left queued
|
||
|
||
// The view owns the session's input capture: handlers attach now, but nothing is
|
||
// forwarded until capture engages (captureEnabled + auto-engage or a click).
|
||
let capture = InputCapture(connection: connection)
|
||
capture.onToggleCapture = { [weak self] in
|
||
// The ⌘⎋ monitor is app-wide — only the key window's stream owns the toggle
|
||
// (two stream windows would otherwise flip each other's capture).
|
||
guard let self, self.window?.isKeyWindow == true else { return }
|
||
if self.captured {
|
||
self.releaseCapture()
|
||
} else {
|
||
self.engageCapture(fromClick: false)
|
||
}
|
||
}
|
||
capture.onPreempted = { [weak self] in
|
||
// A newer session took the GC handler slots — staying "captured" here would
|
||
// be a cursor trap with dead input.
|
||
self?.releaseCapture()
|
||
}
|
||
capture.start()
|
||
inputCapture = capture
|
||
|
||
let thread = Thread {
|
||
var format: CMVideoFormatDescription?
|
||
while token.isLive {
|
||
do {
|
||
guard let au = try connection.nextAU(timeoutMs: 100) else { continue }
|
||
onFrame?(au)
|
||
if let f = AnnexB.formatDescription(fromIDR: au.data) {
|
||
format = f // refreshed on every IDR (mode changes included)
|
||
}
|
||
if layer.status == .failed {
|
||
// Decode wedged: flush and re-gate on the next in-band parameter
|
||
// sets — resuming with a delta frame can't recover. (A
|
||
// request-IDR channel on punktfunk/1 is a host-side TODO; with the
|
||
// host's infinite GOP this may otherwise stay black until the
|
||
// next recovery keyframe.)
|
||
layer.flush()
|
||
format = AnnexB.formatDescription(fromIDR: au.data)
|
||
}
|
||
guard let f = format,
|
||
let sample = AnnexB.sampleBuffer(au: au, format: f),
|
||
token.isLive // don't enqueue a stale frame after a restart
|
||
else { continue }
|
||
layer.enqueue(sample)
|
||
} catch {
|
||
if token.isLive {
|
||
onSessionEnd?()
|
||
}
|
||
break // session closed
|
||
}
|
||
}
|
||
}
|
||
thread.name = "punktfunk-pump"
|
||
thread.qualityOfService = .userInteractive
|
||
thread.start()
|
||
requestAutoCapture() // entering a session is the deliberate "capture me" moment
|
||
}
|
||
|
||
/// Stop pumping (≤ one poll timeout). Does not close the connection — that stays with
|
||
/// whoever owns it (PunktfunkConnection.close() is safe alongside a draining pump).
|
||
public func stop() {
|
||
releaseCapture()
|
||
inputCapture?.stop()
|
||
inputCapture = nil
|
||
token?.cancel()
|
||
token = nil
|
||
connection = nil
|
||
}
|
||
|
||
deinit {
|
||
appObservers.forEach(NotificationCenter.default.removeObserver(_:))
|
||
windowObservers.forEach(NotificationCenter.default.removeObserver(_:))
|
||
token?.cancel()
|
||
}
|
||
}
|
||
#endif
|