diff --git a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift index 0a7b13a7..9accfa0c 100644 --- a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift +++ b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView+Sections.swift @@ -522,7 +522,25 @@ extension SettingsView { } described(inhibitShortcutsDescription, field: "inhibit_shortcuts") { Toggle("Capture system shortcuts", isOn: scoped(SettingsFields.inhibitShortcuts)) + // Turning it ON is the moment to ask for Accessibility — never at stream start, + // where a TCC dialog over a captured stream would be the surprise. + .onChange(of: effective.inhibitShortcuts) { was, on in + if on, !was, !accessibilityTrusted { InputCapture.requestSystemShortcutAccess() } + } + if effective.inhibitShortcuts, !accessibilityTrusted { + Button("Allow Accessibility access…") { + InputCapture.requestSystemShortcutAccess() + // The prompt's own "Open System Settings" only shows the FIRST time the system + // asks; after that the user has to find the pane themselves — open it for them. + if let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility") { + NSWorkspace.shared.open(url) + } + } + } } + .onReceive(NotificationCenter.default.publisher( + for: NSApplication.didBecomeActiveNotification + )) { _ in accessibilityTrusted = InputCapture.systemShortcutsAvailable } #endif described( (ModifierLayout(rawValue: effective.modifierLayout) ?? .mac).detail, @@ -549,8 +567,13 @@ extension SettingsView { if (MouseInputMode(rawValue: effective.mouseMode) ?? .capture) == .desktop { return "No effect under the desktop mouse model — switch Mouse input to Capture." } - return "Sends ⌘ shortcuts to the host while captured. ⌘⎋ always stays local — it " - + "releases capture." + if accessibilityTrusted { + return "Sends ⌘ shortcuts — ⌘Space, ⌘Tab and Mission Control included — to the host " + + "while captured. ⌘⎋ always stays local — it releases capture." + } + return "Sends the app's ⌘ shortcuts (⌘Q, ⌘W, ⌘H…) to the host while captured. ⌘Space, " + + "⌘Tab and Mission Control need Accessibility access — macOS claims them before any " + + "app sees them. ⌘⎋ always stays local — it releases capture." } /// The SELECTED mouse model explained — dynamic, like the touch-mode caption. diff --git a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift index 1e40e287..58278049 100644 --- a/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift +++ b/clients/apple/Sources/PunktfunkClient/Settings/SettingsView.swift @@ -124,6 +124,10 @@ struct SettingsView: View { /// instead of the app menu while captured). macOS-only: it is the one platform whose window /// system hands a plain app no keyboard grab, so the client has to claim the chords itself. @AppStorage(DefaultsKey.inhibitShortcuts) var inhibitShortcuts = true + /// Accessibility granted? Gates the system-shortcut half of `inhibit_shortcuts` (⌘Space, ⌘Tab… + /// need the event tap). Re-read whenever the app comes back to the front — that is when the + /// user returns from flipping the switch in System Settings. + @State var accessibilityTrusted = InputCapture.systemShortcutsAvailable @AppStorage(DefaultsKey.speakerUID) var speakerUID = "" @AppStorage(DefaultsKey.micUID) var micUID = "" @AppStorage(DefaultsKey.micChannel) var micChannel = 0 diff --git a/clients/apple/Sources/PunktfunkKit/Input/InputCapture.swift b/clients/apple/Sources/PunktfunkKit/Input/InputCapture.swift index f6d60a1f..a2fce2c1 100644 --- a/clients/apple/Sources/PunktfunkKit/Input/InputCapture.swift +++ b/clients/apple/Sources/PunktfunkKit/Input/InputCapture.swift @@ -60,6 +60,10 @@ public final class InputCapture { private var keyboards: [GCKeyboard] = [] #if os(macOS) private var keyEventMonitor: Any? + /// The system-shortcut tap (see `installSystemKeyTap`) and its run-loop source. Live only + /// while forwarding with `inhibit_shortcuts` on AND Accessibility granted; nil otherwise. + private var systemKeyTap: CFMachPort? + private var systemKeyTapSource: CFRunLoopSource? #endif // Main-queue-only state (see header comment). @@ -194,7 +198,13 @@ public final class InputCapture { if on { forwarding = true suppressedButton = suppressClick ? 1 : nil + #if os(macOS) + installSystemKeyTap() + #endif } else if forwarding { + #if os(macOS) + removeSystemKeyTap() + #endif releaseAll() forwarding = false suppressedButton = nil @@ -369,6 +379,7 @@ public final class InputCapture { NSEvent.removeMonitor(monitor) keyEventMonitor = nil } + removeSystemKeyTap() #endif // Don't clobber the handlers if a newer capture has taken the global devices. if Self.activeCapture === self || Self.activeCapture == nil { @@ -672,6 +683,128 @@ public final class InputCapture { } commandChordVKs.removeAll() } + + // MARK: - System shortcut tap + + /// Whether the system-shortcut tap CAN run: Accessibility granted to this process. Read live + /// (the user flips it in System Settings while the app runs); never prompts — the prompt is the + /// Settings toggle's job (`requestSystemShortcutAccess`), not something a stream start springs. + public static var systemShortcutsAvailable: Bool { AXIsProcessTrusted() } + + /// Show the one-time Accessibility prompt (a no-op once granted). Called from Settings when the + /// user turns "Capture system shortcuts" on or presses the grant button. + public static func requestSystemShortcutAccess() { + let opts = [kAXTrustedCheckOptionPrompt.takeUnretainedValue(): true] as CFDictionary + _ = AXIsProcessTrustedWithOptions(opts) + } + + /// The other half of `inhibit_shortcuts` on macOS. The keyDown monitor above claims the ⌘ + /// chords that REACH the app — but ⌘Space, ⌘Tab, ⌃↑ and the rest of System Settings › Keyboard + /// › Shortcuts never do: WindowServer hands them to Spotlight / the Dock / Mission Control before + /// any app sees them. The SDL clients get those through a private CGS hotkey-mode call that a + /// sandboxed app cannot make; the sandbox-legal way is a session-level event tap, which sees + /// every key ahead of the hotkey dispatch and only exists with Accessibility granted. + /// + /// The tap does NOT forward anything itself. It takes each keyDown/keyUp off the system and + /// re-posts it, addressed to the key window, into THIS app's event queue (`NSApp.postEvent`), so + /// it arrives exactly where the same key would have arrived had macOS not claimed it — the + /// monitor first (client chords, ⌘ chords → host), then `StreamLayerView.keyDown/keyUp` + /// (everything else → host). One key path, no second VK table, no second release bookkeeping. + /// In-process posts don't re-enter the tap, so there is no loop. Keys the system would have + /// delivered anyway are unaffected (we drop the original and deliver the copy) — the tap only + /// changes what happens to the ones it wouldn't. Bonus: the keyUp of a ⌘-chord key now arrives + /// too (the tap sees HID, which never stopped delivering it), so `flushCommandChord` has less + /// to synthesize. + /// + /// Gating, every event: `forwarding` (capture engaged — and capture releases on any focus loss, + /// so this is never true with another app frontmost), `!desktopMouse` (system chords stay local + /// under the desktop model, like every other client), `NSApp.isActive` as belt-and-braces. + /// Anything else passes through untouched — a tap that swallows keys for the whole Mac is the + /// failure mode to design against. Installed on the main run loop on purpose: a hung main thread + /// trips the tap's timeout and macOS disables it, handing the keyboard back. + private func installSystemKeyTap() { + // `desktopMouse` is NOT an install condition: ⌃⌥⇧M flips it mid-capture, so the callback + // reads it per event instead and the tap simply idles under the desktop model. + guard systemKeyTap == nil, SessionSettings.current.inhibitShortcuts, AXIsProcessTrusted() + else { return } + let mask = (1 << CGEventType.keyDown.rawValue) | (1 << CGEventType.keyUp.rawValue) + let callback: CGEventTapCallBack = { _, type, event, userInfo in + guard let userInfo else { return Unmanaged.passUnretained(event) } + let capture = Unmanaged.fromOpaque(userInfo).takeUnretainedValue() + return capture.handleTapped(type: type, event: event) + } + guard let tap = CGEvent.tapCreate( + tap: .cgSessionEventTap, place: .headInsertEventTap, options: .defaultTap, + eventsOfInterest: CGEventMask(mask), callback: callback, + userInfo: Unmanaged.passUnretained(self).toOpaque()) + else { + inputLog.error("system shortcut tap: tapCreate failed (Accessibility revoked?)") + return + } + let source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) + CFRunLoopAddSource(CFRunLoopGetMain(), source, .commonModes) + CGEvent.tapEnable(tap: tap, enable: true) + systemKeyTap = tap + systemKeyTapSource = source + if inputDebug { inputLog.debug("system shortcut tap installed") } + } + + private func removeSystemKeyTap() { + guard let tap = systemKeyTap else { return } + CGEvent.tapEnable(tap: tap, enable: false) + if let source = systemKeyTapSource { + CFRunLoopRemoveSource(CFRunLoopGetMain(), source, .commonModes) + } + CFMachPortInvalidate(tap) + systemKeyTap = nil + systemKeyTapSource = nil + if inputDebug { inputLog.debug("system shortcut tap removed") } + } + + /// The tap callback body (main run loop). Returns the event to let it through, nil to swallow. + private func handleTapped(type: CGEventType, event: CGEvent) -> Unmanaged? { + switch type { + case .tapDisabledByTimeout, .tapDisabledByUserInput: + // macOS switched us off (main thread stalled past the tap's deadline, or a + // system-level interruption); re-arm if still wanted, else stay down. + if let tap = systemKeyTap, forwarding { CGEvent.tapEnable(tap: tap, enable: true) } + return Unmanaged.passUnretained(event) + case .keyDown, .keyUp: + // Stamped with the KEY window: `NSApp.sendEvent` routes a key event by `event.window`, + // and an NSEvent wrapped straight from the CGEvent has none — it reaches the local + // monitor but not the first responder (verified in a harness). The key window is the + // stream window whenever `forwarding` is true (capture releases on resignKey); if there + // somehow is none, let the key go rather than swallow it into nothing. + guard Self.tapClaims(forwarding: forwarding, desktopMouse: desktopMouse, + appActive: NSApp.isActive), + let windowNumber = NSApp.keyWindow?.windowNumber, + let copy = event.copy(), let raw = NSEvent(cgEvent: copy), + let stamped = Self.restamp(raw, windowNumber: windowNumber) + else { return Unmanaged.passUnretained(event) } + NSApp.postEvent(stamped, atStart: false) + return nil + default: + return Unmanaged.passUnretained(event) + } + } + + /// The same key event, addressed to `windowNumber` (see `handleTapped`). + static func restamp(_ raw: NSEvent, windowNumber: Int) -> NSEvent? { + NSEvent.keyEvent( + with: raw.type, location: .zero, modifierFlags: raw.modifierFlags, + timestamp: raw.timestamp, windowNumber: windowNumber, context: nil, + characters: raw.characters ?? "", + charactersIgnoringModifiers: raw.charactersIgnoringModifiers ?? "", + isARepeat: raw.type == .keyDown && raw.isARepeat, keyCode: raw.keyCode) + } + + /// Does the system-shortcut tap take this key off macOS and hand it to the app's own key path? + /// Pure, for the tests: only while captured, only under the capture mouse model, only with the + /// app frontmost. The `inhibit_shortcuts` setting is checked once at install time (the tap does + /// not exist with it off). + static func tapClaims(forwarding: Bool, desktopMouse: Bool, appActive: Bool) -> Bool { + forwarding && !desktopMouse && appActive + } #endif private func attach(mouse: GCMouse) { diff --git a/clients/apple/Tests/PunktfunkKitTests/CommandChordTests.swift b/clients/apple/Tests/PunktfunkKitTests/CommandChordTests.swift index 571ff4c5..87d7af12 100644 --- a/clients/apple/Tests/PunktfunkKitTests/CommandChordTests.swift +++ b/clients/apple/Tests/PunktfunkKitTests/CommandChordTests.swift @@ -106,6 +106,25 @@ final class CommandChordTests: XCTestCase { XCTAssertEqual(InputCapture.keyCodeToVK[leftArrow], 0x25) // VK_LEFT } + /// The system-shortcut tap (⌘Space, ⌘Tab — the keys macOS claims before the app sees them) + /// takes keys off the system ONLY while captured, under the capture mouse model, with the app + /// frontmost. Any other state must pass through: a tap that eats keys for the whole Mac is the + /// failure to pin here. + func testTheSystemShortcutTapOnlyClaimsWhileCapturedAndFrontmost() { + XCTAssertTrue(InputCapture.tapClaims(forwarding: true, desktopMouse: false, appActive: true)) + XCTAssertFalse(InputCapture.tapClaims(forwarding: false, desktopMouse: false, appActive: true)) + XCTAssertFalse(InputCapture.tapClaims(forwarding: true, desktopMouse: true, appActive: true)) + XCTAssertFalse(InputCapture.tapClaims(forwarding: true, desktopMouse: false, appActive: false)) + } + + /// The keys the tap exists for must have host VKs — it reposts them into the ordinary key path, + /// which drops unmapped keyCodes on the floor. + func testTheSystemShortcutKeysMapToHostVKs() { + XCTAssertEqual(InputCapture.keyCodeToVK[49], 0x20) // Space (⌘Space) + XCTAssertEqual(InputCapture.keyCodeToVK[48], 0x09) // Tab (⌘Tab) + XCTAssertEqual(InputCapture.keyCodeToVK[126], 0x26) // Up arrow (⌃↑ Mission Control) + } + private func keyEvent(_ keyCode: UInt16, _ flags: NSEvent.ModifierFlags) -> NSEvent? { NSEvent.keyEvent( with: .keyDown, location: .zero, modifierFlags: flags, timestamp: 0, diff --git a/clients/apple/store/review-notes.md b/clients/apple/store/review-notes.md index 41bef563..c0d5bd76 100644 --- a/clients/apple/store/review-notes.md +++ b/clients/apple/store/review-notes.md @@ -81,6 +81,15 @@ WHY THE APP ASKS FOR WHAT IT ASKS FOR - network.server (macOS): the app is outbound-only, but the App Sandbox gates bind() itself. Our QUIC endpoint and UDP socket each bind a local port to receive host-to-client datagrams; without this, no video, audio or rumble arrives. +- Accessibility (macOS, optional, never requested unprompted): "Capture system shortcuts" in + Settings > Input lets ⌘Space, ⌘Tab and Mission Control reach the remote desktop instead of the + Mac while the stream has captured the keyboard -- the same thing every remote-desktop/VM app + offers. macOS delivers those keys to Spotlight/the Dock before any app, so the only way to + receive them is a keyboard event tap, which needs Accessibility. The prompt appears only when + the user turns the toggle on or presses "Allow Accessibility access…"; the tap exists only while + a stream has the keyboard captured and the app is frontmost, and it reads nothing -- keys are + handed to the app's own stream window, never logged or stored. Without the grant, the toggle + still works for the app's own ⌘ shortcuts and simply says the system ones need Accessibility. - UIBackgroundModes "audio" (iPhone/iPad): a session carries real, audible audio from the host, and this keeps it alive if the user steps away briefly. Backgrounded, video decoding stops, only the real audio keeps rendering, and a bounded timer disconnects automatically. We never play diff --git a/docs/releases/v0.31.0.md b/docs/releases/v0.31.0.md index 5099c9ec..989aea24 100644 --- a/docs/releases/v0.31.0.md +++ b/docs/releases/v0.31.0.md @@ -35,6 +35,7 @@ Most of this release is things that were wrong in ways nothing announced. A Dual - **Windows: the plugin runner writes a log file you can read.** A field report on a 0.30 host had plugins installed, the runner running, an empty library and "no logs at all" — and that was by design, since the runner's only way to speak was through the host it could not reach. It now writes a plain log file next to its plugin state, and the console's empty-library hint tells you where it is. - **Apple gamepad screens move like the desktop's.** Screen transitions in the Mac, iPhone, iPad and Apple TV controller shell use the same spring the desktop console uses, and they can be interrupted — press B mid-flight and the same spring carries you back. Reduce Motion crossfades instead of snapping. - **The Apple library fits a phone.** The grid fills the width instead of leaving a fifth of it empty on a phone; in a landscape phone's height it holds two rows instead of one; the shoulder-button hint hides on any phone and the sort bar has become a tray you pull down with ▲ and dismiss with ▼, A or B, so the field keeps every point of height it has. Navigating the grid no longer scrolls twice for one move, and a diagonal flick of the stick is one move, not two. +- **Mac: "Capture system shortcuts" can now capture the system's shortcuts.** 0.29 taught the Mac client to send ⌘ shortcuts to the host, but only the ones macOS lets an app see — ⌘Q, ⌘W and their like. ⌘Space, ⌘Tab and Mission Control never got that far: macOS hands them to Spotlight and the Dock before any app is asked, so a captured stream still opened Spotlight. With Accessibility access granted, those reach the host too while your input is captured; ⌘⎋ still releases capture and stays local. The grant is optional and only ever asked for from the setting itself, never at stream start — without it, the setting keeps doing what it did in 0.29 and its caption now says exactly that. - **The on-screen keyboard on Mac and iPad behaves.** The row you are typing into flies from its place in the list to a seat directly above the keys and back, instead of the empty row doing the flying while the real one appeared from nowhere; a hardware keyboard types straight into the field while the tray is up, including characters the on-screen grid does not offer; and Esc means Done rather than closing the whole screen. - **The controller speaker sounds like a speaker.** Both controller-audio lanes were compressed with the low-delay voice profile that suits rumble — on the speaker it sounded, in the words of the person who heard it, insanely compressed. The speaker lane now uses the full-quality music coder at a higher bitrate; the rumble lane is unchanged. - **A silent controller speaker is no longer indistinguishable from broken hardware.** On Android the controller speaker is off by default — deliberately, it is a small loudspeaker in your hands — but nothing said so, and one field session spent an evening measuring the host for a speaker that was simply switched off on the phone. The setting now states its default, and the host's controller-audio test tells you up front whether your client would even be asked to play what the test is about to prove works.