"Capture system shortcuts" on Mac reaches ⌘Space and ⌘Tab, with Accessibility #335

Merged
enricobuehler merged 1 commits from worktree-apple-system-shortcut-tap into main 2026-08-19 14:40:19 +00:00
6 changed files with 191 additions and 2 deletions
@@ -522,7 +522,25 @@ extension SettingsView {
} }
described(inhibitShortcutsDescription, field: "inhibit_shortcuts") { described(inhibitShortcutsDescription, field: "inhibit_shortcuts") {
Toggle("Capture system shortcuts", isOn: scoped(SettingsFields.inhibitShortcuts)) Toggle("Capture system shortcuts", isOn: scoped(SettingsFields.inhibitShortcuts))
// Turning it ON is the moment to ask for Accessibility never at stream start,
// where a TCC dialog over a captured stream would be the surprise.
.onChange(of: effective.inhibitShortcuts) { was, on in
if on, !was, !accessibilityTrusted { InputCapture.requestSystemShortcutAccess() }
}
if effective.inhibitShortcuts, !accessibilityTrusted {
Button("Allow Accessibility access…") {
InputCapture.requestSystemShortcutAccess()
// The prompt's own "Open System Settings" only shows the FIRST time the system
// asks; after that the user has to find the pane themselves open it for them.
if let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility") {
NSWorkspace.shared.open(url)
}
}
}
} }
.onReceive(NotificationCenter.default.publisher(
for: NSApplication.didBecomeActiveNotification
)) { _ in accessibilityTrusted = InputCapture.systemShortcutsAvailable }
#endif #endif
described( described(
(ModifierLayout(rawValue: effective.modifierLayout) ?? .mac).detail, (ModifierLayout(rawValue: effective.modifierLayout) ?? .mac).detail,
@@ -549,8 +567,13 @@ extension SettingsView {
if (MouseInputMode(rawValue: effective.mouseMode) ?? .capture) == .desktop { if (MouseInputMode(rawValue: effective.mouseMode) ?? .capture) == .desktop {
return "No effect under the desktop mouse model — switch Mouse input to Capture." return "No effect under the desktop mouse model — switch Mouse input to Capture."
} }
return "Sends ⌘ shortcuts to the host while captured. ⌘⎋ always stays local — it " if accessibilityTrusted {
+ "releases capture." return "Sends ⌘ shortcuts — ⌘Space, ⌘Tab and Mission Control included — to the host "
+ "while captured. ⌘⎋ always stays local — it releases capture."
}
return "Sends the app's ⌘ shortcuts (⌘Q, ⌘W, ⌘H…) to the host while captured. ⌘Space, "
+ "⌘Tab and Mission Control need Accessibility access — macOS claims them before any "
+ "app sees them. ⌘⎋ always stays local — it releases capture."
} }
/// The SELECTED mouse model explained dynamic, like the touch-mode caption. /// The SELECTED mouse model explained dynamic, like the touch-mode caption.
@@ -124,6 +124,10 @@ struct SettingsView: View {
/// instead of the app menu while captured). macOS-only: it is the one platform whose window /// instead of the app menu while captured). macOS-only: it is the one platform whose window
/// system hands a plain app no keyboard grab, so the client has to claim the chords itself. /// system hands a plain app no keyboard grab, so the client has to claim the chords itself.
@AppStorage(DefaultsKey.inhibitShortcuts) var inhibitShortcuts = true @AppStorage(DefaultsKey.inhibitShortcuts) var inhibitShortcuts = true
/// Accessibility granted? Gates the system-shortcut half of `inhibit_shortcuts` (Space, Tab
/// need the event tap). Re-read whenever the app comes back to the front that is when the
/// user returns from flipping the switch in System Settings.
@State var accessibilityTrusted = InputCapture.systemShortcutsAvailable
@AppStorage(DefaultsKey.speakerUID) var speakerUID = "" @AppStorage(DefaultsKey.speakerUID) var speakerUID = ""
@AppStorage(DefaultsKey.micUID) var micUID = "" @AppStorage(DefaultsKey.micUID) var micUID = ""
@AppStorage(DefaultsKey.micChannel) var micChannel = 0 @AppStorage(DefaultsKey.micChannel) var micChannel = 0
@@ -60,6 +60,10 @@ public final class InputCapture {
private var keyboards: [GCKeyboard] = [] private var keyboards: [GCKeyboard] = []
#if os(macOS) #if os(macOS)
private var keyEventMonitor: Any? private var keyEventMonitor: Any?
/// The system-shortcut tap (see `installSystemKeyTap`) and its run-loop source. Live only
/// while forwarding with `inhibit_shortcuts` on AND Accessibility granted; nil otherwise.
private var systemKeyTap: CFMachPort?
private var systemKeyTapSource: CFRunLoopSource?
#endif #endif
// Main-queue-only state (see header comment). // Main-queue-only state (see header comment).
@@ -194,7 +198,13 @@ public final class InputCapture {
if on { if on {
forwarding = true forwarding = true
suppressedButton = suppressClick ? 1 : nil suppressedButton = suppressClick ? 1 : nil
#if os(macOS)
installSystemKeyTap()
#endif
} else if forwarding { } else if forwarding {
#if os(macOS)
removeSystemKeyTap()
#endif
releaseAll() releaseAll()
forwarding = false forwarding = false
suppressedButton = nil suppressedButton = nil
@@ -369,6 +379,7 @@ public final class InputCapture {
NSEvent.removeMonitor(monitor) NSEvent.removeMonitor(monitor)
keyEventMonitor = nil keyEventMonitor = nil
} }
removeSystemKeyTap()
#endif #endif
// Don't clobber the handlers if a newer capture has taken the global devices. // Don't clobber the handlers if a newer capture has taken the global devices.
if Self.activeCapture === self || Self.activeCapture == nil { if Self.activeCapture === self || Self.activeCapture == nil {
@@ -672,6 +683,128 @@ public final class InputCapture {
} }
commandChordVKs.removeAll() commandChordVKs.removeAll()
} }
// MARK: - System shortcut tap
/// Whether the system-shortcut tap CAN run: Accessibility granted to this process. Read live
/// (the user flips it in System Settings while the app runs); never prompts the prompt is the
/// Settings toggle's job (`requestSystemShortcutAccess`), not something a stream start springs.
public static var systemShortcutsAvailable: Bool { AXIsProcessTrusted() }
/// Show the one-time Accessibility prompt (a no-op once granted). Called from Settings when the
/// user turns "Capture system shortcuts" on or presses the grant button.
public static func requestSystemShortcutAccess() {
let opts = [kAXTrustedCheckOptionPrompt.takeUnretainedValue(): true] as CFDictionary
_ = AXIsProcessTrustedWithOptions(opts)
}
/// The other half of `inhibit_shortcuts` on macOS. The keyDown monitor above claims the
/// chords that REACH the app but Space, Tab, and the rest of System Settings Keyboard
/// Shortcuts never do: WindowServer hands them to Spotlight / the Dock / Mission Control before
/// any app sees them. The SDL clients get those through a private CGS hotkey-mode call that a
/// sandboxed app cannot make; the sandbox-legal way is a session-level event tap, which sees
/// every key ahead of the hotkey dispatch and only exists with Accessibility granted.
///
/// The tap does NOT forward anything itself. It takes each keyDown/keyUp off the system and
/// re-posts it, addressed to the key window, into THIS app's event queue (`NSApp.postEvent`), so
/// it arrives exactly where the same key would have arrived had macOS not claimed it the
/// monitor first (client chords, chords host), then `StreamLayerView.keyDown/keyUp`
/// (everything else host). One key path, no second VK table, no second release bookkeeping.
/// In-process posts don't re-enter the tap, so there is no loop. Keys the system would have
/// delivered anyway are unaffected (we drop the original and deliver the copy) the tap only
/// changes what happens to the ones it wouldn't. Bonus: the keyUp of a -chord key now arrives
/// too (the tap sees HID, which never stopped delivering it), so `flushCommandChord` has less
/// to synthesize.
///
/// Gating, every event: `forwarding` (capture engaged and capture releases on any focus loss,
/// so this is never true with another app frontmost), `!desktopMouse` (system chords stay local
/// under the desktop model, like every other client), `NSApp.isActive` as belt-and-braces.
/// Anything else passes through untouched a tap that swallows keys for the whole Mac is the
/// failure mode to design against. Installed on the main run loop on purpose: a hung main thread
/// trips the tap's timeout and macOS disables it, handing the keyboard back.
private func installSystemKeyTap() {
// `desktopMouse` is NOT an install condition: M flips it mid-capture, so the callback
// reads it per event instead and the tap simply idles under the desktop model.
guard systemKeyTap == nil, SessionSettings.current.inhibitShortcuts, AXIsProcessTrusted()
else { return }
let mask = (1 << CGEventType.keyDown.rawValue) | (1 << CGEventType.keyUp.rawValue)
let callback: CGEventTapCallBack = { _, type, event, userInfo in
guard let userInfo else { return Unmanaged.passUnretained(event) }
let capture = Unmanaged<InputCapture>.fromOpaque(userInfo).takeUnretainedValue()
return capture.handleTapped(type: type, event: event)
}
guard let tap = CGEvent.tapCreate(
tap: .cgSessionEventTap, place: .headInsertEventTap, options: .defaultTap,
eventsOfInterest: CGEventMask(mask), callback: callback,
userInfo: Unmanaged.passUnretained(self).toOpaque())
else {
inputLog.error("system shortcut tap: tapCreate failed (Accessibility revoked?)")
return
}
let source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0)
CFRunLoopAddSource(CFRunLoopGetMain(), source, .commonModes)
CGEvent.tapEnable(tap: tap, enable: true)
systemKeyTap = tap
systemKeyTapSource = source
if inputDebug { inputLog.debug("system shortcut tap installed") }
}
private func removeSystemKeyTap() {
guard let tap = systemKeyTap else { return }
CGEvent.tapEnable(tap: tap, enable: false)
if let source = systemKeyTapSource {
CFRunLoopRemoveSource(CFRunLoopGetMain(), source, .commonModes)
}
CFMachPortInvalidate(tap)
systemKeyTap = nil
systemKeyTapSource = nil
if inputDebug { inputLog.debug("system shortcut tap removed") }
}
/// The tap callback body (main run loop). Returns the event to let it through, nil to swallow.
private func handleTapped(type: CGEventType, event: CGEvent) -> Unmanaged<CGEvent>? {
switch type {
case .tapDisabledByTimeout, .tapDisabledByUserInput:
// macOS switched us off (main thread stalled past the tap's deadline, or a
// system-level interruption); re-arm if still wanted, else stay down.
if let tap = systemKeyTap, forwarding { CGEvent.tapEnable(tap: tap, enable: true) }
return Unmanaged.passUnretained(event)
case .keyDown, .keyUp:
// Stamped with the KEY window: `NSApp.sendEvent` routes a key event by `event.window`,
// and an NSEvent wrapped straight from the CGEvent has none it reaches the local
// monitor but not the first responder (verified in a harness). The key window is the
// stream window whenever `forwarding` is true (capture releases on resignKey); if there
// somehow is none, let the key go rather than swallow it into nothing.
guard Self.tapClaims(forwarding: forwarding, desktopMouse: desktopMouse,
appActive: NSApp.isActive),
let windowNumber = NSApp.keyWindow?.windowNumber,
let copy = event.copy(), let raw = NSEvent(cgEvent: copy),
let stamped = Self.restamp(raw, windowNumber: windowNumber)
else { return Unmanaged.passUnretained(event) }
NSApp.postEvent(stamped, atStart: false)
return nil
default:
return Unmanaged.passUnretained(event)
}
}
/// The same key event, addressed to `windowNumber` (see `handleTapped`).
static func restamp(_ raw: NSEvent, windowNumber: Int) -> NSEvent? {
NSEvent.keyEvent(
with: raw.type, location: .zero, modifierFlags: raw.modifierFlags,
timestamp: raw.timestamp, windowNumber: windowNumber, context: nil,
characters: raw.characters ?? "",
charactersIgnoringModifiers: raw.charactersIgnoringModifiers ?? "",
isARepeat: raw.type == .keyDown && raw.isARepeat, keyCode: raw.keyCode)
}
/// Does the system-shortcut tap take this key off macOS and hand it to the app's own key path?
/// Pure, for the tests: only while captured, only under the capture mouse model, only with the
/// app frontmost. The `inhibit_shortcuts` setting is checked once at install time (the tap does
/// not exist with it off).
static func tapClaims(forwarding: Bool, desktopMouse: Bool, appActive: Bool) -> Bool {
forwarding && !desktopMouse && appActive
}
#endif #endif
private func attach(mouse: GCMouse) { private func attach(mouse: GCMouse) {
@@ -106,6 +106,25 @@ final class CommandChordTests: XCTestCase {
XCTAssertEqual(InputCapture.keyCodeToVK[leftArrow], 0x25) // VK_LEFT XCTAssertEqual(InputCapture.keyCodeToVK[leftArrow], 0x25) // VK_LEFT
} }
/// The system-shortcut tap (Space, Tab the keys macOS claims before the app sees them)
/// takes keys off the system ONLY while captured, under the capture mouse model, with the app
/// frontmost. Any other state must pass through: a tap that eats keys for the whole Mac is the
/// failure to pin here.
func testTheSystemShortcutTapOnlyClaimsWhileCapturedAndFrontmost() {
XCTAssertTrue(InputCapture.tapClaims(forwarding: true, desktopMouse: false, appActive: true))
XCTAssertFalse(InputCapture.tapClaims(forwarding: false, desktopMouse: false, appActive: true))
XCTAssertFalse(InputCapture.tapClaims(forwarding: true, desktopMouse: true, appActive: true))
XCTAssertFalse(InputCapture.tapClaims(forwarding: true, desktopMouse: false, appActive: false))
}
/// The keys the tap exists for must have host VKs it reposts them into the ordinary key path,
/// which drops unmapped keyCodes on the floor.
func testTheSystemShortcutKeysMapToHostVKs() {
XCTAssertEqual(InputCapture.keyCodeToVK[49], 0x20) // Space (Space)
XCTAssertEqual(InputCapture.keyCodeToVK[48], 0x09) // Tab (Tab)
XCTAssertEqual(InputCapture.keyCodeToVK[126], 0x26) // Up arrow ( Mission Control)
}
private func keyEvent(_ keyCode: UInt16, _ flags: NSEvent.ModifierFlags) -> NSEvent? { private func keyEvent(_ keyCode: UInt16, _ flags: NSEvent.ModifierFlags) -> NSEvent? {
NSEvent.keyEvent( NSEvent.keyEvent(
with: .keyDown, location: .zero, modifierFlags: flags, timestamp: 0, with: .keyDown, location: .zero, modifierFlags: flags, timestamp: 0,
+9
View File
@@ -81,6 +81,15 @@ WHY THE APP ASKS FOR WHAT IT ASKS FOR
- network.server (macOS): the app is outbound-only, but the App Sandbox gates bind() itself. Our - network.server (macOS): the app is outbound-only, but the App Sandbox gates bind() itself. Our
QUIC endpoint and UDP socket each bind a local port to receive host-to-client datagrams; QUIC endpoint and UDP socket each bind a local port to receive host-to-client datagrams;
without this, no video, audio or rumble arrives. without this, no video, audio or rumble arrives.
- Accessibility (macOS, optional, never requested unprompted): "Capture system shortcuts" in
Settings > Input lets ⌘Space, ⌘Tab and Mission Control reach the remote desktop instead of the
Mac while the stream has captured the keyboard -- the same thing every remote-desktop/VM app
offers. macOS delivers those keys to Spotlight/the Dock before any app, so the only way to
receive them is a keyboard event tap, which needs Accessibility. The prompt appears only when
the user turns the toggle on or presses "Allow Accessibility access…"; the tap exists only while
a stream has the keyboard captured and the app is frontmost, and it reads nothing -- keys are
handed to the app's own stream window, never logged or stored. Without the grant, the toggle
still works for the app's own ⌘ shortcuts and simply says the system ones need Accessibility.
- UIBackgroundModes "audio" (iPhone/iPad): a session carries real, audible audio from the host, - UIBackgroundModes "audio" (iPhone/iPad): a session carries real, audible audio from the host,
and this keeps it alive if the user steps away briefly. Backgrounded, video decoding stops, only and this keeps it alive if the user steps away briefly. Backgrounded, video decoding stops, only
the real audio keeps rendering, and a bounded timer disconnects automatically. We never play the real audio keeps rendering, and a bounded timer disconnects automatically. We never play
+1
View File
@@ -35,6 +35,7 @@ Most of this release is things that were wrong in ways nothing announced. A Dual
- **Windows: the plugin runner writes a log file you can read.** A field report on a 0.30 host had plugins installed, the runner running, an empty library and "no logs at all" — and that was by design, since the runner's only way to speak was through the host it could not reach. It now writes a plain log file next to its plugin state, and the console's empty-library hint tells you where it is. - **Windows: the plugin runner writes a log file you can read.** A field report on a 0.30 host had plugins installed, the runner running, an empty library and "no logs at all" — and that was by design, since the runner's only way to speak was through the host it could not reach. It now writes a plain log file next to its plugin state, and the console's empty-library hint tells you where it is.
- **Apple gamepad screens move like the desktop's.** Screen transitions in the Mac, iPhone, iPad and Apple TV controller shell use the same spring the desktop console uses, and they can be interrupted — press B mid-flight and the same spring carries you back. Reduce Motion crossfades instead of snapping. - **Apple gamepad screens move like the desktop's.** Screen transitions in the Mac, iPhone, iPad and Apple TV controller shell use the same spring the desktop console uses, and they can be interrupted — press B mid-flight and the same spring carries you back. Reduce Motion crossfades instead of snapping.
- **The Apple library fits a phone.** The grid fills the width instead of leaving a fifth of it empty on a phone; in a landscape phone's height it holds two rows instead of one; the shoulder-button hint hides on any phone and the sort bar has become a tray you pull down with ▲ and dismiss with ▼, A or B, so the field keeps every point of height it has. Navigating the grid no longer scrolls twice for one move, and a diagonal flick of the stick is one move, not two. - **The Apple library fits a phone.** The grid fills the width instead of leaving a fifth of it empty on a phone; in a landscape phone's height it holds two rows instead of one; the shoulder-button hint hides on any phone and the sort bar has become a tray you pull down with ▲ and dismiss with ▼, A or B, so the field keeps every point of height it has. Navigating the grid no longer scrolls twice for one move, and a diagonal flick of the stick is one move, not two.
- **Mac: "Capture system shortcuts" can now capture the system's shortcuts.** 0.29 taught the Mac client to send ⌘ shortcuts to the host, but only the ones macOS lets an app see — ⌘Q, ⌘W and their like. ⌘Space, ⌘Tab and Mission Control never got that far: macOS hands them to Spotlight and the Dock before any app is asked, so a captured stream still opened Spotlight. With Accessibility access granted, those reach the host too while your input is captured; ⌘⎋ still releases capture and stays local. The grant is optional and only ever asked for from the setting itself, never at stream start — without it, the setting keeps doing what it did in 0.29 and its caption now says exactly that.
- **The on-screen keyboard on Mac and iPad behaves.** The row you are typing into flies from its place in the list to a seat directly above the keys and back, instead of the empty row doing the flying while the real one appeared from nowhere; a hardware keyboard types straight into the field while the tray is up, including characters the on-screen grid does not offer; and Esc means Done rather than closing the whole screen. - **The on-screen keyboard on Mac and iPad behaves.** The row you are typing into flies from its place in the list to a seat directly above the keys and back, instead of the empty row doing the flying while the real one appeared from nowhere; a hardware keyboard types straight into the field while the tray is up, including characters the on-screen grid does not offer; and Esc means Done rather than closing the whole screen.
- **The controller speaker sounds like a speaker.** Both controller-audio lanes were compressed with the low-delay voice profile that suits rumble — on the speaker it sounded, in the words of the person who heard it, insanely compressed. The speaker lane now uses the full-quality music coder at a higher bitrate; the rumble lane is unchanged. - **The controller speaker sounds like a speaker.** Both controller-audio lanes were compressed with the low-delay voice profile that suits rumble — on the speaker it sounded, in the words of the person who heard it, insanely compressed. The speaker lane now uses the full-quality music coder at a higher bitrate; the rumble lane is unchanged.
- **A silent controller speaker is no longer indistinguishable from broken hardware.** On Android the controller speaker is off by default — deliberately, it is a small loudspeaker in your hands — but nothing said so, and one field session spent an evening measuring the host for a speaker that was simply switched off on the phone. The setting now states its default, and the host's controller-audio test tells you up front whether your client would even be asked to play what the test is about to prove works. - **A silent controller speaker is no longer indistinguishable from broken hardware.** On Android the controller speaker is off by default — deliberately, it is a small loudspeaker in your hands — but nothing said so, and one field session spent an evening measuring the host for a speaker that was simply switched off on the phone. The setting now states its default, and the host's controller-audio test tells you up front whether your client would even be asked to play what the test is about to prove works.