feat(client): cross-target input handling + LAN mDNS discovery

Input handling, building on macOS/iOS/tvOS:
- macOS recapture after navigating out: engageCapture no longer latches
  captured=true when the cursor grab is refused mid app-activation (which left
  a free cursor that no later click could re-grab); cursorCapture.capture() now
  reports success. + canBecomeKeyView.
- iOS/iPadOS recapture: restore the prior capture on didBecomeActive (nothing
  re-grabbed mouse/keyboard on return before).
- iPad indirect pointer (no lock) is forwarded as an absolute MOUSE (move +
  buttons + scroll via hover / UITouch.indirectPointer), not as touch, with the
  local cursor visible; GCMouse owns the locked regime, gated so the two never
  double-send. Adds the MouseMoveAbs wire helper.
- Trackpad scroll on iOS (was entirely missing): GCMouse scroll dpad when
  locked + a scroll-only UIPanGestureRecognizer otherwise.
- tvOS: no focusable control during play (a focusable Disconnect button ate the
  controller's A in the focus engine); Siri Remote Menu disconnects.
- Don't leak touch to the host under the TOFU trust prompt (gate on
  captureEnabled).

LAN discovery: HostDiscovery (NWBrowser over _punktfunk._udp, the host's
crate::discovery advert) resolves each service to IP:port and parses the TXT
(fp advisory, pair, id); an "On this network" section in the grid (tap to save
+ connect, or pair if required). iOS/tvOS get NSBonjourServices via a merged
Config/Info.plist. Integration-tested end to end against a fake NWListener advert.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-12 14:05:21 +02:00
parent 6b4de5d738
commit 6d3ff37d9e
9 changed files with 723 additions and 83 deletions
+15
View File
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<!-- Custom keys merged into the auto-generated Info.plist (GENERATE_INFOPLIST_FILE=YES
supplies the rest). NSBonjourServices is required for NWBrowser to browse this
service type on iOS/tvOS — without it the system blocks the browse and discovery
returns nothing. Kept OUT of the synchronized App/ + Sources/ groups so it isn't
auto-added as a bundle resource (which collides with Info.plist processing). -->
<key>NSBonjourServices</key>
<array>
<string>_punktfunk._udp</string>
</array>
</dict>
</plist>
@@ -361,6 +361,7 @@
DEAD_CODE_STRIPPING = YES; DEAD_CODE_STRIPPING = YES;
DEVELOPMENT_TEAM = F4H37KF6WC; DEVELOPMENT_TEAM = F4H37KF6WC;
GENERATE_INFOPLIST_FILE = YES; GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = Config/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger"; INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
INFOPLIST_KEY_NSHumanReadableCopyright = ""; INFOPLIST_KEY_NSHumanReadableCopyright = "";
@@ -394,6 +395,7 @@
DEAD_CODE_STRIPPING = YES; DEAD_CODE_STRIPPING = YES;
DEVELOPMENT_TEAM = F4H37KF6WC; DEVELOPMENT_TEAM = F4H37KF6WC;
GENERATE_INFOPLIST_FILE = YES; GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = Config/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger"; INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
INFOPLIST_KEY_NSHumanReadableCopyright = ""; INFOPLIST_KEY_NSHumanReadableCopyright = "";
@@ -423,6 +425,7 @@
CURRENT_PROJECT_VERSION = 1; CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = F4H37KF6WC; DEVELOPMENT_TEAM = F4H37KF6WC;
GENERATE_INFOPLIST_FILE = YES; GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = Config/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger"; INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input."; INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
@@ -460,6 +463,7 @@
CURRENT_PROJECT_VERSION = 1; CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = F4H37KF6WC; DEVELOPMENT_TEAM = F4H37KF6WC;
GENERATE_INFOPLIST_FILE = YES; GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = Config/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger"; INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input."; INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
@@ -496,6 +500,7 @@
CURRENT_PROJECT_VERSION = 1; CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = F4H37KF6WC; DEVELOPMENT_TEAM = F4H37KF6WC;
GENERATE_INFOPLIST_FILE = YES; GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = Config/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger"; INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input."; INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
@@ -524,6 +529,7 @@
CURRENT_PROJECT_VERSION = 1; CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = F4H37KF6WC; DEVELOPMENT_TEAM = F4H37KF6WC;
GENERATE_INFOPLIST_FILE = YES; GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = Config/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger"; INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input."; INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
+41 -16
View File
@@ -39,9 +39,10 @@ What's here, all compiled and tested on macOS (Xcode 26.5 / Swift 6.3):
thread per view, token-cancelled so reconnects can't double-pump. thread per view, token-cancelled so reconnects can't double-pump.
- `InputCapture.swift``GCMouse` raw deltas + `GCKeyboard` HID→VK mapping (the host's - `InputCapture.swift``GCMouse` raw deltas + `GCKeyboard` HID→VK mapping (the host's
`vk_to_evdev` consumes Windows VKs), with fractional-delta accumulation so sub-pixel `vk_to_evdev` consumes Windows VKs), with fractional-delta accumulation so sub-pixel
motion isn't truncated away. Buttons use GameStream ids (1=left … 5=X2). Scroll motion isn't truncated away. Buttons use GameStream ids (1=left … 5=X2). Scroll is
arrives via the stream view's `scrollWheel` override instead of GC (trackpad/Magic WHEEL_DELTA(120)-scaled: macOS via the stream view's `scrollWheel` override, iPad via
Mouse gestures never reach GCMouse's scroll dpad), WHEEL_DELTA(120)-scaled. GCMouse's scroll dpad when pointer-locked and a scroll-only `UIPanGestureRecognizer`
otherwise (trackpad gestures never reach GC's scroll dpad).
- `GamepadManager.swift` — app-lifetime controller discovery + selection (`.shared`): - `GamepadManager.swift` — app-lifetime controller discovery + selection (`.shared`):
watches `GCController` connect/disconnect, fingerprints each pad for the Settings UI watches `GCController` connect/disconnect, fingerprints each pad for the Settings UI
(name, capabilities, battery), and selects the ONE controller forwarded to the host (name, capabilities, battery), and selects the ONE controller forwarded to the host
@@ -56,8 +57,15 @@ What's here, all compiled and tested on macOS (Xcode 26.5 / Swift 6.3):
locality) and `nextHidOutput()` (lightbar → `GCDeviceLight`, player LEDs → locality) and `nextHidOutput()` (lightbar → `GCDeviceLight`, player LEDs →
`playerIndex`, adaptive-trigger effect blocks → a total, table-driven parser → `playerIndex`, adaptive-trigger effect blocks → a total, table-driven parser →
`GCDualSenseAdaptiveTrigger`, exact for the 10-zone positional modes). `GCDualSenseAdaptiveTrigger`, exact for the 10-zone positional modes).
- **`PunktfunkClient`** (the app): hosts grid (saved in UserDefaults), "+" toolbar - `HostDiscovery.swift` — LAN auto-discovery: an `NWBrowser` over `_punktfunk._udp`
sheet to add hosts, stream mode in Settings (⌘,), two trust flows — the (the host's `crate::discovery` mDNS advert), resolving each service to an IP:port via a
throwaway `NWConnection` and parsing the TXT (`fp` advisory cert fingerprint, `pair`,
stable `id`). iOS/tvOS need `NSBonjourServices` (`Config/Info.plist`) or the system
blocks the browse.
- **`PunktfunkClient`** (the app): hosts grid (saved in UserDefaults) with an **On this
network** section listing mDNS-discovered hosts (tap to save + connect, or pair if the
host requires it), "+" toolbar sheet to add hosts manually, stream mode in Settings (⌘,),
two trust flows — the
trust-on-first-use fingerprint prompt over the live-but-blurred stream, and SPAKE2 PIN trust-on-first-use fingerprint prompt over the live-but-blurred stream, and SPAKE2 PIN
pairing (`PairSheet`, from a host card's context menu or the trust prompt; pairing (`PairSheet`, from a host card's context menu or the trust prompt;
`ClientIdentityStore` keeps the client identity in the Keychain and presents it on `ClientIdentityStore` keeps the client identity in the Keychain and presents it on
@@ -74,16 +82,25 @@ What's here, all compiled and tested on macOS (Xcode 26.5 / Swift 6.3):
("Controller type": Automatic / Xbox 360 / DualSense — Automatic matches the physical ("Controller type": Automatic / Xbox 360 / DualSense — Automatic matches the physical
pad; resolved at connect time, the host pad is fixed per session). Gamepad capture + pad; resolved at connect time, the host pad is fixed per session). Gamepad capture +
feedback run with streaming (`SessionModel` owns them, same trust gate as audio). feedback run with streaming (`SessionModel` owns them, same trust gate as audio).
Settings also sets the **Bitrate** (Automatic toggle = host default; manual is a
log-scale slider, 2 Mbps 3 Gbps, snapped to two significant figures — above 1 Gbps
an inline warning says to run a speed test first; tvOS uses a preset picker instead,
Slider doesn't exist there; negotiated via the Hello on every connect), and a host
card's context menu offers **"Test Network Speed…"** (`SpeedTestSheet`): connects, has
the host burst probe filler over the real data plane (up to the host's 3 Gbps probe
ceiling for 2 s, roadmap §9),
shows measured goodput · loss · a recommended bitrate (≈70% of measured), and applies
it in one tap.
- **Tests** (`swift test`): byte-level Annex-B units; a real-codec round trip - **Tests** (`swift test`): byte-level Annex-B units; a real-codec round trip
(VTCompressionSession-encoded HEVC rebuilt as the host's wire shape → `AnnexB` (VTCompressionSession-encoded HEVC rebuilt as the host's wire shape → `AnnexB`
VTDecompressionSession → pixels); table-driven DualSense trigger-effect parsing VTDecompressionSession → pixels); table-driven DualSense trigger-effect parsing
(`DualSenseTriggerEffectTests`) and the gamepad wire conversions (`DualSenseTriggerEffectTests`) and the gamepad wire conversions
(`GamepadWireTests`); loopback integration against real local hosts (`GamepadWireTests`); loopback integration against real local hosts
(`test-loopback.sh` — stream round trip incl. gamepad/touchpad/motion sends and a (`test-loopback.sh` — stream round trip incl. gamepad/touchpad/motion sends, a
host-scripted feedback burst asserted on the rumble + HID-output planes host-scripted feedback burst asserted on the rumble + HID-output planes
(`PUNKTFUNK_TEST_FEEDBACK=1`), plus the PIN pairing ceremony and the (`PUNKTFUNK_TEST_FEEDBACK=1`), the bitrate-negotiation echo and a real 20 Mbps
`--require-pairing` gate against a second, armed host); the remote first-light test bandwidth probe, plus the PIN pairing ceremony and the `--require-pairing` gate
above. against a second, armed host); the remote first-light test above.
## Build / run / test (on a Mac) ## Build / run / test (on a Mac)
@@ -224,23 +241,31 @@ signing, bundle id `io.unom.punktfunk`. Notes:
as the macOS one, so the SwiftUI shell is identical) hosts the shared `StreamPump` in as the macOS one, so the SwiftUI shell is identical) hosts the shared `StreamPump` in
a view controller for `prefersPointerLocked`: with a hardware mouse/trackpad that is a view controller for `prefersPointerLocked`: with a hardware mouse/trackpad that is
the iPadOS cursor capture (system honors it fullscreen-and-frontmost; in Stage the iPadOS cursor capture (system honors it fullscreen-and-frontmost; in Stage
Manager it degrades to both-cursors forwarding). Touch is always forwarded — every Manager it degrades to absolute-mouse forwarding). Input is routed by kind: DIRECT
finger gets a wire touch id and coordinates map through the aspect-fit letterbox fingers / Pencil are touches (each gets a wire touch id, coordinates mapped through the
into host-mode pixels (surface == host mode, so the host rescale is the identity). aspect-fit letterbox into host-mode pixels surface == host mode, so the host rescale is
the identity), while a mouse/trackpad is a MOUSE — pointer-LOCKED it is GCMouse relative
deltas; unlocked it is absolute moves + buttons + scroll over the UIKit pointer path
(hover + `.indirectPointer` touches), the local cursor staying visible so you can aim. An
indirect pointer is never sent as a touch. Touch is gated on trust (not forwarded under
the TOFU prompt), and returning to the foreground restores the capture you had on leaving.
`InputCapture` is cross-platform (GC works the same on iPadOS; ⌘⎋ is detected from `InputCapture` is cross-platform (GC works the same on iPadOS; ⌘⎋ is detected from
the HID stream there); audio routes via `AVAudioSession` (the Settings device the HID stream there); audio routes via `AVAudioSession` (the Settings device
pickers are macOS-only). For the iPad-with-external-display setup: the target pickers are macOS-only). For the iPad-with-external-display setup: the target
enables multiple scenes + indirect input events — on Stage Manager iPads, drag the enables multiple scenes + indirect input events — on Stage Manager iPads, drag the
punktfunk window onto the external screen and the stream runs there with full punktfunk window onto the external screen and the stream runs there with full
keyboard/mouse/touch. While streaming the session is immersive (edge-to-edge, keyboard/mouse/touch. While streaming the session is immersive (edge-to-edge,
status bar + home indicator hidden) and the iPadOS cursor is hidden over the video status bar + home indicator hidden) and the iPadOS cursor is hidden over the video only
(`UIPointerInteraction` `.hidden()` — visible again when ⌘⎋ releases capture); on while the scene is actually pointer-LOCKED (`UIPointerInteraction` `.hidden()`); when the
lock isn't held it stays visible and the mouse forwards as an absolute cursor instead; on
iOS first run the stream mode defaults to the device's native screen so the video iOS first run the stream mode defaults to the device's native screen so the video
fills the display. **tvOS** runs the same app (target **Punktfunk-tvOS**, first-lit fills the display. **tvOS** runs the same app (target **Punktfunk-tvOS**, first-lit
in the Apple TV simulator at 720p60): playback-only audio (no mic on tvOS), in the Apple TV simulator at 720p60): playback-only audio (no mic on tvOS),
focus-driven UI (`.card` host tiles), no kb/mouse capture yet — input lands with focus-driven UI (`.card` host tiles), no kb/mouse capture yet — input lands with
gamepad support, the natural tvOS input anyway; core slices are tier-3 Rust targets gamepad support, the natural tvOS input anyway. While streaming there is NO focusable
(see Build above). Known gaps: true pointer LOCK (`prefersPointerLocked`) isn't control (a focusable Disconnect button would let the focus engine eat the controller's A
before the host sees it); the Siri Remote's **Menu** button disconnects (`.onExitCommand`).
Core slices are tier-3 Rust targets (see Build above). Known gaps: true pointer LOCK (`prefersPointerLocked`) isn't
consulted through UIHostingController, so the hidden cursor can still drift onto a consulted through UIHostingController, so the hidden cursor can still drift onto a
second screen (fixing it means putting the controller into the UIKit presentation second screen (fixing it means putting the controller into the UIKit presentation
chain); and chain); and
@@ -20,13 +20,16 @@ import SwiftUINavigationTransitions
struct ContentView: View { struct ContentView: View {
@StateObject private var model = SessionModel() @StateObject private var model = SessionModel()
@StateObject private var store = HostStore() @StateObject private var store = HostStore()
@StateObject private var discovery = HostDiscovery()
@AppStorage("punktfunk.width") private var width = 1920 @AppStorage("punktfunk.width") private var width = 1920
@AppStorage("punktfunk.height") private var height = 1080 @AppStorage("punktfunk.height") private var height = 1080
@AppStorage("punktfunk.hz") private var hz = 60 @AppStorage("punktfunk.hz") private var hz = 60
@AppStorage("punktfunk.compositor") private var compositor = 0 @AppStorage("punktfunk.compositor") private var compositor = 0
@AppStorage("punktfunk.gamepadType") private var gamepadType = 0 @AppStorage("punktfunk.gamepadType") private var gamepadType = 0
@AppStorage("punktfunk.bitrateKbps") private var bitrateKbps = 0
@State private var showAddHost = false @State private var showAddHost = false
@State private var pairingTarget: StoredHost? @State private var pairingTarget: StoredHost?
@State private var speedTestTarget: StoredHost?
#if !os(macOS) #if !os(macOS)
@State private var showSettings = false @State private var showSettings = false
#endif #endif
@@ -71,6 +74,9 @@ struct ContentView: View {
connect(pinned) connect(pinned)
} }
} }
.sheet(item: $speedTestTarget) { host in
SpeedTestSheet(host: host)
}
#endif #endif
} }
@@ -104,6 +110,11 @@ struct ContentView: View {
#else #else
.background(Color.black) .background(Color.black)
.ignoresSafeArea() .ignoresSafeArea()
// Siri Remote MENU = disconnect (the idiomatic tvOS "back"). With no focusable
// disconnect control during play, the controller's buttons flow to the host instead of
// driving the focus engine. NOTE: a game controller's Menu is also forwarded to the
// host as Start the Siri Remote is the intended disconnect path.
.onExitCommand { model.disconnect() }
#endif #endif
} }
@@ -112,16 +123,21 @@ struct ContentView: View {
private var home: some View { private var home: some View {
NavigationStack { NavigationStack {
Group { Group {
if store.hosts.isEmpty { if store.hosts.isEmpty && discoveredUnsaved.isEmpty {
emptyState emptyState
} else { } else {
ScrollView { ScrollView {
LazyVGrid(columns: gridColumns, spacing: gridSpacing) { if !store.hosts.isEmpty {
ForEach(store.hosts) { host in LazyVGrid(columns: gridColumns, spacing: gridSpacing) {
hostCard(host) ForEach(store.hosts) { host in
hostCard(host)
}
} }
.padding()
}
if !discoveredUnsaved.isEmpty {
discoveredSection
} }
.padding()
#if os(tvOS) #if os(tvOS)
// Actions live below the hosts, not between them. // Actions live below the hosts, not between them.
HStack(spacing: 32) { HStack(spacing: 32) {
@@ -142,6 +158,10 @@ struct ContentView: View {
} }
} }
.navigationTitle("Punktfunkempfänger") .navigationTitle("Punktfunkempfänger")
// Browse the LAN for advertised hosts only while the grid is up not during a
// session. The home appears/disappears as the stream swaps in and out.
.onAppear { discovery.start() }
.onDisappear { discovery.stop() }
#if os(tvOS) #if os(tvOS)
// Pushed routes the Settings-app navigation feel (push animation, Menu // Pushed routes the Settings-app navigation feel (push animation, Menu
// pops) instead of modal overlays. // pops) instead of modal overlays.
@@ -160,6 +180,9 @@ struct ContentView: View {
connect(pinned) connect(pinned)
} }
} }
.navigationDestination(item: $speedTestTarget) { host in
SpeedTestSheet(host: host)
}
#endif #endif
#if !os(tvOS) #if !os(tvOS)
.toolbar { .toolbar {
@@ -354,6 +377,10 @@ struct ContentView: View {
guard !model.isBusy else { return } guard !model.isBusy else { return }
pairingTarget = host pairingTarget = host
} }
Button("Test Network Speed…") {
guard !model.isBusy else { return }
speedTestTarget = host
}
if host.pinnedSHA256 != nil { if host.pinnedSHA256 != nil {
Button("Forget Identity") { store.forgetIdentity(host) } Button("Forget Identity") { store.forgetIdentity(host) }
} }
@@ -394,7 +421,106 @@ struct ContentView: View {
rawValue: UInt32(clamping: compositor)) ?? .auto, rawValue: UInt32(clamping: compositor)) ?? .auto,
gamepad: GamepadManager.shared.resolveType( gamepad: GamepadManager.shared.resolveType(
setting: PunktfunkConnection.GamepadType( setting: PunktfunkConnection.GamepadType(
rawValue: UInt32(clamping: gamepadType)) ?? .auto)) rawValue: UInt32(clamping: gamepadType)) ?? .auto),
bitrateKbps: UInt32(clamping: bitrateKbps))
}
// MARK: - LAN discovery (mDNS)
/// Discovered hosts not already saved (matched by address+port) the saved grid shows
/// the rest, so this section only surfaces genuinely-new hosts on the network.
private var discoveredUnsaved: [DiscoveredHost] {
discovery.hosts.filter { d in
!store.hosts.contains { $0.address == d.host && $0.port == d.port }
}
}
private var discoveredSection: some View {
VStack(alignment: .leading, spacing: 10) {
Label("On this network", systemImage: "antenna.radiowaves.left.and.right")
.font(.headline)
.foregroundStyle(.secondary)
.padding(.horizontal)
LazyVGrid(columns: gridColumns, spacing: gridSpacing) {
ForEach(discoveredUnsaved) { discoveredCard($0) }
}
}
.padding([.horizontal, .bottom])
.padding(.top, store.hosts.isEmpty ? 0 : 8)
}
private func discoveredCard(_ d: DiscoveredHost) -> some View {
#if os(iOS)
let iconSize: CGFloat = 56
let iconBox: CGFloat = 76
let cardPadding: CGFloat = 28
let nameFont = Font.title3.weight(.semibold)
#else
let iconSize: CGFloat = 42
let iconBox: CGFloat = 56
let cardPadding: CGFloat = 18
let nameFont = Font.headline
#endif
return Button {
connectDiscovered(d)
} label: {
VStack(spacing: 10) {
Image(systemName: "play.display")
.font(.system(size: iconSize, weight: .light))
.foregroundStyle(.tint)
.frame(height: iconBox)
VStack(spacing: 2) {
Text(d.name)
.font(nameFont)
.lineLimit(1)
HStack(spacing: 4) {
Image(systemName: d.requiresPairing ? "lock.fill" : "wifi")
.font(.system(size: 9))
.foregroundStyle(.secondary)
Text("\(d.host):\(String(d.port))")
.font(.caption)
.foregroundStyle(.secondary)
.lineLimit(1)
}
Text(d.requiresPairing ? "Pairing required" : "Discovered")
.font(.caption2)
.foregroundStyle(.tertiary)
}
}
.frame(maxWidth: .infinity)
.padding(.vertical, cardPadding)
.padding(.horizontal, 12)
#if !os(tvOS)
// A dashed ring distinguishes a not-yet-saved discovered host from saved cards.
.background(.regularMaterial, in: RoundedRectangle(cornerRadius: 14))
.overlay {
RoundedRectangle(cornerRadius: 14)
.strokeBorder(
Color.secondary.opacity(0.25),
style: StrokeStyle(lineWidth: 1, dash: [4, 3]))
}
#endif
}
#if os(tvOS)
.buttonStyle(.card)
#else
.buttonStyle(.plain)
#endif
.disabled(model.isBusy)
}
/// Tap a discovered host: save it (so the session has a stored identity and the trust pin
/// persists), then connect TOFU shows the fingerprint, which should match the advertised
/// `fp`. A `pair=required` host goes straight to the pairing ceremony instead.
private func connectDiscovered(_ d: DiscoveredHost) {
guard !model.isBusy else { return }
let host = StoredHost(name: d.name, address: d.host, port: d.port)
store.add(host)
if d.requiresPairing {
pairingTarget = host
} else {
connect(host)
}
} }
// MARK: - Trust on first use // MARK: - Trust on first use
@@ -526,11 +652,18 @@ struct ContentView: View {
.font(.caption2) .font(.caption2)
.foregroundStyle(.secondary) .foregroundStyle(.secondary)
#endif #endif
#if os(tvOS)
// No focusable control during play: a focusable button steals the controller's
// A press (the focus engine consumes it before the host sees it). Disconnect is
// the Siri Remote's Menu button (.onExitCommand on the stream) just hint it.
Text("Press Menu to disconnect")
.font(.caption)
.foregroundStyle(.secondary)
#else
Button("Disconnect (⌘D)") { model.disconnect() } Button("Disconnect (⌘D)") { model.disconnect() }
.font(.caption) .font(.caption)
#if !os(tvOS)
.keyboardShortcut("d", modifiers: .command) .keyboardShortcut("d", modifiers: .command)
#endif #endif
} }
.padding(10) .padding(10)
.background(.regularMaterial, in: RoundedRectangle(cornerRadius: 10)) .background(.regularMaterial, in: RoundedRectangle(cornerRadius: 10))
@@ -572,12 +705,18 @@ struct ContentView: View {
let g = PunktfunkConnection.GamepadType(name: name) { let g = PunktfunkConnection.GamepadType(name: name) {
pad = g pad = g
} }
var bitrate = UInt32(clamping: bitrateKbps)
if let kbps = ProcessInfo.processInfo.environment["PUNKTFUNK_BITRATE_KBPS"],
let v = UInt32(kbps) {
bitrate = v
}
model.connect( model.connect(
to: host, to: host,
width: UInt32(clamping: width), height: UInt32(clamping: height), width: UInt32(clamping: width), height: UInt32(clamping: height),
hz: UInt32(clamping: hz), hz: UInt32(clamping: hz),
compositor: pref, compositor: pref,
gamepad: pad, gamepad: pad,
bitrateKbps: bitrate,
autoTrust: true) autoTrust: true)
} }
} }
@@ -0,0 +1,183 @@
// LAN auto-discovery of punktfunk/1 hosts over mDNS the client side of the host's
// `crate::discovery` advert (`_punktfunk._udp`). Browses with NWBrowser (TXT rides in the
// result metadata), resolves each service to a connectable IP:port with a throwaway
// NWConnection, and publishes the live set.
//
// The advertised `fp` (host cert SHA-256) is ADVISORY: mDNS is unauthenticated, so TOFU /
// pinning still verifies the host on connect it's surfaced only so a picker can show it and
// pre-fill. `pair=required` lets the UI route straight to the pairing ceremony.
//
// iOS/tvOS gate Bonjour browsing on Info.plist `NSBonjourServices` listing `_punktfunk._udp`
// (Config/Info.plist) without it the system blocks the browse and nothing is returned.
#if canImport(Network)
import Foundation
import Network
/// A punktfunk/1 host found on the LAN. `fingerprintHex` is advisory (see file header).
public struct DiscoveredHost: Identifiable, Sendable, Equatable {
/// Stable host id (mDNS `id` TXT); falls back to the Bonjour instance name.
public let id: String
/// Bonjour instance name (the host's chosen label).
public let name: String
/// Resolved address to hand to `PunktfunkConnection`.
public let host: String
public let port: UInt16
/// Host cert SHA-256 (lowercase hex) the host advertised, or nil if absent.
public let fingerprintHex: String?
/// The host advertised `pair=required` a client must pair before it can stream.
public let requiresPairing: Bool
}
@MainActor
public final class HostDiscovery: ObservableObject {
/// Currently-visible hosts, deduped by `id`, sorted by name. Main-actor.
@Published public private(set) var hosts: [DiscoveredHost] = []
private var browser: NWBrowser?
/// Keyed by the service endpoint's description (a stable, Sendable handle we can capture
/// into the resolve callbacks without smuggling non-Sendable Network types across hops).
private var resolved: [String: DiscoveredHost] = [:]
private var connections: [String: NWConnection] = [:]
public init() {}
/// Start browsing `_punktfunk._udp`. Idempotent a second call while live is a no-op.
public func start() {
guard browser == nil else { return }
let browser = NWBrowser(
for: .bonjourWithTXTRecord(type: "_punktfunk._udp", domain: nil),
using: NWParameters())
browser.browseResultsChangedHandler = { results, _ in
MainActor.assumeIsolated { [weak self] in self?.reconcile(results) }
}
browser.stateUpdateHandler = { state in
// A failed browser never recovers on its own; tear down and re-arm so transient
// network changes (Wi-Fi flip, VPN) don't leave discovery silently dead.
MainActor.assumeIsolated { [weak self] in
if case .failed = state { self?.restart() }
}
}
self.browser = browser
browser.start(queue: .main)
}
/// Stop browsing and drop all discovered state.
public func stop() {
browser?.cancel()
browser = nil
for conn in connections.values { conn.cancel() }
connections.removeAll()
resolved.removeAll()
if !hosts.isEmpty { hosts = [] }
}
deinit {
browser?.cancel()
for conn in connections.values { conn.cancel() }
}
private func restart() {
stop()
start()
}
/// Diff the browser's current result set against what we're tracking: drop departed
/// services, resolve newly-seen ones.
private func reconcile(_ results: Set<NWBrowser.Result>) {
let live = Set(results.map { Self.key($0) })
for key in resolved.keys where !live.contains(key) { resolved[key] = nil }
for key in connections.keys where !live.contains(key) {
connections[key]?.cancel()
connections[key] = nil
}
for result in results {
let key = Self.key(result)
if resolved[key] == nil, connections[key] == nil { resolve(result) }
}
publish()
}
/// Resolve one service to IP:port via a short UDP connection (it reaches `.ready` once the
/// path is established no data is sent), reading the TXT up front so the callback only
/// captures Sendable values + the endpoint key.
private func resolve(_ result: NWBrowser.Result) {
let key = Self.key(result)
let name = Self.instanceName(result.endpoint)
var fp: String?
var pair: String?
var id: String?
if case let .bonjour(txt) = result.metadata {
fp = Self.entry(txt, "fp")
pair = Self.entry(txt, "pair")
id = Self.entry(txt, "id")
}
let conn = NWConnection(to: result.endpoint, using: .udp)
connections[key] = conn
conn.stateUpdateHandler = { state in
MainActor.assumeIsolated { [weak self] in
guard let self, let conn = self.connections[key] else { return }
switch state {
case .ready:
if case let .hostPort(host, port)? = conn.currentPath?.remoteEndpoint,
let address = Self.hostString(host) {
self.resolved[key] = DiscoveredHost(
id: (id?.isEmpty == false) ? id! : name,
name: name, host: address, port: port.rawValue,
fingerprintHex: fp, requiresPairing: pair == "required")
self.publish()
}
conn.cancel()
self.connections[key] = nil
case .failed, .cancelled:
self.connections[key] = nil
default:
break
}
}
}
conn.start(queue: .main)
}
/// Publish the resolved set, deduped by `id` (a host on several interfaces / re-advertising
/// collapses to one row), sorted by name.
private func publish() {
var byID: [String: DiscoveredHost] = [:]
for host in resolved.values { byID[host.id] = host }
let next = byID.values.sorted {
$0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending
}
if next != hosts { hosts = next }
}
private static func key(_ result: NWBrowser.Result) -> String {
"\(result.endpoint)"
}
private static func instanceName(_ endpoint: NWEndpoint) -> String {
if case let .service(name, _, _, _) = endpoint { return name }
return "punktfunk host"
}
private static func entry(_ txt: NWTXTRecord, _ field: String) -> String? {
if case let .string(value) = txt.getEntry(for: field), !value.isEmpty { return value }
return nil
}
/// A resolved `NWEndpoint.Host` a plain address string for `PunktfunkConnection` (the
/// scope id on a link-local address is stripped the host+port pair is resolved again on
/// the Rust side, which can't parse the `%iface` suffix).
private static func hostString(_ host: NWEndpoint.Host) -> String? {
switch host {
case .ipv4(let address):
return "\(address)".split(separator: "%").first.map(String.init)
case .ipv6(let address):
return "\(address)".split(separator: "%").first.map(String.init)
case .name(let name, _):
return name
@unknown default:
return nil
}
}
}
#endif
@@ -98,6 +98,15 @@ public final class InputCapture {
/// window, clicking the HUD) and nothing is forwarded. Main-queue only. /// window, clicking the HUD) and nothing is forwarded. Main-queue only.
public private(set) var forwarding = false public private(set) var forwarding = false
/// iPad pointer routing (the StreamViewController mirrors the scene's live pointer-lock
/// state into this). GCMouse only delivers relative deltas + buttons while the scene is
/// LOCKED, so this is true then and the GCMouse handlers forward. When the scene can't
/// lock (Stage Manager, not frontmost, iPhone) the iPad routes the mouse through UIKit's
/// pointer path as ABSOLUTE moves (`sendMouseAbs`) instead so this goes false, gating
/// GCMouse off and enabling the absolute path, the two never double-sending. Moot on
/// macOS (no GCMouse handlers installed; `sendMouseAbs` is never called there). Main-queue.
public var gcMouseForwarding = false
/// Fired on (the capture toggle detected here so it works in both states; the /// Fired on (the capture toggle detected here so it works in both states; the
/// event itself is swallowed). Main queue. /// event itself is swallowed). Main queue.
public var onToggleCapture: (() -> Void)? public var onToggleCapture: (() -> Void)?
@@ -257,6 +266,17 @@ public final class InputCapture {
#endif #endif
} }
/// Release any held MOUSE buttons host-side, leaving keyboard state untouched. Used when
/// the iPad pointer lock drops while a GCMouse button is held: by then the GCMouse release
/// handler is gated off (`gcMouseForwarding` is false), so it can't deliver the release
/// itself and the button would otherwise stick until the next `releaseAll` (blur / stop).
public func releaseMouseButtons() {
for button in pressedButtons {
connection.send(.mouseButton(button, down: false))
}
pressedButtons.removeAll()
}
private func sendButton(_ button: UInt32, pressed: Bool) { private func sendButton(_ button: UInt32, pressed: Bool) {
guard forwarding else { return } guard forwarding else { return }
if button == suppressedButton { if button == suppressedButton {
@@ -365,7 +385,7 @@ public final class InputCapture {
// pointer lock). See the file header. // pointer lock). See the file header.
#if !os(macOS) #if !os(macOS)
input.mouseMovedHandler = { [weak self] _, dx, dy in input.mouseMovedHandler = { [weak self] _, dx, dy in
guard let self, self.forwarding else { return } guard let self, self.forwarding, self.gcMouseForwarding else { return }
// GC gives +y up; the host expects screen-space (+y down). // GC gives +y up; the host expects screen-space (+y down).
let fx = dx + self.residualX let fx = dx + self.residualX
let fy = -dy + self.residualY let fy = -dy + self.residualY
@@ -387,28 +407,40 @@ public final class InputCapture {
} }
} }
} }
// Buttons take the GCMouse path only while the scene is pointer-locked; when it
// isn't, the UIKit indirect-pointer path carries them (gcMouseForwarding gates here
// so the two can't double-send).
input.leftButton.pressedChangedHandler = { [weak self] _, _, pressed in input.leftButton.pressedChangedHandler = { [weak self] _, _, pressed in
self?.sendButton(1, pressed: pressed) guard let self, self.gcMouseForwarding else { return }
self.sendButton(1, pressed: pressed)
} }
input.rightButton?.pressedChangedHandler = { [weak self] _, _, pressed in input.rightButton?.pressedChangedHandler = { [weak self] _, _, pressed in
self?.sendButton(3, pressed: pressed) guard let self, self.gcMouseForwarding else { return }
self.sendButton(3, pressed: pressed)
} }
input.middleButton?.pressedChangedHandler = { [weak self] _, _, pressed in input.middleButton?.pressedChangedHandler = { [weak self] _, _, pressed in
self?.sendButton(2, pressed: pressed) guard let self, self.gcMouseForwarding else { return }
self.sendButton(2, pressed: pressed)
} }
// First two side buttons GameStream X1/X2. // First two side buttons GameStream X1/X2.
if let aux = input.auxiliaryButtons { if let aux = input.auxiliaryButtons {
for (i, button) in aux.prefix(2).enumerated() { for (i, button) in aux.prefix(2).enumerated() {
button.pressedChangedHandler = { [weak self] _, _, pressed in button.pressedChangedHandler = { [weak self] _, _, pressed in
self?.sendButton(UInt32(4 + i), pressed: pressed) guard let self, self.gcMouseForwarding else { return }
self.sendButton(UInt32(4 + i), pressed: pressed)
} }
} }
} }
// Scroll WHEEL (plain HID mice) while pointer-locked: GCMouse's scroll dpad reports
// wheel deltas here, +y up / +x right already the host's WHEEL convention, one unit
// per notch ×120 (WHEEL_DELTA), residual-accumulated by sendScroll. (Trackpad
// two-finger scrolling is gesture-based and does NOT reach GameController that
// arrives via the stream view's scroll pan recognizer; on macOS, via scrollWheel.)
input.scroll.valueChangedHandler = { [weak self] _, dx, dy in
guard let self, self.forwarding, self.gcMouseForwarding else { return }
self.sendScroll(dx: dx * 120, dy: dy * 120)
}
#endif #endif
// NOTE: no scroll handler here. GCMouse's scroll dpad only fires for plain HID
// wheel deltas trackpad/Magic Mouse scrolling is gesture-based and never
// reaches GameController. Scroll arrives via the stream view's scrollWheel
// override (NSEvent covers wheels too) sendScroll().
} }
/// Forward relative mouse motion (macOS). Fed by StreamLayerView's NSEvent monitor /// Forward relative mouse motion (macOS). Fed by StreamLayerView's NSEvent monitor
@@ -440,6 +472,18 @@ public final class InputCapture {
} }
} }
/// Forward an ABSOLUTE cursor position (iPad pointer fallback). Fed by the iOS stream
/// view's hover / indirect-pointer path when the scene can't pointer-lock: the host
/// places its cursor at this client-surface pixel the same letterbox mapping the touch
/// path uses. Gated by `forwarding` AND `!gcMouseForwarding` (the relative GCMouse path
/// owns motion while locked), so absolute and relative motion never both fire. No residual
/// accumulation the value is absolute, not a delta.
public func sendMouseAbs(x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32) {
guard forwarding, !gcMouseForwarding else { return }
connection.send(.mouseMoveAbs(
x: x, y: y, surfaceWidth: surfaceWidth, surfaceHeight: surfaceHeight))
}
/// Forward a scroll gesture, WHEEL_DELTA(120)-scaled (positive = up / right, /// Forward a scroll gesture, WHEEL_DELTA(120)-scaled (positive = up / right,
/// Moonlight's convention). Fed by StreamLayerView.scrollWheel the only delivery /// Moonlight's convention). Fed by StreamLayerView.scrollWheel the only delivery
/// path that covers trackpad/Magic Mouse gestures (GCMouse never reports them). /// path that covers trackpad/Magic Mouse gestures (GCMouse never reports them).
@@ -38,16 +38,21 @@ private let streamInputDebug =
private final class CursorCapture { private final class CursorCapture {
private var captured = false private var captured = false
func capture(in view: NSView) { /// Returns whether capture actually engaged. It can fail mid app-activation the click
guard !captured, let window = view.window, view.bounds.width > 0 else { return } /// that reactivates the app delivers `mouseDown` before the app is frontmost, and
/// `CGAssociateMouseAndMouseCursorPosition` is refused then so the caller must stay
/// released and let the NEXT click retry, never latching a half-captured state.
func capture(in view: NSView) -> Bool {
guard !captured, let window = view.window, view.bounds.width > 0 else { return false }
// Park the cursor mid-view so a click can't land in (and activate) another app. // Park the cursor mid-view so a click can't land in (and activate) another app.
let rectOnScreen = window.convertToScreen(view.convert(view.bounds, to: nil)) let rectOnScreen = window.convertToScreen(view.convert(view.bounds, to: nil))
let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 let primaryHeight = NSScreen.screens.first?.frame.height ?? 0
CGWarpMouseCursorPosition( CGWarpMouseCursorPosition(
CGPoint(x: rectOnScreen.midX, y: primaryHeight - rectOnScreen.midY)) CGPoint(x: rectOnScreen.midX, y: primaryHeight - rectOnScreen.midY))
CGAssociateMouseAndMouseCursorPosition(0) guard CGAssociateMouseAndMouseCursorPosition(0) == .success else { return false }
NSCursor.hide() NSCursor.hide()
captured = true captured = true
return true
} }
func release() { func release() {
@@ -194,6 +199,10 @@ public final class StreamLayerView: NSView {
/// InputCapture suppresses its press/release toward the host. Clicks while captured /// InputCapture suppresses its press/release toward the host. Clicks while captured
/// are the host's (GC forwards them) nothing to do here. /// are the host's (GC forwards them) nothing to do here.
public override func mouseDown(with event: NSEvent) { public override func mouseDown(with event: NSEvent) {
if streamInputDebug {
streamInputLog.debug(
"mouseDown: captureEnabled=\(self.captureEnabled, privacy: .public) captured=\(self.captured, privacy: .public)")
}
if captureEnabled, !captured { if captureEnabled, !captured {
engageCapture(fromClick: true) engageCapture(fromClick: true)
return return
@@ -239,6 +248,11 @@ public final class StreamLayerView: NSView {
// here as a send; -combos still arrive via performKeyEquivalent and stay functional (D). // here as a send; -combos still arrive via performKeyEquivalent and stay functional (D).
// Modifier keys never fire keyDown/keyUp they come through flagsChanged below. // Modifier keys never fire keyDown/keyUp they come through flagsChanged below.
public override var acceptsFirstResponder: Bool { true } public override var acceptsFirstResponder: Bool { true }
// A click after the app was inactive (Cmd-Tab away and back) must reach mouseDown so the
// user can re-capture the deliberate design is that becoming active does NOT auto-grab;
// you click into the video. Default NSViews aren't key-view candidates, which can drop
// that first click; opting in keeps the view a valid click/responder target.
public override var canBecomeKeyView: Bool { true }
public override func keyDown(with event: NSEvent) { public override func keyDown(with event: NSEvent) {
if captured { if captured {
if let ic = inputCapture, let vk = InputCapture.keyCodeToVK[event.keyCode] { if let ic = inputCapture, let vk = InputCapture.keyCodeToVK[event.keyCode] {
@@ -285,7 +299,10 @@ public final class StreamLayerView: NSView {
guard captureEnabled, !captured, pump != nil, window != nil, guard captureEnabled, !captured, pump != nil, window != nil,
fromClick || (NSApp.isActive && window?.isKeyWindow == true) fromClick || (NSApp.isActive && window?.isKeyWindow == true)
else { return } else { return }
cursorCapture.capture(in: self) // If the cursor grab is refused (e.g. the reactivating click arrives before the app is
// frontmost), stay released so the NEXT click retries never latch captured=true over
// a free cursor, which would make mouseDown's `!captured` guard reject every later click.
guard cursorCapture.capture(in: self) else { return }
inputCapture?.setForwarding(true, suppressClick: fromClick) inputCapture?.setForwarding(true, suppressClick: fromClick)
// Install AFTER the warp + setForwarding: the engage warp generates no forwarded // Install AFTER the warp + setForwarding: the engage warp generates no forwarded
// delta (the monitor isn't up yet), and the engage click's suppression latch is // delta (the monitor isn't up yet), and the engage click's suppression latch is
@@ -5,12 +5,21 @@
// fullscreen-and-frontmost, so in Stage Manager it degrades to Mac-style "both cursors // fullscreen-and-frontmost, so in Stage Manager it degrades to Mac-style "both cursors
// visible" forwarding). // visible" forwarding).
// //
// Touch is the primary input and is always forwarded (touching the video IS explicit // FINGER touch and INDIRECT POINTER (mouse/trackpad) are routed apart by UITouch.type.
// intent): every finger maps to a wire touch id, coordinates are mapped through the // Direct fingers (and Pencil) always forward as wire touches every finger maps to a touch
// aspect-fit letterbox into host-mode pixels, so surface == host mode and the host's // id, coordinates mapped through the aspect-fit letterbox into host-mode pixels (surface ==
// rescale is the identity. Hardware keyboard/mouse forwarding shares InputCapture with // host mode, so the host's rescale is the identity).
// macOS auto-engaged when streaming starts, toggles (detected from the HID stream; //
// there is no NSEvent monitor here). // A hardware mouse/trackpad is a pointer, not a finger. When the scene is pointer-LOCKED
// (full-screen + frontmost iPad) GCMouse delivers raw relative deltas and the system hides
// the cursor the gaming-grade path. When it CAN'T lock (Stage Manager, not frontmost,
// iPhone) the system shows its own cursor and routes the mouse through UIKit's pointer path:
// hover + indirect-pointer touches, which we forward as ABSOLUTE cursor position (+ buttons)
// so the host cursor tracks the visible local one. We never forward an indirect pointer as a
// touch doing so hid the cursor and made the host see taps instead of a moving mouse.
// GCMouse is gated off whenever the lock isn't held so the two paths can't double-send.
// Hardware keyboard forwarding shares InputCapture with macOS auto-engaged when streaming
// starts, toggles (detected from the HID stream; there is no NSEvent monitor here).
// //
// The public type is named StreamView like its macOS twin (each is platform-gated), so // The public type is named StreamView like its macOS twin (each is platform-gated), so
// the SwiftUI app layer is identical on both platforms. // the SwiftUI app layer is identical on both platforms.
@@ -82,6 +91,9 @@ public final class StreamViewController: UIViewController {
private var inputCapture: InputCapture? private var inputCapture: InputCapture?
fileprivate var captured = false fileprivate var captured = false
private var pointerInteraction: UIPointerInteraction? private var pointerInteraction: UIPointerInteraction?
/// Capture state at the last resign, restored on the next foreground otherwise the
/// mouse/keyboard stay released after navigating out and nothing re-grabs them.
private var wasCapturedOnResign = false
#endif #endif
/// Reads whether the scene's pointer is actually locked right now; nil = state /// Reads whether the scene's pointer is actually locked right now; nil = state
@@ -156,9 +168,29 @@ public final class StreamViewController: UIViewController {
let mode = connection.currentMode() let mode = connection.currentMode()
return CGSize(width: Double(mode.width), height: Double(mode.height)) return CGSize(width: Double(mode.width), height: Double(mode.height))
} }
streamView.onTouchEvent = { [weak connection] event in streamView.onTouchEvent = { [weak self, weak connection] event in
// Touch IS the intent during a trusted session, but must not leak to the host
// while a trust prompt is up (captureEnabled == false) gate it on that. The
// mouse/keyboard toggle (captured) deliberately does NOT gate touch.
guard self?.captureEnabled == true else { return }
connection?.send(event) connection?.send(event)
} }
// Indirect pointer (mouse/trackpad with no lock) absolute cursor + buttons, routed
// through InputCapture so the forwarding gate and release-on-blur apply uniformly.
streamView.onPointerMoveAbs = { [weak self] p in
self?.inputCapture?.sendMouseAbs(
x: p.x, y: p.y, surfaceWidth: p.w, surfaceHeight: p.h)
}
streamView.onPointerButton = { [weak self] button, down in
self?.inputCapture?.sendMouseButton(button, pressed: down)
}
// Trackpad two-finger / wheel scroll host scroll. The pan recognizer is the
// UNLOCKED regime; while locked, GCMouse's scroll handler owns it mirror the
// sendMouseAbs !gcMouseForwarding gate so the two can't double-send.
streamView.onScroll = { [weak self] dx, dy in
guard let self, self.inputCapture?.gcMouseForwarding == false else { return }
self.inputCapture?.sendScroll(dx: dx, dy: dy)
}
let capture = InputCapture(connection: connection) let capture = InputCapture(connection: connection)
capture.onToggleCapture = { [weak self] in capture.onToggleCapture = { [weak self] in
@@ -185,17 +217,28 @@ public final class StreamViewController: UIViewController {
observers.append(NotificationCenter.default.addObserver( observers.append(NotificationCenter.default.addObserver(
forName: UIApplication.willResignActiveNotification, object: nil, queue: .main forName: UIApplication.willResignActiveNotification, object: nil, queue: .main
) { [weak self] _ in ) { [weak self] _ in
self?.setCaptured(false) guard let self else { return }
self.wasCapturedOnResign = self.captured
self.setCaptured(false)
}) })
// The system can drop the lock without us asking (Slide Over, Stage Manager, leaving // Returning to the foreground restores the capture the user had before leaving
// foregroundActive). Surface it so the user sees, in PUNKTFUNK_INPUT_DEBUG, when // without this the mouse/keyboard stay released and nothing re-grabs them (touch
// GCMouse delivery has silently stopped and we've fallen back to touch. // always plays regardless). The macOS twin re-engages on a click into the video.
observers.append(NotificationCenter.default.addObserver(
forName: UIApplication.didBecomeActiveNotification, object: nil, queue: .main
) { [weak self] _ in
guard let self, self.wasCapturedOnResign, self.captureEnabled, self.pump != nil
else { return }
self.setCaptured(true)
})
// The system can grant or drop the lock without us asking (Slide Over, Stage Manager,
// entering/leaving foregroundActive). Re-resolve the mouse routing on every change:
// GCMouse (locked) vs the absolute UIKit pointer path (unlocked), and the
// hidden-vs-visible local cursor.
observers.append(NotificationCenter.default.addObserver( observers.append(NotificationCenter.default.addObserver(
forName: UIPointerLockState.didChangeNotification, object: nil, queue: .main forName: UIPointerLockState.didChangeNotification, object: nil, queue: .main
) { [weak self] _ in ) { [weak self] _ in
guard let self, iosInputDebug else { return } self?.syncPointerLock()
let locked = self.pointerLockEngaged().map(String.init(describing:)) ?? "unavailable"
iosInputLog.debug("pointer lock changed: isLocked=\(locked, privacy: .public)")
}) })
if captureEnabled { if captureEnabled {
@@ -212,6 +255,9 @@ public final class StreamViewController: UIViewController {
inputCapture?.stop() inputCapture?.stop()
inputCapture = nil inputCapture = nil
streamView.onTouchEvent = nil streamView.onTouchEvent = nil
streamView.onPointerMoveAbs = nil
streamView.onPointerButton = nil
streamView.onScroll = nil
streamView.currentHostMode = nil streamView.currentHostMode = nil
#endif #endif
pump?.stop() pump?.stop()
@@ -231,18 +277,35 @@ public final class StreamViewController: UIViewController {
captured = false captured = false
} }
setNeedsUpdateOfPrefersPointerLocked() setNeedsUpdateOfPrefersPointerLocked()
pointerInteraction?.invalidate() // re-resolve the hidden/visible pointer style syncPointerLock() // resolve cursor + GCMouse/absolute routing for the current state
let onCaptureChange = onCaptureChange let onCaptureChange = onCaptureChange
let captured = captured let captured = captured
DispatchQueue.main.async { [weak self] in DispatchQueue.main.async { [weak self] in
onCaptureChange?(captured) onCaptureChange?(captured)
// The lock request is async read the resolved state next turn. If it didn't // The lock request is async the resolved state can land a runloop later, and the
// engage, GCMouse won't deliver and the always-on touch path carries input. // initial grant may precede our didChange observer, so re-resolve the routing here.
if iosInputDebug, let self { self?.syncPointerLock()
let locked = self.pointerLockEngaged().map(String.init(describing:)) ?? "unavailable" }
iosInputLog.debug( }
"setCaptured(\(captured, privacy: .public)) → pointer lock isLocked=\(locked, privacy: .public)")
} /// Resolve the mouse routing for the scene's CURRENT pointer-lock state: GCMouse (relative
/// deltas + buttons) while locked, the absolute UIKit pointer path while not, and the
/// hidden-vs-visible local cursor to match. Idempotent safe to call on every lock-state
/// change and capture toggle. Main queue.
private func syncPointerLock() {
let locked = pointerLockEngaged() == true
let useGCMouse = captured && locked
// Lock dropped (or capture ended) while the GCMouse path held a button down: once
// gcMouseForwarding flips false its release handler is gated off, so flush any held
// mouse button here before the switch otherwise it sticks down on the host.
if inputCapture?.gcMouseForwarding == true, !useGCMouse {
inputCapture?.releaseMouseButtons()
}
inputCapture?.gcMouseForwarding = useGCMouse
pointerInteraction?.invalidate() // re-resolve the hidden/visible cursor for the state
if iosInputDebug {
iosInputLog.debug(
"pointer lock isLocked=\(locked, privacy: .public) captured=\(self.captured, privacy: .public)")
} }
} }
#endif #endif
@@ -258,7 +321,11 @@ extension StreamViewController: UIPointerInteractionDelegate {
public func pointerInteraction( public func pointerInteraction(
_ interaction: UIPointerInteraction, styleFor region: UIPointerRegion _ interaction: UIPointerInteraction, styleFor region: UIPointerRegion
) -> UIPointerStyle? { ) -> UIPointerStyle? {
captured ? .hidden() : nil // Hide the local cursor only when the scene is actually pointer-LOCKED then the
// host renders its own cursor from GCMouse deltas and a visible local one would just
// diverge. When the lock isn't held the cursor stays VISIBLE so the user can aim; the
// pointer is forwarded as an absolute position, both cursors tracking together.
captured && pointerLockEngaged() == true ? .hidden() : nil
} }
} }
#endif #endif
@@ -274,12 +341,26 @@ final class StreamLayerUIView: UIView {
} }
#if os(iOS) #if os(iOS)
/// Reads the LIVE negotiated mode in pixels (the touch coordinate space). /// A position already mapped into host-mode pixels, with the surface dims the host
var currentHostMode: (() -> CGSize)? /// rescales against (== host mode, so its rescale is the identity).
var onTouchEvent: ((PunktfunkInputEvent) -> Void)? struct HostPoint { let x: Int32; let y: Int32; let w: UInt32; let h: UInt32 }
/// Wire touch ids per active UITouch; ids are reused after the touch ends. /// Reads the LIVE negotiated mode in pixels (the touch/pointer coordinate space).
var currentHostMode: (() -> CGSize)?
/// Direct fingers / Pencil wire touch events.
var onTouchEvent: ((PunktfunkInputEvent) -> Void)?
/// Indirect pointer (mouse/trackpad with no lock) absolute cursor moves.
var onPointerMoveAbs: ((HostPoint) -> Void)?
/// Indirect-pointer buttons (GameStream ids: 1=left 3=right); `down` = press.
var onPointerButton: ((_ button: UInt32, _ down: Bool) -> Void)?
/// Trackpad two-finger / wheel scroll (no lock) host scroll deltas, WHEEL(120)-scaled.
var onScroll: ((_ dx: Float, _ dy: Float) -> Void)?
/// Wire touch ids per active direct UITouch; ids are reused after the touch ends.
private var touchIDs: [ObjectIdentifier: UInt32] = [:] private var touchIDs: [ObjectIdentifier: UInt32] = [:]
/// GameStream button held per active indirect-pointer touch (one click/drag session);
/// released when that touch ends.
private var pointerButtons: [ObjectIdentifier: UInt32] = [:]
#endif #endif
override init(frame: CGRect) { override init(frame: CGRect) {
@@ -287,6 +368,17 @@ final class StreamLayerUIView: UIView {
displayLayer.videoGravity = .resizeAspect displayLayer.videoGravity = .resizeAspect
#if os(iOS) #if os(iOS)
isMultipleTouchEnabled = true isMultipleTouchEnabled = true
// Button-less mouse/trackpad movement (no lock) arrives as hover, not touches
// forward it as absolute cursor moves so the host cursor tracks without a click held.
addGestureRecognizer(
UIHoverGestureRecognizer(target: self, action: #selector(handleHover)))
// Trackpad two-finger / wheel scroll a scroll-ONLY pan: allowedTouchTypes = []
// rejects finger drags (those stay host touches), allowedScrollTypesMask accepts the
// indirect scroll devices. Forwarded as host scroll deltas.
let scrollPan = UIPanGestureRecognizer(target: self, action: #selector(handleScroll))
scrollPan.allowedScrollTypesMask = .all
scrollPan.allowedTouchTypes = []
addGestureRecognizer(scrollPan)
#endif #endif
backgroundColor = .black backgroundColor = .black
} }
@@ -296,26 +388,58 @@ final class StreamLayerUIView: UIView {
#if os(iOS) #if os(iOS)
override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) { override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
forward(touches, kind: .down) route(touches, event: event, kind: .down)
} }
override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) { override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) {
forward(touches, kind: .move) route(touches, event: event, kind: .move)
} }
override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) { override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) {
forward(touches, kind: .up) route(touches, event: event, kind: .up)
} }
override func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent?) { override func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent?) {
forward(touches, kind: .up) route(touches, event: event, kind: .up)
} }
private enum TouchKind { case down, move, up } private enum TouchKind { case down, move, up }
private func forward(_ touches: Set<UITouch>, kind: TouchKind) { /// Split a touch batch by kind: an INDIRECT POINTER (mouse/trackpad with no lock) drives
guard let hostMode = currentHostMode?(), /// the host cursor as an absolute mouse; everything else (direct finger, Pencil) is a host
hostMode.width > 0, hostMode.height > 0, onTouchEvent != nil /// touch. Mixed batches are possible, so partition rather than branch on the first touch.
else { return } private func route(_ touches: Set<UITouch>, event: UIEvent?, kind: TouchKind) {
let video = AVMakeRect(aspectRatio: hostMode, insideRect: bounds) var fingers: Set<UITouch> = []
guard video.width > 0, video.height > 0 else { return } for touch in touches {
if touch.type == .indirectPointer {
handleIndirectPointer(touch, event: event, kind: kind)
} else {
fingers.insert(touch)
}
}
if !fingers.isEmpty { forwardTouches(fingers, kind: kind) }
}
/// An indirect-pointer touch is a button-held click/drag session: forward its position as
/// an absolute cursor move and its button as a mouse button (down on begin, up on end).
private func handleIndirectPointer(_ touch: UITouch, event: UIEvent?, kind: TouchKind) {
let key = ObjectIdentifier(touch)
let host = hostPoint(from: touch.location(in: self))
switch kind {
case .down:
let button = Self.gsButton(for: event?.buttonMask ?? .primary)
pointerButtons[key] = button
if let host { onPointerMoveAbs?(host) } // place the cursor, then press
onPointerButton?(button, true)
case .move:
if let host { onPointerMoveAbs?(host) }
case .up:
if let host { onPointerMoveAbs?(host) }
if let button = pointerButtons.removeValue(forKey: key) {
onPointerButton?(button, false)
}
}
}
private func forwardTouches(_ touches: Set<UITouch>, kind: TouchKind) {
guard onTouchEvent != nil else { return }
for touch in touches { for touch in touches {
let key = ObjectIdentifier(touch) let key = ObjectIdentifier(touch)
let id: UInt32 let id: UInt32
@@ -332,20 +456,53 @@ final class StreamLayerUIView: UIView {
onTouchEvent?(.touchUp(id: id)) onTouchEvent?(.touchUp(id: id))
continue continue
} }
let p = touch.location(in: self) guard let h = hostPoint(from: touch.location(in: self)) else { continue }
let x = Int32(((p.x - video.minX) / video.width * hostMode.width)
.rounded().clamped(to: 0...(hostMode.width - 1)))
let y = Int32(((p.y - video.minY) / video.height * hostMode.height)
.rounded().clamped(to: 0...(hostMode.height - 1)))
let w = UInt32(hostMode.width)
let h = UInt32(hostMode.height)
onTouchEvent?( onTouchEvent?(
kind == .down kind == .down
? .touchDown(id: id, x: x, y: y, surfaceWidth: w, surfaceHeight: h) ? .touchDown(id: id, x: h.x, y: h.y, surfaceWidth: h.w, surfaceHeight: h.h)
: .touchMove(id: id, x: x, y: y, surfaceWidth: w, surfaceHeight: h)) : .touchMove(id: id, x: h.x, y: h.y, surfaceWidth: h.w, surfaceHeight: h.h))
} }
} }
/// Button-less mouse/trackpad movement (no lock) absolute cursor move.
@objc private func handleHover(_ recognizer: UIHoverGestureRecognizer) {
switch recognizer.state {
case .began, .changed:
if let h = hostPoint(from: recognizer.location(in: self)) { onPointerMoveAbs?(h) }
default:
break
}
}
/// Trackpad / wheel scroll (no lock) host scroll deltas. The translation is consumed
/// each callback so the next is a fresh delta. Sign/scale are tunable ( one notch per
/// ~10 pt): finger up scrolls up (host +y), x passes through the host WHEEL convention.
@objc private func handleScroll(_ g: UIPanGestureRecognizer) {
guard g.state == .began || g.state == .changed else { return }
let t = g.translation(in: self)
g.setTranslation(.zero, in: self)
onScroll?(Float(t.x) * 12, Float(-t.y) * 12)
}
/// Map a view-space point through the aspect-fit letterbox into host-mode pixels; points
/// outside the video area clamp onto its edge. nil until a mode is negotiated.
private func hostPoint(from p: CGPoint) -> HostPoint? {
guard let hostMode = currentHostMode?(), hostMode.width > 0, hostMode.height > 0
else { return nil }
let video = AVMakeRect(aspectRatio: hostMode, insideRect: bounds)
guard video.width > 0, video.height > 0 else { return nil }
let x = Int32(((p.x - video.minX) / video.width * hostMode.width)
.rounded().clamped(to: 0...(hostMode.width - 1)))
let y = Int32(((p.y - video.minY) / video.height * hostMode.height)
.rounded().clamped(to: 0...(hostMode.height - 1)))
return HostPoint(x: x, y: y, w: UInt32(hostMode.width), h: UInt32(hostMode.height))
}
/// `.secondary` (right button / two-finger click) GameStream right (3); else left (1).
private static func gsButton(for mask: UIEvent.ButtonMask) -> UInt32 {
mask.contains(.secondary) ? 3 : 1
}
private func nextFreeID() -> UInt32 { private func nextFreeID() -> UInt32 {
var id: UInt32 = 0 var id: UInt32 = 0
while touchIDs.values.contains(id) { id += 1 } while touchIDs.values.contains(id) { id += 1 }
@@ -0,0 +1,54 @@
// Advertise a fake punktfunk/1 host over real mDNS (NWListener) and assert HostDiscovery's
// NWBrowser finds it, resolves an address+port, and parses the TXT (id / pair / fp). This
// exercises the whole client discovery path on the loopback/LAN; it self-skips if the test
// environment blocks Bonjour (sandboxed CI without local-network access).
import Network
import PunktfunkKit
import XCTest
final class HostDiscoveryTests: XCTestCase {
func testFindsAdvertisedHost() async throws {
let serviceName = "PunktfunkTest-\(UUID().uuidString.prefix(8))"
let uniqueid = "test-\(UUID().uuidString)"
var txt = NWTXTRecord()
txt["proto"] = "punktfunk/1"
txt["fp"] = String(repeating: "ab", count: 32) // 64 hex chars, like a real cert SHA-256
txt["pair"] = "required"
txt["id"] = uniqueid
let listener = try NWListener(using: .udp)
listener.service = NWListener.Service(
name: String(serviceName), type: "_punktfunk._udp", txtRecord: txt)
// The resolver opens a throwaway UDP flow to read the resolved endpoint accept and
// drop it so it doesn't linger.
listener.newConnectionHandler = { connection in connection.cancel() }
listener.start(queue: .global())
defer { listener.cancel() }
let discovery = await HostDiscovery()
await discovery.start()
defer { Task { await discovery.stop() } }
// Poll up to ~10s for the advert to be browsed AND resolved.
var found: DiscoveredHost?
let deadline = Date().addingTimeInterval(10)
while Date() < deadline {
if let host = await discovery.hosts.first(where: { $0.name == String(serviceName) }) {
found = host
break
}
try await Task.sleep(nanoseconds: 200_000_000)
}
guard let host = found else {
throw XCTSkip("mDNS discovery unavailable in this environment (no local network).")
}
XCTAssertEqual(host.id, uniqueid, "the stable mDNS id should key the host")
XCTAssertTrue(host.requiresPairing, "pair=required must surface as requiresPairing")
XCTAssertEqual(host.fingerprintHex, String(repeating: "ab", count: 32))
XCTAssertFalse(host.host.isEmpty, "a resolved address is required to connect")
XCTAssertGreaterThan(host.port, 0, "a resolved port is required to connect")
}
}