diff --git a/clients/apple/Config/Info.plist b/clients/apple/Config/Info.plist
new file mode 100644
index 0000000..2a8bac2
--- /dev/null
+++ b/clients/apple/Config/Info.plist
@@ -0,0 +1,15 @@
+
+
+
+
+
+ NSBonjourServices
+
+ _punktfunk._udp
+
+
+
diff --git a/clients/apple/Punktfunk.xcodeproj/project.pbxproj b/clients/apple/Punktfunk.xcodeproj/project.pbxproj
index 4598429..c610c3f 100644
--- a/clients/apple/Punktfunk.xcodeproj/project.pbxproj
+++ b/clients/apple/Punktfunk.xcodeproj/project.pbxproj
@@ -361,6 +361,7 @@
DEAD_CODE_STRIPPING = YES;
DEVELOPMENT_TEAM = F4H37KF6WC;
GENERATE_INFOPLIST_FILE = YES;
+ INFOPLIST_FILE = Config/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
INFOPLIST_KEY_NSHumanReadableCopyright = "";
@@ -394,6 +395,7 @@
DEAD_CODE_STRIPPING = YES;
DEVELOPMENT_TEAM = F4H37KF6WC;
GENERATE_INFOPLIST_FILE = YES;
+ INFOPLIST_FILE = Config/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
INFOPLIST_KEY_NSHumanReadableCopyright = "";
@@ -423,6 +425,7 @@
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = F4H37KF6WC;
GENERATE_INFOPLIST_FILE = YES;
+ INFOPLIST_FILE = Config/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
@@ -460,6 +463,7 @@
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = F4H37KF6WC;
GENERATE_INFOPLIST_FILE = YES;
+ INFOPLIST_FILE = Config/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
@@ -496,6 +500,7 @@
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = F4H37KF6WC;
GENERATE_INFOPLIST_FILE = YES;
+ INFOPLIST_FILE = Config/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
@@ -524,6 +529,7 @@
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = F4H37KF6WC;
GENERATE_INFOPLIST_FILE = YES;
+ INFOPLIST_FILE = Config/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
diff --git a/clients/apple/README.md b/clients/apple/README.md
index 4b5a9de..32b71d7 100644
--- a/clients/apple/README.md
+++ b/clients/apple/README.md
@@ -39,9 +39,10 @@ What's here, all compiled and tested on macOS (Xcode 26.5 / Swift 6.3):
thread per view, token-cancelled so reconnects can't double-pump.
- `InputCapture.swift` — `GCMouse` raw deltas + `GCKeyboard` HID→VK mapping (the host's
`vk_to_evdev` consumes Windows VKs), with fractional-delta accumulation so sub-pixel
- motion isn't truncated away. Buttons use GameStream ids (1=left … 5=X2). Scroll
- arrives via the stream view's `scrollWheel` override instead of GC (trackpad/Magic
- Mouse gestures never reach GCMouse's scroll dpad), WHEEL_DELTA(120)-scaled.
+ motion isn't truncated away. Buttons use GameStream ids (1=left … 5=X2). Scroll is
+ WHEEL_DELTA(120)-scaled: macOS via the stream view's `scrollWheel` override, iPad via
+ GCMouse's scroll dpad when pointer-locked and a scroll-only `UIPanGestureRecognizer`
+ otherwise (trackpad gestures never reach GC's scroll dpad).
- `GamepadManager.swift` — app-lifetime controller discovery + selection (`.shared`):
watches `GCController` connect/disconnect, fingerprints each pad for the Settings UI
(name, capabilities, battery), and selects the ONE controller forwarded to the host
@@ -56,8 +57,15 @@ What's here, all compiled and tested on macOS (Xcode 26.5 / Swift 6.3):
locality) and `nextHidOutput()` (lightbar → `GCDeviceLight`, player LEDs →
`playerIndex`, adaptive-trigger effect blocks → a total, table-driven parser →
`GCDualSenseAdaptiveTrigger`, exact for the 10-zone positional modes).
-- **`PunktfunkClient`** (the app): hosts grid (saved in UserDefaults), "+" toolbar
- sheet to add hosts, stream mode in Settings (⌘,), two trust flows — the
+ - `HostDiscovery.swift` — LAN auto-discovery: an `NWBrowser` over `_punktfunk._udp`
+ (the host's `crate::discovery` mDNS advert), resolving each service to an IP:port via a
+ throwaway `NWConnection` and parsing the TXT (`fp` advisory cert fingerprint, `pair`,
+ stable `id`). iOS/tvOS need `NSBonjourServices` (`Config/Info.plist`) or the system
+ blocks the browse.
+- **`PunktfunkClient`** (the app): hosts grid (saved in UserDefaults) with an **On this
+ network** section listing mDNS-discovered hosts (tap to save + connect, or pair if the
+ host requires it), "+" toolbar sheet to add hosts manually, stream mode in Settings (⌘,),
+ two trust flows — the
trust-on-first-use fingerprint prompt over the live-but-blurred stream, and SPAKE2 PIN
pairing (`PairSheet`, from a host card's context menu or the trust prompt;
`ClientIdentityStore` keeps the client identity in the Keychain and presents it on
@@ -74,16 +82,25 @@ What's here, all compiled and tested on macOS (Xcode 26.5 / Swift 6.3):
("Controller type": Automatic / Xbox 360 / DualSense — Automatic matches the physical
pad; resolved at connect time, the host pad is fixed per session). Gamepad capture +
feedback run with streaming (`SessionModel` owns them, same trust gate as audio).
+ Settings also sets the **Bitrate** (Automatic toggle = host default; manual is a
+ log-scale slider, 2 Mbps – 3 Gbps, snapped to two significant figures — above 1 Gbps
+ an inline warning says to run a speed test first; tvOS uses a preset picker instead,
+ Slider doesn't exist there; negotiated via the Hello on every connect), and a host
+ card's context menu offers **"Test Network Speed…"** (`SpeedTestSheet`): connects, has
+ the host burst probe filler over the real data plane (up to the host's 3 Gbps probe
+ ceiling for 2 s, roadmap §9),
+ shows measured goodput · loss · a recommended bitrate (≈70% of measured), and applies
+ it in one tap.
- **Tests** (`swift test`): byte-level Annex-B units; a real-codec round trip
(VTCompressionSession-encoded HEVC rebuilt as the host's wire shape → `AnnexB` →
VTDecompressionSession → pixels); table-driven DualSense trigger-effect parsing
(`DualSenseTriggerEffectTests`) and the gamepad wire conversions
(`GamepadWireTests`); loopback integration against real local hosts
- (`test-loopback.sh` — stream round trip incl. gamepad/touchpad/motion sends and a
+ (`test-loopback.sh` — stream round trip incl. gamepad/touchpad/motion sends, a
host-scripted feedback burst asserted on the rumble + HID-output planes
- (`PUNKTFUNK_TEST_FEEDBACK=1`), plus the PIN pairing ceremony and the
- `--require-pairing` gate against a second, armed host); the remote first-light test
- above.
+ (`PUNKTFUNK_TEST_FEEDBACK=1`), the bitrate-negotiation echo and a real 20 Mbps
+ bandwidth probe, plus the PIN pairing ceremony and the `--require-pairing` gate
+ against a second, armed host); the remote first-light test above.
## Build / run / test (on a Mac)
@@ -224,23 +241,31 @@ signing, bundle id `io.unom.punktfunk`. Notes:
as the macOS one, so the SwiftUI shell is identical) hosts the shared `StreamPump` in
a view controller for `prefersPointerLocked`: with a hardware mouse/trackpad that is
the iPadOS cursor capture (system honors it fullscreen-and-frontmost; in Stage
- Manager it degrades to both-cursors forwarding). Touch is always forwarded — every
- finger gets a wire touch id and coordinates map through the aspect-fit letterbox
- into host-mode pixels (surface == host mode, so the host rescale is the identity).
+ Manager it degrades to absolute-mouse forwarding). Input is routed by kind: DIRECT
+ fingers / Pencil are touches (each gets a wire touch id, coordinates mapped through the
+ aspect-fit letterbox into host-mode pixels — surface == host mode, so the host rescale is
+ the identity), while a mouse/trackpad is a MOUSE — pointer-LOCKED it is GCMouse relative
+ deltas; unlocked it is absolute moves + buttons + scroll over the UIKit pointer path
+ (hover + `.indirectPointer` touches), the local cursor staying visible so you can aim. An
+ indirect pointer is never sent as a touch. Touch is gated on trust (not forwarded under
+ the TOFU prompt), and returning to the foreground restores the capture you had on leaving.
`InputCapture` is cross-platform (GC works the same on iPadOS; ⌘⎋ is detected from
the HID stream there); audio routes via `AVAudioSession` (the Settings device
pickers are macOS-only). For the iPad-with-external-display setup: the target
enables multiple scenes + indirect input events — on Stage Manager iPads, drag the
punktfunk window onto the external screen and the stream runs there with full
keyboard/mouse/touch. While streaming the session is immersive (edge-to-edge,
- status bar + home indicator hidden) and the iPadOS cursor is hidden over the video
- (`UIPointerInteraction` `.hidden()` — visible again when ⌘⎋ releases capture); on
+ status bar + home indicator hidden) and the iPadOS cursor is hidden over the video only
+ while the scene is actually pointer-LOCKED (`UIPointerInteraction` `.hidden()`); when the
+ lock isn't held it stays visible and the mouse forwards as an absolute cursor instead; on
iOS first run the stream mode defaults to the device's native screen so the video
fills the display. **tvOS** runs the same app (target **Punktfunk-tvOS**, first-lit
in the Apple TV simulator at 720p60): playback-only audio (no mic on tvOS),
focus-driven UI (`.card` host tiles), no kb/mouse capture yet — input lands with
- gamepad support, the natural tvOS input anyway; core slices are tier-3 Rust targets
- (see Build above). Known gaps: true pointer LOCK (`prefersPointerLocked`) isn't
+ gamepad support, the natural tvOS input anyway. While streaming there is NO focusable
+ control (a focusable Disconnect button would let the focus engine eat the controller's A
+ before the host sees it); the Siri Remote's **Menu** button disconnects (`.onExitCommand`).
+ Core slices are tier-3 Rust targets (see Build above). Known gaps: true pointer LOCK (`prefersPointerLocked`) isn't
consulted through UIHostingController, so the hidden cursor can still drift onto a
second screen (fixing it means putting the controller into the UIKit presentation
chain); and
diff --git a/clients/apple/Sources/PunktfunkClient/ContentView.swift b/clients/apple/Sources/PunktfunkClient/ContentView.swift
index fc31f57..6bc3d1a 100644
--- a/clients/apple/Sources/PunktfunkClient/ContentView.swift
+++ b/clients/apple/Sources/PunktfunkClient/ContentView.swift
@@ -20,13 +20,16 @@ import SwiftUINavigationTransitions
struct ContentView: View {
@StateObject private var model = SessionModel()
@StateObject private var store = HostStore()
+ @StateObject private var discovery = HostDiscovery()
@AppStorage("punktfunk.width") private var width = 1920
@AppStorage("punktfunk.height") private var height = 1080
@AppStorage("punktfunk.hz") private var hz = 60
@AppStorage("punktfunk.compositor") private var compositor = 0
@AppStorage("punktfunk.gamepadType") private var gamepadType = 0
+ @AppStorage("punktfunk.bitrateKbps") private var bitrateKbps = 0
@State private var showAddHost = false
@State private var pairingTarget: StoredHost?
+ @State private var speedTestTarget: StoredHost?
#if !os(macOS)
@State private var showSettings = false
#endif
@@ -71,6 +74,9 @@ struct ContentView: View {
connect(pinned)
}
}
+ .sheet(item: $speedTestTarget) { host in
+ SpeedTestSheet(host: host)
+ }
#endif
}
@@ -104,6 +110,11 @@ struct ContentView: View {
#else
.background(Color.black)
.ignoresSafeArea()
+ // Siri Remote MENU = disconnect (the idiomatic tvOS "back"). With no focusable
+ // disconnect control during play, the controller's buttons flow to the host instead of
+ // driving the focus engine. NOTE: a game controller's Menu is also forwarded to the
+ // host as Start — the Siri Remote is the intended disconnect path.
+ .onExitCommand { model.disconnect() }
#endif
}
@@ -112,16 +123,21 @@ struct ContentView: View {
private var home: some View {
NavigationStack {
Group {
- if store.hosts.isEmpty {
+ if store.hosts.isEmpty && discoveredUnsaved.isEmpty {
emptyState
} else {
ScrollView {
- LazyVGrid(columns: gridColumns, spacing: gridSpacing) {
- ForEach(store.hosts) { host in
- hostCard(host)
+ if !store.hosts.isEmpty {
+ LazyVGrid(columns: gridColumns, spacing: gridSpacing) {
+ ForEach(store.hosts) { host in
+ hostCard(host)
+ }
}
+ .padding()
+ }
+ if !discoveredUnsaved.isEmpty {
+ discoveredSection
}
- .padding()
#if os(tvOS)
// Actions live below the hosts, not between them.
HStack(spacing: 32) {
@@ -142,6 +158,10 @@ struct ContentView: View {
}
}
.navigationTitle("Punktfunkempfänger")
+ // Browse the LAN for advertised hosts only while the grid is up — not during a
+ // session. The home appears/disappears as the stream swaps in and out.
+ .onAppear { discovery.start() }
+ .onDisappear { discovery.stop() }
#if os(tvOS)
// Pushed routes — the Settings-app navigation feel (push animation, Menu
// pops) instead of modal overlays.
@@ -160,6 +180,9 @@ struct ContentView: View {
connect(pinned)
}
}
+ .navigationDestination(item: $speedTestTarget) { host in
+ SpeedTestSheet(host: host)
+ }
#endif
#if !os(tvOS)
.toolbar {
@@ -354,6 +377,10 @@ struct ContentView: View {
guard !model.isBusy else { return }
pairingTarget = host
}
+ Button("Test Network Speed…") {
+ guard !model.isBusy else { return }
+ speedTestTarget = host
+ }
if host.pinnedSHA256 != nil {
Button("Forget Identity") { store.forgetIdentity(host) }
}
@@ -394,7 +421,106 @@ struct ContentView: View {
rawValue: UInt32(clamping: compositor)) ?? .auto,
gamepad: GamepadManager.shared.resolveType(
setting: PunktfunkConnection.GamepadType(
- rawValue: UInt32(clamping: gamepadType)) ?? .auto))
+ rawValue: UInt32(clamping: gamepadType)) ?? .auto),
+ bitrateKbps: UInt32(clamping: bitrateKbps))
+ }
+
+ // MARK: - LAN discovery (mDNS)
+
+ /// Discovered hosts not already saved (matched by address+port) — the saved grid shows
+ /// the rest, so this section only surfaces genuinely-new hosts on the network.
+ private var discoveredUnsaved: [DiscoveredHost] {
+ discovery.hosts.filter { d in
+ !store.hosts.contains { $0.address == d.host && $0.port == d.port }
+ }
+ }
+
+ private var discoveredSection: some View {
+ VStack(alignment: .leading, spacing: 10) {
+ Label("On this network", systemImage: "antenna.radiowaves.left.and.right")
+ .font(.headline)
+ .foregroundStyle(.secondary)
+ .padding(.horizontal)
+ LazyVGrid(columns: gridColumns, spacing: gridSpacing) {
+ ForEach(discoveredUnsaved) { discoveredCard($0) }
+ }
+ }
+ .padding([.horizontal, .bottom])
+ .padding(.top, store.hosts.isEmpty ? 0 : 8)
+ }
+
+ private func discoveredCard(_ d: DiscoveredHost) -> some View {
+ #if os(iOS)
+ let iconSize: CGFloat = 56
+ let iconBox: CGFloat = 76
+ let cardPadding: CGFloat = 28
+ let nameFont = Font.title3.weight(.semibold)
+ #else
+ let iconSize: CGFloat = 42
+ let iconBox: CGFloat = 56
+ let cardPadding: CGFloat = 18
+ let nameFont = Font.headline
+ #endif
+ return Button {
+ connectDiscovered(d)
+ } label: {
+ VStack(spacing: 10) {
+ Image(systemName: "play.display")
+ .font(.system(size: iconSize, weight: .light))
+ .foregroundStyle(.tint)
+ .frame(height: iconBox)
+ VStack(spacing: 2) {
+ Text(d.name)
+ .font(nameFont)
+ .lineLimit(1)
+ HStack(spacing: 4) {
+ Image(systemName: d.requiresPairing ? "lock.fill" : "wifi")
+ .font(.system(size: 9))
+ .foregroundStyle(.secondary)
+ Text("\(d.host):\(String(d.port))")
+ .font(.caption)
+ .foregroundStyle(.secondary)
+ .lineLimit(1)
+ }
+ Text(d.requiresPairing ? "Pairing required" : "Discovered")
+ .font(.caption2)
+ .foregroundStyle(.tertiary)
+ }
+ }
+ .frame(maxWidth: .infinity)
+ .padding(.vertical, cardPadding)
+ .padding(.horizontal, 12)
+ #if !os(tvOS)
+ // A dashed ring distinguishes a not-yet-saved discovered host from saved cards.
+ .background(.regularMaterial, in: RoundedRectangle(cornerRadius: 14))
+ .overlay {
+ RoundedRectangle(cornerRadius: 14)
+ .strokeBorder(
+ Color.secondary.opacity(0.25),
+ style: StrokeStyle(lineWidth: 1, dash: [4, 3]))
+ }
+ #endif
+ }
+ #if os(tvOS)
+ .buttonStyle(.card)
+ #else
+ .buttonStyle(.plain)
+ #endif
+ .disabled(model.isBusy)
+ }
+
+ /// Tap a discovered host: save it (so the session has a stored identity and the trust pin
+ /// persists), then connect — TOFU shows the fingerprint, which should match the advertised
+ /// `fp`. A `pair=required` host goes straight to the pairing ceremony instead.
+ private func connectDiscovered(_ d: DiscoveredHost) {
+ guard !model.isBusy else { return }
+ let host = StoredHost(name: d.name, address: d.host, port: d.port)
+ store.add(host)
+ if d.requiresPairing {
+ pairingTarget = host
+ } else {
+ connect(host)
+ }
}
// MARK: - Trust on first use
@@ -526,11 +652,18 @@ struct ContentView: View {
.font(.caption2)
.foregroundStyle(.secondary)
#endif
+ #if os(tvOS)
+ // No focusable control during play: a focusable button steals the controller's
+ // A press (the focus engine consumes it before the host sees it). Disconnect is
+ // the Siri Remote's Menu button (.onExitCommand on the stream) — just hint it.
+ Text("Press Menu to disconnect")
+ .font(.caption)
+ .foregroundStyle(.secondary)
+ #else
Button("Disconnect (⌘D)") { model.disconnect() }
.font(.caption)
- #if !os(tvOS)
.keyboardShortcut("d", modifiers: .command)
- #endif
+ #endif
}
.padding(10)
.background(.regularMaterial, in: RoundedRectangle(cornerRadius: 10))
@@ -572,12 +705,18 @@ struct ContentView: View {
let g = PunktfunkConnection.GamepadType(name: name) {
pad = g
}
+ var bitrate = UInt32(clamping: bitrateKbps)
+ if let kbps = ProcessInfo.processInfo.environment["PUNKTFUNK_BITRATE_KBPS"],
+ let v = UInt32(kbps) {
+ bitrate = v
+ }
model.connect(
to: host,
width: UInt32(clamping: width), height: UInt32(clamping: height),
hz: UInt32(clamping: hz),
compositor: pref,
gamepad: pad,
+ bitrateKbps: bitrate,
autoTrust: true)
}
}
diff --git a/clients/apple/Sources/PunktfunkKit/HostDiscovery.swift b/clients/apple/Sources/PunktfunkKit/HostDiscovery.swift
new file mode 100644
index 0000000..e7083b3
--- /dev/null
+++ b/clients/apple/Sources/PunktfunkKit/HostDiscovery.swift
@@ -0,0 +1,183 @@
+// LAN auto-discovery of punktfunk/1 hosts over mDNS — the client side of the host's
+// `crate::discovery` advert (`_punktfunk._udp`). Browses with NWBrowser (TXT rides in the
+// result metadata), resolves each service to a connectable IP:port with a throwaway
+// NWConnection, and publishes the live set.
+//
+// The advertised `fp` (host cert SHA-256) is ADVISORY: mDNS is unauthenticated, so TOFU /
+// pinning still verifies the host on connect — it's surfaced only so a picker can show it and
+// pre-fill. `pair=required` lets the UI route straight to the pairing ceremony.
+//
+// iOS/tvOS gate Bonjour browsing on Info.plist `NSBonjourServices` listing `_punktfunk._udp`
+// (Config/Info.plist) — without it the system blocks the browse and nothing is returned.
+
+#if canImport(Network)
+import Foundation
+import Network
+
+/// A punktfunk/1 host found on the LAN. `fingerprintHex` is advisory (see file header).
+public struct DiscoveredHost: Identifiable, Sendable, Equatable {
+ /// Stable host id (mDNS `id` TXT); falls back to the Bonjour instance name.
+ public let id: String
+ /// Bonjour instance name (the host's chosen label).
+ public let name: String
+ /// Resolved address to hand to `PunktfunkConnection`.
+ public let host: String
+ public let port: UInt16
+ /// Host cert SHA-256 (lowercase hex) the host advertised, or nil if absent.
+ public let fingerprintHex: String?
+ /// The host advertised `pair=required` — a client must pair before it can stream.
+ public let requiresPairing: Bool
+}
+
+@MainActor
+public final class HostDiscovery: ObservableObject {
+ /// Currently-visible hosts, deduped by `id`, sorted by name. Main-actor.
+ @Published public private(set) var hosts: [DiscoveredHost] = []
+
+ private var browser: NWBrowser?
+ /// Keyed by the service endpoint's description (a stable, Sendable handle we can capture
+ /// into the resolve callbacks without smuggling non-Sendable Network types across hops).
+ private var resolved: [String: DiscoveredHost] = [:]
+ private var connections: [String: NWConnection] = [:]
+
+ public init() {}
+
+ /// Start browsing `_punktfunk._udp`. Idempotent — a second call while live is a no-op.
+ public func start() {
+ guard browser == nil else { return }
+ let browser = NWBrowser(
+ for: .bonjourWithTXTRecord(type: "_punktfunk._udp", domain: nil),
+ using: NWParameters())
+ browser.browseResultsChangedHandler = { results, _ in
+ MainActor.assumeIsolated { [weak self] in self?.reconcile(results) }
+ }
+ browser.stateUpdateHandler = { state in
+ // A failed browser never recovers on its own; tear down and re-arm so transient
+ // network changes (Wi-Fi flip, VPN) don't leave discovery silently dead.
+ MainActor.assumeIsolated { [weak self] in
+ if case .failed = state { self?.restart() }
+ }
+ }
+ self.browser = browser
+ browser.start(queue: .main)
+ }
+
+ /// Stop browsing and drop all discovered state.
+ public func stop() {
+ browser?.cancel()
+ browser = nil
+ for conn in connections.values { conn.cancel() }
+ connections.removeAll()
+ resolved.removeAll()
+ if !hosts.isEmpty { hosts = [] }
+ }
+
+ deinit {
+ browser?.cancel()
+ for conn in connections.values { conn.cancel() }
+ }
+
+ private func restart() {
+ stop()
+ start()
+ }
+
+ /// Diff the browser's current result set against what we're tracking: drop departed
+ /// services, resolve newly-seen ones.
+ private func reconcile(_ results: Set) {
+ let live = Set(results.map { Self.key($0) })
+ for key in resolved.keys where !live.contains(key) { resolved[key] = nil }
+ for key in connections.keys where !live.contains(key) {
+ connections[key]?.cancel()
+ connections[key] = nil
+ }
+ for result in results {
+ let key = Self.key(result)
+ if resolved[key] == nil, connections[key] == nil { resolve(result) }
+ }
+ publish()
+ }
+
+ /// Resolve one service to IP:port via a short UDP connection (it reaches `.ready` once the
+ /// path is established — no data is sent), reading the TXT up front so the callback only
+ /// captures Sendable values + the endpoint key.
+ private func resolve(_ result: NWBrowser.Result) {
+ let key = Self.key(result)
+ let name = Self.instanceName(result.endpoint)
+ var fp: String?
+ var pair: String?
+ var id: String?
+ if case let .bonjour(txt) = result.metadata {
+ fp = Self.entry(txt, "fp")
+ pair = Self.entry(txt, "pair")
+ id = Self.entry(txt, "id")
+ }
+ let conn = NWConnection(to: result.endpoint, using: .udp)
+ connections[key] = conn
+ conn.stateUpdateHandler = { state in
+ MainActor.assumeIsolated { [weak self] in
+ guard let self, let conn = self.connections[key] else { return }
+ switch state {
+ case .ready:
+ if case let .hostPort(host, port)? = conn.currentPath?.remoteEndpoint,
+ let address = Self.hostString(host) {
+ self.resolved[key] = DiscoveredHost(
+ id: (id?.isEmpty == false) ? id! : name,
+ name: name, host: address, port: port.rawValue,
+ fingerprintHex: fp, requiresPairing: pair == "required")
+ self.publish()
+ }
+ conn.cancel()
+ self.connections[key] = nil
+ case .failed, .cancelled:
+ self.connections[key] = nil
+ default:
+ break
+ }
+ }
+ }
+ conn.start(queue: .main)
+ }
+
+ /// Publish the resolved set, deduped by `id` (a host on several interfaces / re-advertising
+ /// collapses to one row), sorted by name.
+ private func publish() {
+ var byID: [String: DiscoveredHost] = [:]
+ for host in resolved.values { byID[host.id] = host }
+ let next = byID.values.sorted {
+ $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending
+ }
+ if next != hosts { hosts = next }
+ }
+
+ private static func key(_ result: NWBrowser.Result) -> String {
+ "\(result.endpoint)"
+ }
+
+ private static func instanceName(_ endpoint: NWEndpoint) -> String {
+ if case let .service(name, _, _, _) = endpoint { return name }
+ return "punktfunk host"
+ }
+
+ private static func entry(_ txt: NWTXTRecord, _ field: String) -> String? {
+ if case let .string(value) = txt.getEntry(for: field), !value.isEmpty { return value }
+ return nil
+ }
+
+ /// A resolved `NWEndpoint.Host` → a plain address string for `PunktfunkConnection` (the
+ /// scope id on a link-local address is stripped — the host+port pair is resolved again on
+ /// the Rust side, which can't parse the `%iface` suffix).
+ private static func hostString(_ host: NWEndpoint.Host) -> String? {
+ switch host {
+ case .ipv4(let address):
+ return "\(address)".split(separator: "%").first.map(String.init)
+ case .ipv6(let address):
+ return "\(address)".split(separator: "%").first.map(String.init)
+ case .name(let name, _):
+ return name
+ @unknown default:
+ return nil
+ }
+ }
+}
+#endif
diff --git a/clients/apple/Sources/PunktfunkKit/InputCapture.swift b/clients/apple/Sources/PunktfunkKit/InputCapture.swift
index 794fe43..e484a58 100644
--- a/clients/apple/Sources/PunktfunkKit/InputCapture.swift
+++ b/clients/apple/Sources/PunktfunkKit/InputCapture.swift
@@ -98,6 +98,15 @@ public final class InputCapture {
/// window, clicking the HUD) and nothing is forwarded. Main-queue only.
public private(set) var forwarding = false
+ /// iPad pointer routing (the StreamViewController mirrors the scene's live pointer-lock
+ /// state into this). GCMouse only delivers relative deltas + buttons while the scene is
+ /// LOCKED, so this is true then and the GCMouse handlers forward. When the scene can't
+ /// lock (Stage Manager, not frontmost, iPhone) the iPad routes the mouse through UIKit's
+ /// pointer path as ABSOLUTE moves (`sendMouseAbs`) instead — so this goes false, gating
+ /// GCMouse off and enabling the absolute path, the two never double-sending. Moot on
+ /// macOS (no GCMouse handlers installed; `sendMouseAbs` is never called there). Main-queue.
+ public var gcMouseForwarding = false
+
/// Fired on ⌘⎋ (the capture toggle — detected here so it works in both states; the
/// event itself is swallowed). Main queue.
public var onToggleCapture: (() -> Void)?
@@ -257,6 +266,17 @@ public final class InputCapture {
#endif
}
+ /// Release any held MOUSE buttons host-side, leaving keyboard state untouched. Used when
+ /// the iPad pointer lock drops while a GCMouse button is held: by then the GCMouse release
+ /// handler is gated off (`gcMouseForwarding` is false), so it can't deliver the release
+ /// itself and the button would otherwise stick until the next `releaseAll` (blur / stop).
+ public func releaseMouseButtons() {
+ for button in pressedButtons {
+ connection.send(.mouseButton(button, down: false))
+ }
+ pressedButtons.removeAll()
+ }
+
private func sendButton(_ button: UInt32, pressed: Bool) {
guard forwarding else { return }
if button == suppressedButton {
@@ -365,7 +385,7 @@ public final class InputCapture {
// pointer lock). See the file header.
#if !os(macOS)
input.mouseMovedHandler = { [weak self] _, dx, dy in
- guard let self, self.forwarding else { return }
+ guard let self, self.forwarding, self.gcMouseForwarding else { return }
// GC gives +y up; the host expects screen-space (+y down).
let fx = dx + self.residualX
let fy = -dy + self.residualY
@@ -387,28 +407,40 @@ public final class InputCapture {
}
}
}
+ // Buttons take the GCMouse path only while the scene is pointer-locked; when it
+ // isn't, the UIKit indirect-pointer path carries them (gcMouseForwarding gates here
+ // so the two can't double-send).
input.leftButton.pressedChangedHandler = { [weak self] _, _, pressed in
- self?.sendButton(1, pressed: pressed)
+ guard let self, self.gcMouseForwarding else { return }
+ self.sendButton(1, pressed: pressed)
}
input.rightButton?.pressedChangedHandler = { [weak self] _, _, pressed in
- self?.sendButton(3, pressed: pressed)
+ guard let self, self.gcMouseForwarding else { return }
+ self.sendButton(3, pressed: pressed)
}
input.middleButton?.pressedChangedHandler = { [weak self] _, _, pressed in
- self?.sendButton(2, pressed: pressed)
+ guard let self, self.gcMouseForwarding else { return }
+ self.sendButton(2, pressed: pressed)
}
// First two side buttons → GameStream X1/X2.
if let aux = input.auxiliaryButtons {
for (i, button) in aux.prefix(2).enumerated() {
button.pressedChangedHandler = { [weak self] _, _, pressed in
- self?.sendButton(UInt32(4 + i), pressed: pressed)
+ guard let self, self.gcMouseForwarding else { return }
+ self.sendButton(UInt32(4 + i), pressed: pressed)
}
}
}
+ // Scroll WHEEL (plain HID mice) while pointer-locked: GCMouse's scroll dpad reports
+ // wheel deltas here, +y up / +x right — already the host's WHEEL convention, one unit
+ // per notch → ×120 (WHEEL_DELTA), residual-accumulated by sendScroll. (Trackpad
+ // two-finger scrolling is gesture-based and does NOT reach GameController — that
+ // arrives via the stream view's scroll pan recognizer; on macOS, via scrollWheel.)
+ input.scroll.valueChangedHandler = { [weak self] _, dx, dy in
+ guard let self, self.forwarding, self.gcMouseForwarding else { return }
+ self.sendScroll(dx: dx * 120, dy: dy * 120)
+ }
#endif
- // NOTE: no scroll handler here. GCMouse's scroll dpad only fires for plain HID
- // wheel deltas — trackpad/Magic Mouse scrolling is gesture-based and never
- // reaches GameController. Scroll arrives via the stream view's scrollWheel
- // override (NSEvent covers wheels too) → sendScroll().
}
/// Forward relative mouse motion (macOS). Fed by StreamLayerView's NSEvent monitor —
@@ -440,6 +472,18 @@ public final class InputCapture {
}
}
+ /// Forward an ABSOLUTE cursor position (iPad pointer fallback). Fed by the iOS stream
+ /// view's hover / indirect-pointer path when the scene can't pointer-lock: the host
+ /// places its cursor at this client-surface pixel — the same letterbox mapping the touch
+ /// path uses. Gated by `forwarding` AND `!gcMouseForwarding` (the relative GCMouse path
+ /// owns motion while locked), so absolute and relative motion never both fire. No residual
+ /// accumulation — the value is absolute, not a delta.
+ public func sendMouseAbs(x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32) {
+ guard forwarding, !gcMouseForwarding else { return }
+ connection.send(.mouseMoveAbs(
+ x: x, y: y, surfaceWidth: surfaceWidth, surfaceHeight: surfaceHeight))
+ }
+
/// Forward a scroll gesture, WHEEL_DELTA(120)-scaled (positive = up / right,
/// Moonlight's convention). Fed by StreamLayerView.scrollWheel — the only delivery
/// path that covers trackpad/Magic Mouse gestures (GCMouse never reports them).
diff --git a/clients/apple/Sources/PunktfunkKit/StreamView.swift b/clients/apple/Sources/PunktfunkKit/StreamView.swift
index f80d8fe..a0ad446 100644
--- a/clients/apple/Sources/PunktfunkKit/StreamView.swift
+++ b/clients/apple/Sources/PunktfunkKit/StreamView.swift
@@ -38,16 +38,21 @@ private let streamInputDebug =
private final class CursorCapture {
private var captured = false
- func capture(in view: NSView) {
- guard !captured, let window = view.window, view.bounds.width > 0 else { return }
+ /// Returns whether capture actually engaged. It can fail mid app-activation — the click
+ /// that reactivates the app delivers `mouseDown` before the app is frontmost, and
+ /// `CGAssociateMouseAndMouseCursorPosition` is refused then — so the caller must stay
+ /// released and let the NEXT click retry, never latching a half-captured state.
+ func capture(in view: NSView) -> Bool {
+ guard !captured, let window = view.window, view.bounds.width > 0 else { return false }
// Park the cursor mid-view so a click can't land in (and activate) another app.
let rectOnScreen = window.convertToScreen(view.convert(view.bounds, to: nil))
let primaryHeight = NSScreen.screens.first?.frame.height ?? 0
CGWarpMouseCursorPosition(
CGPoint(x: rectOnScreen.midX, y: primaryHeight - rectOnScreen.midY))
- CGAssociateMouseAndMouseCursorPosition(0)
+ guard CGAssociateMouseAndMouseCursorPosition(0) == .success else { return false }
NSCursor.hide()
captured = true
+ return true
}
func release() {
@@ -194,6 +199,10 @@ public final class StreamLayerView: NSView {
/// InputCapture suppresses its press/release toward the host. Clicks while captured
/// are the host's (GC forwards them) — nothing to do here.
public override func mouseDown(with event: NSEvent) {
+ if streamInputDebug {
+ streamInputLog.debug(
+ "mouseDown: captureEnabled=\(self.captureEnabled, privacy: .public) captured=\(self.captured, privacy: .public)")
+ }
if captureEnabled, !captured {
engageCapture(fromClick: true)
return
@@ -239,6 +248,11 @@ public final class StreamLayerView: NSView {
// here as a send; ⌘-combos still arrive via performKeyEquivalent and stay functional (⌘D).
// Modifier keys never fire keyDown/keyUp — they come through flagsChanged below.
public override var acceptsFirstResponder: Bool { true }
+ // A click after the app was inactive (Cmd-Tab away and back) must reach mouseDown so the
+ // user can re-capture — the deliberate design is that becoming active does NOT auto-grab;
+ // you click into the video. Default NSViews aren't key-view candidates, which can drop
+ // that first click; opting in keeps the view a valid click/responder target.
+ public override var canBecomeKeyView: Bool { true }
public override func keyDown(with event: NSEvent) {
if captured {
if let ic = inputCapture, let vk = InputCapture.keyCodeToVK[event.keyCode] {
@@ -285,7 +299,10 @@ public final class StreamLayerView: NSView {
guard captureEnabled, !captured, pump != nil, window != nil,
fromClick || (NSApp.isActive && window?.isKeyWindow == true)
else { return }
- cursorCapture.capture(in: self)
+ // If the cursor grab is refused (e.g. the reactivating click arrives before the app is
+ // frontmost), stay released so the NEXT click retries — never latch captured=true over
+ // a free cursor, which would make mouseDown's `!captured` guard reject every later click.
+ guard cursorCapture.capture(in: self) else { return }
inputCapture?.setForwarding(true, suppressClick: fromClick)
// Install AFTER the warp + setForwarding: the engage warp generates no forwarded
// delta (the monitor isn't up yet), and the engage click's suppression latch is
diff --git a/clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift b/clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift
index 6db5b6c..62826e7 100644
--- a/clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift
+++ b/clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift
@@ -5,12 +5,21 @@
// fullscreen-and-frontmost, so in Stage Manager it degrades to Mac-style "both cursors
// visible" forwarding).
//
-// Touch is the primary input and is always forwarded (touching the video IS explicit
-// intent): every finger maps to a wire touch id, coordinates are mapped through the
-// aspect-fit letterbox into host-mode pixels, so surface == host mode and the host's
-// rescale is the identity. Hardware keyboard/mouse forwarding shares InputCapture with
-// macOS — auto-engaged when streaming starts, ⌘⎋ toggles (detected from the HID stream;
-// there is no NSEvent monitor here).
+// FINGER touch and INDIRECT POINTER (mouse/trackpad) are routed apart by UITouch.type.
+// Direct fingers (and Pencil) always forward as wire touches — every finger maps to a touch
+// id, coordinates mapped through the aspect-fit letterbox into host-mode pixels (surface ==
+// host mode, so the host's rescale is the identity).
+//
+// A hardware mouse/trackpad is a pointer, not a finger. When the scene is pointer-LOCKED
+// (full-screen + frontmost iPad) GCMouse delivers raw relative deltas and the system hides
+// the cursor — the gaming-grade path. When it CAN'T lock (Stage Manager, not frontmost,
+// iPhone) the system shows its own cursor and routes the mouse through UIKit's pointer path:
+// hover + indirect-pointer touches, which we forward as ABSOLUTE cursor position (+ buttons)
+// so the host cursor tracks the visible local one. We never forward an indirect pointer as a
+// touch — doing so hid the cursor and made the host see taps instead of a moving mouse.
+// GCMouse is gated off whenever the lock isn't held so the two paths can't double-send.
+// Hardware keyboard forwarding shares InputCapture with macOS — auto-engaged when streaming
+// starts, ⌘⎋ toggles (detected from the HID stream; there is no NSEvent monitor here).
//
// The public type is named StreamView like its macOS twin (each is platform-gated), so
// the SwiftUI app layer is identical on both platforms.
@@ -82,6 +91,9 @@ public final class StreamViewController: UIViewController {
private var inputCapture: InputCapture?
fileprivate var captured = false
private var pointerInteraction: UIPointerInteraction?
+ /// Capture state at the last resign, restored on the next foreground — otherwise the
+ /// mouse/keyboard stay released after navigating out and nothing re-grabs them.
+ private var wasCapturedOnResign = false
#endif
/// Reads whether the scene's pointer is actually locked right now; nil = state
@@ -156,9 +168,29 @@ public final class StreamViewController: UIViewController {
let mode = connection.currentMode()
return CGSize(width: Double(mode.width), height: Double(mode.height))
}
- streamView.onTouchEvent = { [weak connection] event in
+ streamView.onTouchEvent = { [weak self, weak connection] event in
+ // Touch IS the intent during a trusted session, but must not leak to the host
+ // while a trust prompt is up (captureEnabled == false) — gate it on that. The
+ // ⌘⎋ mouse/keyboard toggle (captured) deliberately does NOT gate touch.
+ guard self?.captureEnabled == true else { return }
connection?.send(event)
}
+ // Indirect pointer (mouse/trackpad with no lock) → absolute cursor + buttons, routed
+ // through InputCapture so the forwarding gate and release-on-blur apply uniformly.
+ streamView.onPointerMoveAbs = { [weak self] p in
+ self?.inputCapture?.sendMouseAbs(
+ x: p.x, y: p.y, surfaceWidth: p.w, surfaceHeight: p.h)
+ }
+ streamView.onPointerButton = { [weak self] button, down in
+ self?.inputCapture?.sendMouseButton(button, pressed: down)
+ }
+ // Trackpad two-finger / wheel scroll → host scroll. The pan recognizer is the
+ // UNLOCKED regime; while locked, GCMouse's scroll handler owns it — mirror the
+ // sendMouseAbs !gcMouseForwarding gate so the two can't double-send.
+ streamView.onScroll = { [weak self] dx, dy in
+ guard let self, self.inputCapture?.gcMouseForwarding == false else { return }
+ self.inputCapture?.sendScroll(dx: dx, dy: dy)
+ }
let capture = InputCapture(connection: connection)
capture.onToggleCapture = { [weak self] in
@@ -185,17 +217,28 @@ public final class StreamViewController: UIViewController {
observers.append(NotificationCenter.default.addObserver(
forName: UIApplication.willResignActiveNotification, object: nil, queue: .main
) { [weak self] _ in
- self?.setCaptured(false)
+ guard let self else { return }
+ self.wasCapturedOnResign = self.captured
+ self.setCaptured(false)
})
- // The system can drop the lock without us asking (Slide Over, Stage Manager, leaving
- // foregroundActive). Surface it so the user sees, in PUNKTFUNK_INPUT_DEBUG, when
- // GCMouse delivery has silently stopped and we've fallen back to touch.
+ // Returning to the foreground restores the capture the user had before leaving —
+ // without this the mouse/keyboard stay released and nothing re-grabs them (touch
+ // always plays regardless). The macOS twin re-engages on a click into the video.
+ observers.append(NotificationCenter.default.addObserver(
+ forName: UIApplication.didBecomeActiveNotification, object: nil, queue: .main
+ ) { [weak self] _ in
+ guard let self, self.wasCapturedOnResign, self.captureEnabled, self.pump != nil
+ else { return }
+ self.setCaptured(true)
+ })
+ // The system can grant or drop the lock without us asking (Slide Over, Stage Manager,
+ // entering/leaving foregroundActive). Re-resolve the mouse routing on every change:
+ // GCMouse (locked) vs the absolute UIKit pointer path (unlocked), and the
+ // hidden-vs-visible local cursor.
observers.append(NotificationCenter.default.addObserver(
forName: UIPointerLockState.didChangeNotification, object: nil, queue: .main
) { [weak self] _ in
- guard let self, iosInputDebug else { return }
- let locked = self.pointerLockEngaged().map(String.init(describing:)) ?? "unavailable"
- iosInputLog.debug("pointer lock changed: isLocked=\(locked, privacy: .public)")
+ self?.syncPointerLock()
})
if captureEnabled {
@@ -212,6 +255,9 @@ public final class StreamViewController: UIViewController {
inputCapture?.stop()
inputCapture = nil
streamView.onTouchEvent = nil
+ streamView.onPointerMoveAbs = nil
+ streamView.onPointerButton = nil
+ streamView.onScroll = nil
streamView.currentHostMode = nil
#endif
pump?.stop()
@@ -231,18 +277,35 @@ public final class StreamViewController: UIViewController {
captured = false
}
setNeedsUpdateOfPrefersPointerLocked()
- pointerInteraction?.invalidate() // re-resolve the hidden/visible pointer style
+ syncPointerLock() // resolve cursor + GCMouse/absolute routing for the current state
let onCaptureChange = onCaptureChange
let captured = captured
DispatchQueue.main.async { [weak self] in
onCaptureChange?(captured)
- // The lock request is async — read the resolved state next turn. If it didn't
- // engage, GCMouse won't deliver and the always-on touch path carries input.
- if iosInputDebug, let self {
- let locked = self.pointerLockEngaged().map(String.init(describing:)) ?? "unavailable"
- iosInputLog.debug(
- "setCaptured(\(captured, privacy: .public)) → pointer lock isLocked=\(locked, privacy: .public)")
- }
+ // The lock request is async — the resolved state can land a runloop later, and the
+ // initial grant may precede our didChange observer, so re-resolve the routing here.
+ self?.syncPointerLock()
+ }
+ }
+
+ /// Resolve the mouse routing for the scene's CURRENT pointer-lock state: GCMouse (relative
+ /// deltas + buttons) while locked, the absolute UIKit pointer path while not, and the
+ /// hidden-vs-visible local cursor to match. Idempotent — safe to call on every lock-state
+ /// change and capture toggle. Main queue.
+ private func syncPointerLock() {
+ let locked = pointerLockEngaged() == true
+ let useGCMouse = captured && locked
+ // Lock dropped (or capture ended) while the GCMouse path held a button down: once
+ // gcMouseForwarding flips false its release handler is gated off, so flush any held
+ // mouse button here before the switch — otherwise it sticks down on the host.
+ if inputCapture?.gcMouseForwarding == true, !useGCMouse {
+ inputCapture?.releaseMouseButtons()
+ }
+ inputCapture?.gcMouseForwarding = useGCMouse
+ pointerInteraction?.invalidate() // re-resolve the hidden/visible cursor for the state
+ if iosInputDebug {
+ iosInputLog.debug(
+ "pointer lock isLocked=\(locked, privacy: .public) captured=\(self.captured, privacy: .public)")
}
}
#endif
@@ -258,7 +321,11 @@ extension StreamViewController: UIPointerInteractionDelegate {
public func pointerInteraction(
_ interaction: UIPointerInteraction, styleFor region: UIPointerRegion
) -> UIPointerStyle? {
- captured ? .hidden() : nil
+ // Hide the local cursor only when the scene is actually pointer-LOCKED — then the
+ // host renders its own cursor from GCMouse deltas and a visible local one would just
+ // diverge. When the lock isn't held the cursor stays VISIBLE so the user can aim; the
+ // pointer is forwarded as an absolute position, both cursors tracking together.
+ captured && pointerLockEngaged() == true ? .hidden() : nil
}
}
#endif
@@ -274,12 +341,26 @@ final class StreamLayerUIView: UIView {
}
#if os(iOS)
- /// Reads the LIVE negotiated mode in pixels (the touch coordinate space).
- var currentHostMode: (() -> CGSize)?
- var onTouchEvent: ((PunktfunkInputEvent) -> Void)?
+ /// A position already mapped into host-mode pixels, with the surface dims the host
+ /// rescales against (== host mode, so its rescale is the identity).
+ struct HostPoint { let x: Int32; let y: Int32; let w: UInt32; let h: UInt32 }
- /// Wire touch ids per active UITouch; ids are reused after the touch ends.
+ /// Reads the LIVE negotiated mode in pixels (the touch/pointer coordinate space).
+ var currentHostMode: (() -> CGSize)?
+ /// Direct fingers / Pencil → wire touch events.
+ var onTouchEvent: ((PunktfunkInputEvent) -> Void)?
+ /// Indirect pointer (mouse/trackpad with no lock) → absolute cursor moves.
+ var onPointerMoveAbs: ((HostPoint) -> Void)?
+ /// Indirect-pointer buttons (GameStream ids: 1=left 3=right); `down` = press.
+ var onPointerButton: ((_ button: UInt32, _ down: Bool) -> Void)?
+ /// Trackpad two-finger / wheel scroll (no lock) → host scroll deltas, WHEEL(120)-scaled.
+ var onScroll: ((_ dx: Float, _ dy: Float) -> Void)?
+
+ /// Wire touch ids per active direct UITouch; ids are reused after the touch ends.
private var touchIDs: [ObjectIdentifier: UInt32] = [:]
+ /// GameStream button held per active indirect-pointer touch (one click/drag session);
+ /// released when that touch ends.
+ private var pointerButtons: [ObjectIdentifier: UInt32] = [:]
#endif
override init(frame: CGRect) {
@@ -287,6 +368,17 @@ final class StreamLayerUIView: UIView {
displayLayer.videoGravity = .resizeAspect
#if os(iOS)
isMultipleTouchEnabled = true
+ // Button-less mouse/trackpad movement (no lock) arrives as hover, not touches —
+ // forward it as absolute cursor moves so the host cursor tracks without a click held.
+ addGestureRecognizer(
+ UIHoverGestureRecognizer(target: self, action: #selector(handleHover)))
+ // Trackpad two-finger / wheel scroll → a scroll-ONLY pan: allowedTouchTypes = []
+ // rejects finger drags (those stay host touches), allowedScrollTypesMask accepts the
+ // indirect scroll devices. Forwarded as host scroll deltas.
+ let scrollPan = UIPanGestureRecognizer(target: self, action: #selector(handleScroll))
+ scrollPan.allowedScrollTypesMask = .all
+ scrollPan.allowedTouchTypes = []
+ addGestureRecognizer(scrollPan)
#endif
backgroundColor = .black
}
@@ -296,26 +388,58 @@ final class StreamLayerUIView: UIView {
#if os(iOS)
override func touchesBegan(_ touches: Set, with event: UIEvent?) {
- forward(touches, kind: .down)
+ route(touches, event: event, kind: .down)
}
override func touchesMoved(_ touches: Set, with event: UIEvent?) {
- forward(touches, kind: .move)
+ route(touches, event: event, kind: .move)
}
override func touchesEnded(_ touches: Set, with event: UIEvent?) {
- forward(touches, kind: .up)
+ route(touches, event: event, kind: .up)
}
override func touchesCancelled(_ touches: Set, with event: UIEvent?) {
- forward(touches, kind: .up)
+ route(touches, event: event, kind: .up)
}
private enum TouchKind { case down, move, up }
- private func forward(_ touches: Set, kind: TouchKind) {
- guard let hostMode = currentHostMode?(),
- hostMode.width > 0, hostMode.height > 0, onTouchEvent != nil
- else { return }
- let video = AVMakeRect(aspectRatio: hostMode, insideRect: bounds)
- guard video.width > 0, video.height > 0 else { return }
+ /// Split a touch batch by kind: an INDIRECT POINTER (mouse/trackpad with no lock) drives
+ /// the host cursor as an absolute mouse; everything else (direct finger, Pencil) is a host
+ /// touch. Mixed batches are possible, so partition rather than branch on the first touch.
+ private func route(_ touches: Set, event: UIEvent?, kind: TouchKind) {
+ var fingers: Set = []
+ for touch in touches {
+ if touch.type == .indirectPointer {
+ handleIndirectPointer(touch, event: event, kind: kind)
+ } else {
+ fingers.insert(touch)
+ }
+ }
+ if !fingers.isEmpty { forwardTouches(fingers, kind: kind) }
+ }
+
+ /// An indirect-pointer touch is a button-held click/drag session: forward its position as
+ /// an absolute cursor move and its button as a mouse button (down on begin, up on end).
+ private func handleIndirectPointer(_ touch: UITouch, event: UIEvent?, kind: TouchKind) {
+ let key = ObjectIdentifier(touch)
+ let host = hostPoint(from: touch.location(in: self))
+ switch kind {
+ case .down:
+ let button = Self.gsButton(for: event?.buttonMask ?? .primary)
+ pointerButtons[key] = button
+ if let host { onPointerMoveAbs?(host) } // place the cursor, then press
+ onPointerButton?(button, true)
+ case .move:
+ if let host { onPointerMoveAbs?(host) }
+ case .up:
+ if let host { onPointerMoveAbs?(host) }
+ if let button = pointerButtons.removeValue(forKey: key) {
+ onPointerButton?(button, false)
+ }
+ }
+ }
+
+ private func forwardTouches(_ touches: Set, kind: TouchKind) {
+ guard onTouchEvent != nil else { return }
for touch in touches {
let key = ObjectIdentifier(touch)
let id: UInt32
@@ -332,20 +456,53 @@ final class StreamLayerUIView: UIView {
onTouchEvent?(.touchUp(id: id))
continue
}
- let p = touch.location(in: self)
- let x = Int32(((p.x - video.minX) / video.width * hostMode.width)
- .rounded().clamped(to: 0...(hostMode.width - 1)))
- let y = Int32(((p.y - video.minY) / video.height * hostMode.height)
- .rounded().clamped(to: 0...(hostMode.height - 1)))
- let w = UInt32(hostMode.width)
- let h = UInt32(hostMode.height)
+ guard let h = hostPoint(from: touch.location(in: self)) else { continue }
onTouchEvent?(
kind == .down
- ? .touchDown(id: id, x: x, y: y, surfaceWidth: w, surfaceHeight: h)
- : .touchMove(id: id, x: x, y: y, surfaceWidth: w, surfaceHeight: h))
+ ? .touchDown(id: id, x: h.x, y: h.y, surfaceWidth: h.w, surfaceHeight: h.h)
+ : .touchMove(id: id, x: h.x, y: h.y, surfaceWidth: h.w, surfaceHeight: h.h))
}
}
+ /// Button-less mouse/trackpad movement (no lock) → absolute cursor move.
+ @objc private func handleHover(_ recognizer: UIHoverGestureRecognizer) {
+ switch recognizer.state {
+ case .began, .changed:
+ if let h = hostPoint(from: recognizer.location(in: self)) { onPointerMoveAbs?(h) }
+ default:
+ break
+ }
+ }
+
+ /// Trackpad / wheel scroll (no lock) → host scroll deltas. The translation is consumed
+ /// each callback so the next is a fresh delta. Sign/scale are tunable (≈ one notch per
+ /// ~10 pt): finger up scrolls up (host +y), x passes through — the host WHEEL convention.
+ @objc private func handleScroll(_ g: UIPanGestureRecognizer) {
+ guard g.state == .began || g.state == .changed else { return }
+ let t = g.translation(in: self)
+ g.setTranslation(.zero, in: self)
+ onScroll?(Float(t.x) * 12, Float(-t.y) * 12)
+ }
+
+ /// Map a view-space point through the aspect-fit letterbox into host-mode pixels; points
+ /// outside the video area clamp onto its edge. nil until a mode is negotiated.
+ private func hostPoint(from p: CGPoint) -> HostPoint? {
+ guard let hostMode = currentHostMode?(), hostMode.width > 0, hostMode.height > 0
+ else { return nil }
+ let video = AVMakeRect(aspectRatio: hostMode, insideRect: bounds)
+ guard video.width > 0, video.height > 0 else { return nil }
+ let x = Int32(((p.x - video.minX) / video.width * hostMode.width)
+ .rounded().clamped(to: 0...(hostMode.width - 1)))
+ let y = Int32(((p.y - video.minY) / video.height * hostMode.height)
+ .rounded().clamped(to: 0...(hostMode.height - 1)))
+ return HostPoint(x: x, y: y, w: UInt32(hostMode.width), h: UInt32(hostMode.height))
+ }
+
+ /// `.secondary` (right button / two-finger click) → GameStream right (3); else left (1).
+ private static func gsButton(for mask: UIEvent.ButtonMask) -> UInt32 {
+ mask.contains(.secondary) ? 3 : 1
+ }
+
private func nextFreeID() -> UInt32 {
var id: UInt32 = 0
while touchIDs.values.contains(id) { id += 1 }
diff --git a/clients/apple/Tests/PunktfunkKitTests/HostDiscoveryTests.swift b/clients/apple/Tests/PunktfunkKitTests/HostDiscoveryTests.swift
new file mode 100644
index 0000000..c8b5834
--- /dev/null
+++ b/clients/apple/Tests/PunktfunkKitTests/HostDiscoveryTests.swift
@@ -0,0 +1,54 @@
+// Advertise a fake punktfunk/1 host over real mDNS (NWListener) and assert HostDiscovery's
+// NWBrowser finds it, resolves an address+port, and parses the TXT (id / pair / fp). This
+// exercises the whole client discovery path on the loopback/LAN; it self-skips if the test
+// environment blocks Bonjour (sandboxed CI without local-network access).
+
+import Network
+import PunktfunkKit
+import XCTest
+
+final class HostDiscoveryTests: XCTestCase {
+ func testFindsAdvertisedHost() async throws {
+ let serviceName = "PunktfunkTest-\(UUID().uuidString.prefix(8))"
+ let uniqueid = "test-\(UUID().uuidString)"
+
+ var txt = NWTXTRecord()
+ txt["proto"] = "punktfunk/1"
+ txt["fp"] = String(repeating: "ab", count: 32) // 64 hex chars, like a real cert SHA-256
+ txt["pair"] = "required"
+ txt["id"] = uniqueid
+
+ let listener = try NWListener(using: .udp)
+ listener.service = NWListener.Service(
+ name: String(serviceName), type: "_punktfunk._udp", txtRecord: txt)
+ // The resolver opens a throwaway UDP flow to read the resolved endpoint — accept and
+ // drop it so it doesn't linger.
+ listener.newConnectionHandler = { connection in connection.cancel() }
+ listener.start(queue: .global())
+ defer { listener.cancel() }
+
+ let discovery = await HostDiscovery()
+ await discovery.start()
+ defer { Task { await discovery.stop() } }
+
+ // Poll up to ~10s for the advert to be browsed AND resolved.
+ var found: DiscoveredHost?
+ let deadline = Date().addingTimeInterval(10)
+ while Date() < deadline {
+ if let host = await discovery.hosts.first(where: { $0.name == String(serviceName) }) {
+ found = host
+ break
+ }
+ try await Task.sleep(nanoseconds: 200_000_000)
+ }
+
+ guard let host = found else {
+ throw XCTSkip("mDNS discovery unavailable in this environment (no local network).")
+ }
+ XCTAssertEqual(host.id, uniqueid, "the stable mDNS id should key the host")
+ XCTAssertTrue(host.requiresPairing, "pair=required must surface as requiresPairing")
+ XCTAssertEqual(host.fingerprintHex, String(repeating: "ab", count: 32))
+ XCTAssertFalse(host.host.isEmpty, "a resolved address is required to connect")
+ XCTAssertGreaterThan(host.port, 0, "a resolved port is required to connect")
+ }
+}