feat(apple): iOS/iPadOS client — touch, pointer lock, shared SwiftUI shell
ci / rust (push) Has been cancelled

The whole client now runs on iPadOS/iOS from the same sources, first-lit live in the
iPad simulator against the real host at 1280x720@60 (60 fps on the HUD, capture state
machine active, mic permission flow shown).

- PunktfunkCore.xcframework grows iOS device + universal-simulator slices
  (BUILD_IOS=1; rustup targets aarch64-apple-ios{,-sim} + x86_64-apple-ios).
- The decode pump is extracted into a shared StreamPump (identical IDR re-gate logic on
  both platforms); the iOS StreamView (StreamViewIOS.swift) has the same name/signature
  as the macOS one, so ContentView & co. are byte-identical across platforms — hosted
  in a UIViewController for prefersPointerLocked (the iPadOS cursor capture; see README
  note 9 for the UIHostingController forwarding caveat).
- Touch is always forwarded: per-finger wire ids, coordinates mapped through the
  aspect-fit letterbox into LIVE host-mode pixels (surface == host mode, identity
  rescale host-side; follows mid-stream requestMode switches).
- InputCapture is cross-platform: GC works the same on iPadOS, ⌘⎋ is detected from the
  HID stream there; stale-⌘ tracking after focus loss fixed on both platforms
  (releaseAll now drops the modifier/latch state — a ⌘ released in another app
  otherwise hijacked Esc forever).
- SessionAudio: AVAudioSession on iOS (.playAndRecord + .defaultToSpeaker — without it
  iPhones route host audio to the EARPIECE; deactivated with
  notifyOthersOnDeactivation on stop so interrupted background audio resumes); HAL
  device pinning + the Settings pickers stay macOS-only.
- New Punktfunk-iOS app target (shared synchronized sources, generated Info.plist with
  mic + local-network usage descriptions — QUIC to a LAN host trips local network
  privacy on real devices — scene manifest + indirect input events for Stage Manager /
  external displays), shared scheme, macOS min-window frames gated off iOS.

For the iPad-on-an-external-screen idea: with multiple scenes + indirect input enabled,
Stage Manager iPads can drag the punktfunk window onto the external display and drive
the PC with keyboard/mouse/touch. Known gaps (README note 9): the pointer-lock
preference isn't consulted through UIHostingController (relative mouse works, the local
cursor just stays visible) and AVAudioSession interruptions don't auto-restart audio.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-11 11:18:18 +02:00
parent 136390514d
commit e1af4d57c6
14 changed files with 766 additions and 73 deletions
@@ -99,25 +99,8 @@ public struct StreamView: NSViewRepresentable {
}
public final class StreamLayerView: NSView {
/// Cancellation handle owned by exactly one pump thread a restart hands the old pump
/// its own token, so it can never be revived by a newer start().
private final class PumpToken: @unchecked Sendable {
private let lock = NSLock()
private var live = true
var isLive: Bool {
lock.lock()
defer { lock.unlock() }
return live
}
func cancel() {
lock.lock()
live = false
lock.unlock()
}
}
private let displayLayer = AVSampleBufferDisplayLayer()
private var token: PumpToken?
private var pump: StreamPump?
public private(set) var connection: PunktfunkConnection?
private let cursorCapture = CursorCapture()
private var inputCapture: InputCapture?
@@ -261,7 +244,7 @@ public final class StreamLayerView: NSView {
// A click is explicit intent AND may arrive mid-activation (acceptsFirstMouse:
// NSApp.isActive / isKeyWindow are still false for the click coming in from
// another app) only the auto-engage paths require already-held key status.
guard captureEnabled, !captured, token != nil, window != nil,
guard captureEnabled, !captured, pump != nil, window != nil,
fromClick || (NSApp.isActive && window?.isKeyWindow == true)
else { return }
cursorCapture.capture(in: self)
@@ -297,11 +280,7 @@ public final class StreamLayerView: NSView {
onSessionEnd: (@Sendable () -> Void)? = nil
) {
stop()
let token = PumpToken()
self.token = token
self.connection = connection
let layer = displayLayer
layer.flush() // drop any frames a previous connection left queued
// The view owns the session's input capture: handlers attach now, but nothing is
// forwarded until capture engages (captureEnabled + auto-engage or a click).
@@ -324,40 +303,11 @@ public final class StreamLayerView: NSView {
capture.start()
inputCapture = capture
let thread = Thread {
var format: CMVideoFormatDescription?
while token.isLive {
do {
guard let au = try connection.nextAU(timeoutMs: 100) else { continue }
onFrame?(au)
if let f = AnnexB.formatDescription(fromIDR: au.data) {
format = f // refreshed on every IDR (mode changes included)
}
if layer.status == .failed {
// Decode wedged: flush and re-gate on the next in-band parameter
// sets resuming with a delta frame can't recover. (A
// request-IDR channel on punktfunk/1 is a host-side TODO; with the
// host's infinite GOP this may otherwise stay black until the
// next recovery keyframe.)
layer.flush()
format = AnnexB.formatDescription(fromIDR: au.data)
}
guard let f = format,
let sample = AnnexB.sampleBuffer(au: au, format: f),
token.isLive // don't enqueue a stale frame after a restart
else { continue }
layer.enqueue(sample)
} catch {
if token.isLive {
onSessionEnd?()
}
break // session closed
}
}
}
thread.name = "punktfunk-pump"
thread.qualityOfService = .userInteractive
thread.start()
let pump = StreamPump()
pump.start(
connection: connection, layer: displayLayer,
onFrame: onFrame, onSessionEnd: onSessionEnd)
self.pump = pump
requestAutoCapture() // entering a session is the deliberate "capture me" moment
}
@@ -367,15 +317,15 @@ public final class StreamLayerView: NSView {
releaseCapture()
inputCapture?.stop()
inputCapture = nil
token?.cancel()
token = nil
pump?.stop()
pump = nil
connection = nil
}
deinit {
appObservers.forEach(NotificationCenter.default.removeObserver(_:))
windowObservers.forEach(NotificationCenter.default.removeObserver(_:))
token?.cancel()
pump?.stop()
}
}
#endif