feat(apple): iOS/iPadOS client — touch, pointer lock, shared SwiftUI shell
ci / rust (push) Has been cancelled

The whole client now runs on iPadOS/iOS from the same sources, first-lit live in the
iPad simulator against the real host at 1280x720@60 (60 fps on the HUD, capture state
machine active, mic permission flow shown).

- PunktfunkCore.xcframework grows iOS device + universal-simulator slices
  (BUILD_IOS=1; rustup targets aarch64-apple-ios{,-sim} + x86_64-apple-ios).
- The decode pump is extracted into a shared StreamPump (identical IDR re-gate logic on
  both platforms); the iOS StreamView (StreamViewIOS.swift) has the same name/signature
  as the macOS one, so ContentView & co. are byte-identical across platforms — hosted
  in a UIViewController for prefersPointerLocked (the iPadOS cursor capture; see README
  note 9 for the UIHostingController forwarding caveat).
- Touch is always forwarded: per-finger wire ids, coordinates mapped through the
  aspect-fit letterbox into LIVE host-mode pixels (surface == host mode, identity
  rescale host-side; follows mid-stream requestMode switches).
- InputCapture is cross-platform: GC works the same on iPadOS, ⌘⎋ is detected from the
  HID stream there; stale-⌘ tracking after focus loss fixed on both platforms
  (releaseAll now drops the modifier/latch state — a ⌘ released in another app
  otherwise hijacked Esc forever).
- SessionAudio: AVAudioSession on iOS (.playAndRecord + .defaultToSpeaker — without it
  iPhones route host audio to the EARPIECE; deactivated with
  notifyOthersOnDeactivation on stop so interrupted background audio resumes); HAL
  device pinning + the Settings pickers stay macOS-only.
- New Punktfunk-iOS app target (shared synchronized sources, generated Info.plist with
  mic + local-network usage descriptions — QUIC to a LAN host trips local network
  privacy on real devices — scene manifest + indirect input events for Stage Manager /
  external displays), shared scheme, macOS min-window frames gated off iOS.

For the iPad-on-an-external-screen idea: with multiple scenes + indirect input enabled,
Stage Manager iPads can drag the punktfunk window onto the external display and drive
the PC with keyboard/mouse/touch. Known gaps (README note 9): the pointer-lock
preference isn't consulted through UIHostingController (relative mouse works, the local
cursor just stays visible) and AVAudioSession interruptions don't auto-restart audio.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-11 11:18:18 +02:00
parent 136390514d
commit e1af4d57c6
14 changed files with 766 additions and 73 deletions
@@ -14,7 +14,6 @@
// AVAudioEngine ties input+output to one aggregate clock, separate engines keep
// arbitrary mic/speaker combinations trivial.
#if os(macOS)
import AVFoundation
import os
@@ -140,9 +139,29 @@ public final class SessionAudio {
}
/// Start playback (and, if enabled+authorized, the mic uplink). Empty UIDs = system
/// default device. Main thread (engine setup); returns after the engines start
/// the mic may start slightly later if the permission prompt is pending.
/// default device; on iOS the UIDs are ignored entirely (routes are
/// AVAudioSession-managed). Main thread (engine setup); returns after the engines
/// start the mic may start slightly later if the permission prompt is pending.
public func start(speakerUID: String, micUID: String, micEnabled: Bool) {
#if os(iOS)
// Route + policy live in the session, not per-engine: stereo playback, mic
// capture when enabled, Bluetooth allowed. Failure is non-fatal (defaults).
let session = AVAudioSession.sharedInstance()
do {
if micEnabled {
// .defaultToSpeaker: .playAndRecord otherwise routes to the iPhone
// EARPIECE; only affects the built-in route (headphones/BT still win).
try session.setCategory(
.playAndRecord, mode: .default,
options: [.allowBluetoothA2DP, .defaultToSpeaker])
} else {
try session.setCategory(.playback, mode: .default)
}
try session.setActive(true)
} catch {
log.warning("AVAudioSession setup failed: \(error.localizedDescription)")
}
#endif
startPlayback(speakerUID: speakerUID)
guard micEnabled else { return }
switch AVCaptureDevice.authorizationStatus(for: .audio) {
@@ -180,6 +199,16 @@ public final class SessionAudio {
if wasDraining {
_ = drainDone.wait(timeout: .now() + .milliseconds(400))
}
#if os(iOS)
// Release the session so audio we interrupted (Music, podcasts) gets its
// resume cue.
do {
try AVAudioSession.sharedInstance().setActive(
false, options: .notifyOthersOnDeactivation)
} catch {
log.warning("AVAudioSession deactivation failed: \(error.localizedDescription)")
}
#endif
}
// MARK: - Playback (host speaker)
@@ -190,6 +219,7 @@ public final class SessionAudio {
let ring = AudioRing(capacity: 96_000, prefill: 1920)
let engine = AVAudioEngine()
#if os(macOS)
if !speakerUID.isEmpty {
if let dev = AudioDevices.deviceID(forUID: speakerUID),
let unit = engine.outputNode.audioUnit {
@@ -200,6 +230,7 @@ public final class SessionAudio {
log.warning("speaker \(speakerUID) not present — using default")
}
}
#endif
// Engine-native deinterleaved float; the render block deinterleaves from the ring.
guard let format = AVAudioFormat(standardFormatWithSampleRate: 48_000, channels: 2)
@@ -282,6 +313,7 @@ public final class SessionAudio {
private func startCapture(micUID: String) {
let engine = AVAudioEngine()
let input = engine.inputNode
#if os(macOS)
if !micUID.isEmpty {
if let dev = AudioDevices.deviceID(forUID: micUID), let unit = input.audioUnit {
if !Self.setDevice(dev, on: unit) {
@@ -291,6 +323,7 @@ public final class SessionAudio {
log.warning("microphone \(micUID) not present — using default")
}
}
#endif
let inFormat = input.outputFormat(forBus: 0)
guard inFormat.sampleRate > 0, inFormat.channelCount > 0 else {
@@ -376,11 +409,12 @@ public final class SessionAudio {
log.info("mic uplink started (\(micUID.isEmpty ? "default input" : micUID))")
}
#if os(macOS)
private static func setDevice(_ id: AudioDeviceID, on unit: AudioUnit) -> Bool {
var dev = id
return AudioUnitSetProperty(
unit, kAudioOutputUnitProperty_CurrentDevice, kAudioUnitScope_Global, 0,
&dev, UInt32(MemoryLayout<AudioDeviceID>.size)) == noErr
}
#endif
}
#endif