feat(apple): session audio — host playback + mic uplink, device pickers in Settings
ci / rust (push) Has been cancelled

Both directions of the audio plane, on CoreAudio's built-in Opus codec
(kAudioFormatOpus — no bundled libopus; OpusCodec.swift, round trip unit-tested):

- Playback: a drain thread pulls nextAudio() packets, decodes, and writes a priming
  jitter ring feeding an AVAudioSourceNode (~20 ms prefill, adaptive to the device's
  render quantum so large-buffer devices don't oscillate prime/dropout; a high-water
  clamp sheds stall backlog so one network hiccup can't permanently lag audio behind
  video; underrun re-primes — one dip, not sustained crackle).
- Mic: a second engine taps the input device, resamples to 48 kHz stereo, Opus-encodes
  20 ms chunks and sendMic()s them into the host's virtual PipeWire source. Permission
  via AVCaptureDevice (NSMicrophoneUsageDescription added to the Xcode target).
- Settings: Speaker + Microphone pickers (CoreAudio HAL enumeration, persisted by
  device UID — "System default" leaves the engine unpinned so it follows macOS device
  changes) and a "Send microphone" toggle (default on). Applies from the next session.
- Audio starts with streaming, never during the trust prompt (no host sound — and no
  mic uplink — before the user trusted the host); teardown stops audio before close().

Adversarial-review fixes baked in: stop() and the dangling mic-permission callback
share one lock+flag protocol (no hot mic with no owner), the connect-success handler
bails when the attempt was abandoned mid-handshake (no session/mic for a dead window),
SessionAudio gets a deinit backstop (a dropped instance can't pin the connection via
its drain thread), and the render scratch buffer is block-owned (was leaked per
session).

Verified live against the box: remote test decodes 100 host Opus packets to PCM and
the host opens its virtual mic on the first uplinked frame ("punktfunk/1 virtual mic
ready"); on-glass session runs with both engines up.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-11 09:39:08 +02:00
parent 2372b02620
commit b26f138699
9 changed files with 840 additions and 8 deletions
@@ -59,6 +59,7 @@ final class SessionModel: ObservableObject {
let meter = FrameMeter()
private var statsTimer: Timer?
private var audio: SessionAudio?
var isBusy: Bool { phase != .idle }
@@ -82,6 +83,15 @@ final class SessionModel: ObservableObject {
pinSHA256: pin, identity: identity, compositor: compositor) }
await MainActor.run { [weak self] in
guard let self else { return }
// The user may have abandoned this attempt (window closed, another host
// clicked) while the handshake was in flight don't resurrect a session
// for a dead window, and especially don't start its mic uplink.
guard self.phase == .connecting, self.activeHost?.id == host.id else {
if case .success(let conn) = result {
Task.detached { conn.close() } // joins Rust threads off-main
}
return
}
switch result {
case .success(let conn):
self.connection = conn
@@ -123,10 +133,18 @@ final class SessionModel: ObservableObject {
func disconnect() {
statsTimer?.invalidate()
statsTimer = nil
let audio = self.audio
self.audio = nil
if let conn = connection {
// close() waits out an in-flight poll (100 ms) and joins the Rust worker
// threads keep that off the main actor.
Task.detached { conn.close() }
// Audio teardown waits its drain thread out and close() waits out in-flight
// polls + joins the Rust worker threads keep both off the main actor, in
// this order (no audio poll left when the handle is freed).
Task.detached {
audio?.stop()
conn.close()
}
} else {
Task.detached { audio?.stop() }
}
connection = nil
activeHost = nil
@@ -145,10 +163,20 @@ final class SessionModel: ObservableObject {
}
private func beginStreaming() {
guard connection != nil else { return }
guard let conn = connection else { return }
// Input capture itself is owned by StreamView (engaged by the captureEnabled
// flip this phase change causes, released/re-engaged by the user from there).
phase = .streaming
// Audio starts with streaming, not during the trust prompt no host sound (or
// mic uplink!) before the user trusted the host. Devices come from Settings;
// "" = system default.
let defaults = UserDefaults.standard
let audio = SessionAudio(connection: conn)
audio.start(
speakerUID: defaults.string(forKey: "punktfunk.speakerUID") ?? "",
micUID: defaults.string(forKey: "punktfunk.micUID") ?? "",
micEnabled: defaults.object(forKey: "punktfunk.micEnabled") as? Bool ?? true)
self.audio = audio
}
private func startStatsTimer() {
@@ -11,6 +11,11 @@ struct SettingsView: View {
@AppStorage("punktfunk.height") private var height = 1080
@AppStorage("punktfunk.hz") private var hz = 60
@AppStorage("punktfunk.compositor") private var compositor = 0
@AppStorage("punktfunk.speakerUID") private var speakerUID = ""
@AppStorage("punktfunk.micUID") private var micUID = ""
@AppStorage("punktfunk.micEnabled") private var micEnabled = true
@State private var outputDevices: [AudioDevice] = []
@State private var inputDevices: [AudioDevice] = []
var body: some View {
Form {
@@ -33,6 +38,38 @@ struct SettingsView: View {
.font(.caption)
.foregroundStyle(.secondary)
}
Section {
Picker("Speaker", selection: $speakerUID) {
Text("System default").tag("")
ForEach(outputDevices) { device in
Text(device.name).tag(device.uid)
}
if !speakerUID.isEmpty,
!outputDevices.contains(where: { $0.uid == speakerUID }) {
Text("Unavailable device").tag(speakerUID)
}
}
Toggle("Send microphone to the host", isOn: $micEnabled)
Picker("Microphone", selection: $micUID) {
Text("System default").tag("")
ForEach(inputDevices) { device in
Text(device.name).tag(device.uid)
}
if !micUID.isEmpty,
!inputDevices.contains(where: { $0.uid == micUID }) {
Text("Unavailable device").tag(micUID)
}
}
.disabled(!micEnabled)
} header: {
Text("Audio")
} footer: {
Text("Host audio plays through the speaker; the microphone feeds the "
+ "host's virtual mic. System default follows macOS device changes. "
+ "Applies from the next session.")
.font(.caption)
.foregroundStyle(.secondary)
}
Section {
Picker("Compositor", selection: $compositor) {
Text("Automatic").tag(0)
@@ -54,6 +91,10 @@ struct SettingsView: View {
.formStyle(.grouped)
.frame(width: 380)
.fixedSize()
.onAppear {
outputDevices = AudioDevices.outputs()
inputDevices = AudioDevices.inputs()
}
}
private func fillFromMainScreen() {