feat(apple): session audio — host playback + mic uplink, device pickers in Settings
ci / rust (push) Has been cancelled
ci / rust (push) Has been cancelled
Both directions of the audio plane, on CoreAudio's built-in Opus codec
(kAudioFormatOpus — no bundled libopus; OpusCodec.swift, round trip unit-tested):
- Playback: a drain thread pulls nextAudio() packets, decodes, and writes a priming
jitter ring feeding an AVAudioSourceNode (~20 ms prefill, adaptive to the device's
render quantum so large-buffer devices don't oscillate prime/dropout; a high-water
clamp sheds stall backlog so one network hiccup can't permanently lag audio behind
video; underrun re-primes — one dip, not sustained crackle).
- Mic: a second engine taps the input device, resamples to 48 kHz stereo, Opus-encodes
20 ms chunks and sendMic()s them into the host's virtual PipeWire source. Permission
via AVCaptureDevice (NSMicrophoneUsageDescription added to the Xcode target).
- Settings: Speaker + Microphone pickers (CoreAudio HAL enumeration, persisted by
device UID — "System default" leaves the engine unpinned so it follows macOS device
changes) and a "Send microphone" toggle (default on). Applies from the next session.
- Audio starts with streaming, never during the trust prompt (no host sound — and no
mic uplink — before the user trusted the host); teardown stops audio before close().
Adversarial-review fixes baked in: stop() and the dangling mic-permission callback
share one lock+flag protocol (no hot mic with no owner), the connect-success handler
bails when the attempt was abandoned mid-handshake (no session/mic for a dead window),
SessionAudio gets a deinit backstop (a dropped instance can't pin the connection via
its drain thread), and the render scratch buffer is block-owned (was leaked per
session).
Verified live against the box: remote test decodes 100 host Opus packets to PCM and
the host opens its virtual mic on the first uplinked frame ("punktfunk/1 virtual mic
ready"); on-glass session runs with both engines up.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -59,6 +59,7 @@ final class SessionModel: ObservableObject {
|
||||
|
||||
let meter = FrameMeter()
|
||||
private var statsTimer: Timer?
|
||||
private var audio: SessionAudio?
|
||||
|
||||
var isBusy: Bool { phase != .idle }
|
||||
|
||||
@@ -82,6 +83,15 @@ final class SessionModel: ObservableObject {
|
||||
pinSHA256: pin, identity: identity, compositor: compositor) }
|
||||
await MainActor.run { [weak self] in
|
||||
guard let self else { return }
|
||||
// The user may have abandoned this attempt (window closed, another host
|
||||
// clicked) while the handshake was in flight — don't resurrect a session
|
||||
// for a dead window, and especially don't start its mic uplink.
|
||||
guard self.phase == .connecting, self.activeHost?.id == host.id else {
|
||||
if case .success(let conn) = result {
|
||||
Task.detached { conn.close() } // joins Rust threads — off-main
|
||||
}
|
||||
return
|
||||
}
|
||||
switch result {
|
||||
case .success(let conn):
|
||||
self.connection = conn
|
||||
@@ -123,10 +133,18 @@ final class SessionModel: ObservableObject {
|
||||
func disconnect() {
|
||||
statsTimer?.invalidate()
|
||||
statsTimer = nil
|
||||
let audio = self.audio
|
||||
self.audio = nil
|
||||
if let conn = connection {
|
||||
// close() waits out an in-flight poll (≤100 ms) and joins the Rust worker
|
||||
// threads — keep that off the main actor.
|
||||
Task.detached { conn.close() }
|
||||
// Audio teardown waits its drain thread out and close() waits out in-flight
|
||||
// polls + joins the Rust worker threads — keep both off the main actor, in
|
||||
// this order (no audio poll left when the handle is freed).
|
||||
Task.detached {
|
||||
audio?.stop()
|
||||
conn.close()
|
||||
}
|
||||
} else {
|
||||
Task.detached { audio?.stop() }
|
||||
}
|
||||
connection = nil
|
||||
activeHost = nil
|
||||
@@ -145,10 +163,20 @@ final class SessionModel: ObservableObject {
|
||||
}
|
||||
|
||||
private func beginStreaming() {
|
||||
guard connection != nil else { return }
|
||||
guard let conn = connection else { return }
|
||||
// Input capture itself is owned by StreamView (engaged by the captureEnabled
|
||||
// flip this phase change causes, released/re-engaged by the user from there).
|
||||
phase = .streaming
|
||||
// Audio starts with streaming, not during the trust prompt — no host sound (or
|
||||
// mic uplink!) before the user trusted the host. Devices come from Settings;
|
||||
// "" = system default.
|
||||
let defaults = UserDefaults.standard
|
||||
let audio = SessionAudio(connection: conn)
|
||||
audio.start(
|
||||
speakerUID: defaults.string(forKey: "punktfunk.speakerUID") ?? "",
|
||||
micUID: defaults.string(forKey: "punktfunk.micUID") ?? "",
|
||||
micEnabled: defaults.object(forKey: "punktfunk.micEnabled") as? Bool ?? true)
|
||||
self.audio = audio
|
||||
}
|
||||
|
||||
private func startStatsTimer() {
|
||||
|
||||
@@ -11,6 +11,11 @@ struct SettingsView: View {
|
||||
@AppStorage("punktfunk.height") private var height = 1080
|
||||
@AppStorage("punktfunk.hz") private var hz = 60
|
||||
@AppStorage("punktfunk.compositor") private var compositor = 0
|
||||
@AppStorage("punktfunk.speakerUID") private var speakerUID = ""
|
||||
@AppStorage("punktfunk.micUID") private var micUID = ""
|
||||
@AppStorage("punktfunk.micEnabled") private var micEnabled = true
|
||||
@State private var outputDevices: [AudioDevice] = []
|
||||
@State private var inputDevices: [AudioDevice] = []
|
||||
|
||||
var body: some View {
|
||||
Form {
|
||||
@@ -33,6 +38,38 @@ struct SettingsView: View {
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
Section {
|
||||
Picker("Speaker", selection: $speakerUID) {
|
||||
Text("System default").tag("")
|
||||
ForEach(outputDevices) { device in
|
||||
Text(device.name).tag(device.uid)
|
||||
}
|
||||
if !speakerUID.isEmpty,
|
||||
!outputDevices.contains(where: { $0.uid == speakerUID }) {
|
||||
Text("Unavailable device").tag(speakerUID)
|
||||
}
|
||||
}
|
||||
Toggle("Send microphone to the host", isOn: $micEnabled)
|
||||
Picker("Microphone", selection: $micUID) {
|
||||
Text("System default").tag("")
|
||||
ForEach(inputDevices) { device in
|
||||
Text(device.name).tag(device.uid)
|
||||
}
|
||||
if !micUID.isEmpty,
|
||||
!inputDevices.contains(where: { $0.uid == micUID }) {
|
||||
Text("Unavailable device").tag(micUID)
|
||||
}
|
||||
}
|
||||
.disabled(!micEnabled)
|
||||
} header: {
|
||||
Text("Audio")
|
||||
} footer: {
|
||||
Text("Host audio plays through the speaker; the microphone feeds the "
|
||||
+ "host's virtual mic. System default follows macOS device changes. "
|
||||
+ "Applies from the next session.")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
Section {
|
||||
Picker("Compositor", selection: $compositor) {
|
||||
Text("Automatic").tag(0)
|
||||
@@ -54,6 +91,10 @@ struct SettingsView: View {
|
||||
.formStyle(.grouped)
|
||||
.frame(width: 380)
|
||||
.fixedSize()
|
||||
.onAppear {
|
||||
outputDevices = AudioDevices.outputs()
|
||||
inputDevices = AudioDevices.inputs()
|
||||
}
|
||||
}
|
||||
|
||||
private func fillFromMainScreen() {
|
||||
|
||||
Reference in New Issue
Block a user