From e54258b8acdd496e028a9bbb96a4acf9c21925f9 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 31 Jul 2026 21:57:48 +0200 Subject: [PATCH] feat(apple): the mic uplink drops to mono 10 ms packets and stops shoveling its FIFO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The capture path carried three latencies that had nothing to do with the network: a 2048-frame tap request (42.7 ms bursts before the first sample could even be sliced), 20 ms Opus framing, and a mono signal duplicated into both stereo channels because the encoder was configured like the downlink. CoreAudio's Opus encoder takes mFramesPerPacket=480 and mChannelsPerFrame=1 just fine — probed empirically: the converter truly emits 10 ms mono CELT packets (TOC config 30), one per 480-frame chunk, and the stereo-shaped decoder upmixes them with the tone intact — so the uplink now asks for 10 ms tap buffers and encodes 48 kbps mono 10 ms packets, with 960 kept only as an init-time fallback. The host decodes any Opus frame ≤120 ms, so nothing changes on the wire's far end. The tap thread also stops burning cycles per callback: the mono fold and resampler scratch buffers are allocated once (regrown only if a larger device quantum ever arrives), and the chunk slicer walks a head index instead of removeFirst — which memmoved the entire backlog for every packet on a render-adjacent thread. iOS additionally asks the session for 5 ms IO quanta at 48 kHz when the mic is on (best-effort; the hardware decides). Verified: swift build (macOS arm64), swift test OpusCodecTests + AudioChannelFoldTests, and an iOS arm64 cross-build; the 480/mono behavior confirmed by TOC inspection on macOS 15. Co-Authored-By: Claude Fable 5 --- .../PunktfunkKit/Audio/OpusCodec.swift | 60 +++++++++----- .../PunktfunkKit/Audio/SessionAudio.swift | 80 +++++++++++++------ .../PunktfunkKitTests/OpusCodecTests.swift | 24 +++--- .../RemoteFirstLightTests.swift | 23 +++--- 4 files changed, 123 insertions(+), 64 deletions(-) diff --git a/clients/apple/Sources/PunktfunkKit/Audio/OpusCodec.swift b/clients/apple/Sources/PunktfunkKit/Audio/OpusCodec.swift index 7261cd3a..d1853de5 100644 --- a/clients/apple/Sources/PunktfunkKit/Audio/OpusCodec.swift +++ b/clients/apple/Sources/PunktfunkKit/Audio/OpusCodec.swift @@ -1,7 +1,9 @@ // Opus ⇄ PCM through CoreAudio's built-in codec (kAudioFormatOpus, macOS 10.13+ / iOS // 11+) — no bundled libopus. The host's audio plane is raw Opus packets (48 kHz stereo, -// one frame per packet); AVAudioConverter handles them as single-packet -// AVAudioCompressedBuffers with explicit packet descriptions. +// one frame per packet); the mic uplink is 48 kHz MONO packets (one microphone bus — +// the host's decoder upmixes, so duplicating it into a second channel only cost bits). +// AVAudioConverter handles both as single-packet AVAudioCompressedBuffers with explicit +// packet descriptions. // // Both classes are single-threaded by contract (one per direction, owned by their // drain/capture pipelines). @@ -14,16 +16,16 @@ enum OpusCodecError: Error { case convertFailed(String) } -/// 48 kHz stereo float32 interleaved — the PCM side of both converters and the layout -/// of the playback ring buffer. +/// 48 kHz stereo float32 interleaved — the decoder's PCM side (the host plane's shape). func opusPCMFormat() -> AVAudioFormat? { AVAudioFormat( commonFormat: .pcmFormatFloat32, sampleRate: 48_000, channels: 2, interleaved: true) } /// The compressed side: raw Opus, `framesPerPacket` nominal samples per packet at 48 kHz -/// (240 = the host's 5 ms audio plane; 960 = the 20 ms packets the encoder emits). -private func opusFormat(framesPerPacket: UInt32) -> AVAudioFormat? { +/// (240 = the host's 5 ms audio plane; 480 = the 10 ms packets the encoder emits) and +/// `channels` (2 = the host plane, 1 = the mic uplink). +private func opusFormat(framesPerPacket: UInt32, channels: UInt32) -> AVAudioFormat? { var desc = AudioStreamBasicDescription( mSampleRate: 48_000, mFormatID: kAudioFormatOpus, @@ -31,7 +33,7 @@ private func opusFormat(framesPerPacket: UInt32) -> AVAudioFormat? { mBytesPerPacket: 0, mFramesPerPacket: framesPerPacket, mBytesPerFrame: 0, - mChannelsPerFrame: 2, + mChannelsPerFrame: channels, mBitsPerChannel: 0, mReserved: 0) return AVAudioFormat(streamDescription: &desc) @@ -45,7 +47,8 @@ final class OpusDecoder { /// `framesPerPacket`: the sender's packet duration in samples (host audio = 240). init(framesPerPacket: UInt32) throws { - guard let pcm = opusPCMFormat(), let opus = opusFormat(framesPerPacket: framesPerPacket), + guard let pcm = opusPCMFormat(), + let opus = opusFormat(framesPerPacket: framesPerPacket, channels: 2), let converter = AVAudioConverter(from: opus, to: pcm) else { throw OpusCodecError.unavailable } self.converter = converter @@ -90,24 +93,43 @@ final class OpusDecoder { } final class OpusEncoder { - /// The encoder's packet duration: 960 samples = 20 ms, CoreAudio's default Opus - /// framing. The host's mic service decodes any Opus frame size up to 120 ms. - static let framesPerPacket: AVAudioFrameCount = 960 + /// The encoder's packet duration in samples: 480 = 10 ms, halving the packetization + /// latency of the old 20 ms framing. CoreAudio honors it — mFramesPerPacket 480/mono + /// creates a converter that truly emits 10 ms CELT packets (TOC config 30), one per + /// 480-frame chunk, verified by inspection of the emitted TOC bytes and per-packet + /// frame accounting. 960 stays as the fallback should an older codec refuse 480. + /// The host's mic service decodes any Opus frame size up to 120 ms, so either is + /// wire-compatible. + let framesPerPacket: AVAudioFrameCount + + /// 48 kHz MONO float32 interleaved — the uplink carries one microphone bus. + let pcmFormat: AVAudioFormat private let converter: AVAudioConverter private let outBuf: AVAudioCompressedBuffer - let pcmFormat: AVAudioFormat init() throws { - guard let pcm = opusPCMFormat(), - let opus = opusFormat(framesPerPacket: UInt32(Self.framesPerPacket)), - let converter = AVAudioConverter(from: pcm, to: opus) + guard let pcm = AVAudioFormat( + commonFormat: .pcmFormatFloat32, sampleRate: 48_000, channels: 1, + interleaved: true) else { throw OpusCodecError.unavailable } - converter.bitRate = 96_000 - self.converter = converter - self.pcmFormat = pcm + var made: (converter: AVAudioConverter, fpp: AVAudioFrameCount)? + for fpp: AVAudioFrameCount in [480, 960] { + if let opus = opusFormat(framesPerPacket: UInt32(fpp), channels: 1), + let converter = AVAudioConverter(from: pcm, to: opus) { + made = (converter, fpp) + break + } + } + guard let made else { throw OpusCodecError.unavailable } + // 48 kbps: transparent for mono voice — the old 96 kbps budget was sized for + // the duplicated-stereo framing this encoder no longer emits. + made.converter.bitRate = 48_000 + converter = made.converter + framesPerPacket = made.fpp + pcmFormat = pcm outBuf = AVAudioCompressedBuffer( - format: opus, packetCapacity: 4, maximumPacketSize: 1500) + format: made.converter.outputFormat, packetCapacity: 4, maximumPacketSize: 1500) } /// Encode exactly `framesPerPacket` frames of `pcmFormat` audio; returns the encoded diff --git a/clients/apple/Sources/PunktfunkKit/Audio/SessionAudio.swift b/clients/apple/Sources/PunktfunkKit/Audio/SessionAudio.swift index 18260734..bd228f7f 100644 --- a/clients/apple/Sources/PunktfunkKit/Audio/SessionAudio.swift +++ b/clients/apple/Sources/PunktfunkKit/Audio/SessionAudio.swift @@ -7,7 +7,7 @@ // // mic → host: a second AVAudioEngine taps the input device, folds it to one mono bus (the // chosen channel of a multi-channel interface, or a sum of all channels), resamples to 48 kHz -// stereo, slices 20 ms chunks, Opus-encodes, and sendMic()s each packet — the host feeds them +// mono, slices 10 ms chunks, Opus-encodes, and sendMic()s each packet — the host feeds them // into a virtual PipeWire source. // // Devices are chosen by UID ("" = system default: the engine is then never pinned to a @@ -104,6 +104,12 @@ public final class SessionAudio { try session.setCategory( .playAndRecord, mode: .default, options: [.allowBluetoothA2DP, .defaultToSpeaker]) + // Uplink latency: ask for 5 ms IO quanta at the wire rate (the default ~10-23 ms + // quantum is most of the mic path's burst latency). Best-effort — the hardware + // has the final word (a Bluetooth route will ignore both), and whatever quantum + // is actually granted, the capture tap handles the buffers it gets. + try? session.setPreferredIOBufferDuration(0.005) + try? session.setPreferredSampleRate(48_000) } else { try session.setCategory(.playback, mode: .default) } @@ -378,22 +384,38 @@ public final class SessionAudio { #endif // Encode a single mono bus (folded from `inFormat` in the tap): the resampler goes - // mono@inputSR → the encoder's 48 kHz stereo, so it handles both the rate change and the - // mono→stereo duplication, and the wrong-channel downmix never happens. + // mono@inputSR → the encoder's 48 kHz mono, so it handles the rate change and the + // wrong-channel downmix never happens. Mono end to end — the host's decoder upmixes, + // so the old duplicate-into-stereo step only cost bits and cycles. + // + // `mono`/`staging` are the per-callback scratch buffers, preallocated HERE (grown only + // if a larger-than-expected device quantum ever arrives) — the steady-state tap path + // allocates nothing. + let scratchFrames: AVAudioFrameCount = 8192 + let stagingCapacity = { (frames: AVAudioFrameCount) -> AVAudioFrameCount in + AVAudioFrameCount( + (Double(frames) * 48_000 / inFormat.sampleRate).rounded(.up)) + 64 + } guard let monoFormat = AVAudioFormat( commonFormat: .pcmFormatFloat32, sampleRate: inFormat.sampleRate, channels: 1, interleaved: false), let encoder = try? OpusEncoder(), let resampler = AVAudioConverter(from: monoFormat, to: encoder.pcmFormat), let chunk = AVAudioPCMBuffer( - pcmFormat: encoder.pcmFormat, frameCapacity: OpusEncoder.framesPerPacket) + pcmFormat: encoder.pcmFormat, frameCapacity: encoder.framesPerPacket), + let monoScratch = AVAudioPCMBuffer( + pcmFormat: monoFormat, frameCapacity: scratchFrames), + let stagingScratch = AVAudioPCMBuffer( + pcmFormat: encoder.pcmFormat, frameCapacity: stagingCapacity(scratchFrames)) else { log.error("Opus encoder unavailable — mic uplink disabled") return } - // Tap-thread-confined state: resample into `staging`, accumulate in `fifo`, - // slice 960-frame chunks for the encoder. + // Tap-thread-confined state: fold into `mono`, resample into `staging`, accumulate in + // `fifo`, slice `framesPerPacket` (10 ms) chunks for the encoder. + var mono = monoScratch + var staging = stagingScratch var fifo: [Float] = [] fifo.reserveCapacity(48_000) var seq: UInt32 = 0 @@ -412,14 +434,26 @@ public final class SessionAudio { var inputPeak: Float = 0 var levelReported = false - input.installTap(onBus: 0, bufferSize: 2048, format: inFormat) { buffer, _ in + // 480 frames = 10 ms, matching the packet duration. Advisory — CoreAudio delivers the + // device quantum whatever we ask (the old 2048 request came back as 42.7 ms bursts, most + // of the uplink's latency) — but where the system honors it, the tap fires per-packet. + input.installTap(onBus: 0, bufferSize: 480, format: inFormat) { buffer, _ in if flag.isStopped { return } let frames = Int(buffer.frameLength) - guard frames > 0, let src = buffer.floatChannelData, - let mono = AVAudioPCMBuffer( - pcmFormat: monoFormat, frameCapacity: buffer.frameLength), - let dst = mono.floatChannelData?[0] - else { return } + guard frames > 0, let src = buffer.floatChannelData else { return } + if frames > Int(mono.frameCapacity) { + // A quantum larger than the scratch (bufferSize is advisory both ways) — regrow + // once to the new high-water mark; the steady state stays allocation-free. + guard let biggerMono = AVAudioPCMBuffer( + pcmFormat: monoFormat, frameCapacity: buffer.frameLength), + let biggerStaging = AVAudioPCMBuffer( + pcmFormat: encoder.pcmFormat, + frameCapacity: stagingCapacity(buffer.frameLength)) + else { return } + mono = biggerMono + staging = biggerStaging + } + guard let dst = mono.floatChannelData?[0] else { return } mono.frameLength = buffer.frameLength // Fold the multi-channel input down to the one mono bus we encode. @@ -451,11 +485,6 @@ public final class SessionAudio { } } - let ratio = 48_000 / inFormat.sampleRate - let outCapacity = AVAudioFrameCount((Double(frames) * ratio).rounded(.up) + 64) - guard let staging = AVAudioPCMBuffer( - pcmFormat: encoder.pcmFormat, frameCapacity: outCapacity) - else { return } var fed = false var convError: NSError? let status = resampler.convert(to: staging, error: &convError) { _, outStatus in @@ -469,16 +498,20 @@ public final class SessionAudio { } guard status != .error, let p = staging.floatChannelData?[0] else { return } fifo.append(contentsOf: UnsafeBufferPointer( - start: p, count: Int(staging.frameLength) * 2)) + start: p, count: Int(staging.frameLength))) - let samplesPerChunk = Int(OpusEncoder.framesPerPacket) * 2 - while fifo.count >= samplesPerChunk { - chunk.frameLength = OpusEncoder.framesPerPacket + // Consume whole chunks through a head index, then drop the eaten prefix in ONE + // move of the sub-chunk remainder. The old per-chunk removeFirst memmoved the + // entire backlog for every packet — O(n) on the render-adjacent tap thread. + let samplesPerChunk = Int(encoder.framesPerPacket) + var head = 0 + while fifo.count - head >= samplesPerChunk { + chunk.frameLength = encoder.framesPerPacket fifo.withUnsafeBufferPointer { src in chunk.floatChannelData![0].update( - from: src.baseAddress!, count: samplesPerChunk) + from: src.baseAddress! + head, count: samplesPerChunk) } - fifo.removeFirst(samplesPerChunk) + head += samplesPerChunk guard let packets = try? encoder.encode(chunk) else { continue } for packet in packets { connection.sendMic( @@ -486,6 +519,7 @@ public final class SessionAudio { seq &+= 1 } } + if head > 0 { fifo.removeFirst(head) } // keeps capacity — no realloc } engine.prepare() diff --git a/clients/apple/Tests/PunktfunkKitTests/OpusCodecTests.swift b/clients/apple/Tests/PunktfunkKitTests/OpusCodecTests.swift index 39694419..73503e49 100644 --- a/clients/apple/Tests/PunktfunkKitTests/OpusCodecTests.swift +++ b/clients/apple/Tests/PunktfunkKitTests/OpusCodecTests.swift @@ -9,32 +9,34 @@ import XCTest @testable import PunktfunkKit final class OpusCodecTests: XCTestCase { - /// Encode a 440 Hz stereo tone, decode it back, and require the result to be - /// recognizably the same signal (Opus is lossy — check correlation, not bytes). + /// Encode a 440 Hz mono tone (the uplink's shape), decode it back through the + /// STEREO-configured decoder (the host-plane shape — Opus upmixes mono packets), and + /// require the result to be recognizably the same signal (Opus is lossy — check + /// correlation, not bytes). func testEncodeDecodeRoundTripPreservesTone() throws { let encoder = try OpusEncoder() - let decoder = try OpusDecoder(framesPerPacket: UInt32(OpusEncoder.framesPerPacket)) + let decoder = try OpusDecoder(framesPerPacket: UInt32(encoder.framesPerPacket)) let pcmFormat = encoder.pcmFormat - let frames = OpusEncoder.framesPerPacket + let frames = encoder.framesPerPacket var packets: [Data] = [] var phase: Float = 0 let step = 2 * Float.pi * 440 / 48_000 - // 50 packets = 1 s of tone. - for _ in 0..<50 { + // 1 s of tone, whatever packet duration the encoder chose (10 ms → 100 chunks). + let chunks = Int(48_000 / frames) + for _ in 0..