feat(apple): the mic uplink drops to mono 10 ms packets and stops shoveling its FIFO
The capture path carried three latencies that had nothing to do with the network: a 2048-frame tap request (42.7 ms bursts before the first sample could even be sliced), 20 ms Opus framing, and a mono signal duplicated into both stereo channels because the encoder was configured like the downlink. CoreAudio's Opus encoder takes mFramesPerPacket=480 and mChannelsPerFrame=1 just fine — probed empirically: the converter truly emits 10 ms mono CELT packets (TOC config 30), one per 480-frame chunk, and the stereo-shaped decoder upmixes them with the tone intact — so the uplink now asks for 10 ms tap buffers and encodes 48 kbps mono 10 ms packets, with 960 kept only as an init-time fallback. The host decodes any Opus frame ≤120 ms, so nothing changes on the wire's far end. The tap thread also stops burning cycles per callback: the mono fold and resampler scratch buffers are allocated once (regrown only if a larger device quantum ever arrives), and the chunk slicer walks a head index instead of removeFirst — which memmoved the entire backlog for every packet on a render-adjacent thread. iOS additionally asks the session for 5 ms IO quanta at 48 kHz when the mic is on (best-effort; the hardware decides). Verified: swift build (macOS arm64), swift test OpusCodecTests + AudioChannelFoldTests, and an iOS arm64 cross-build; the 480/mono behavior confirmed by TOC inspection on macOS 15. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -1,7 +1,9 @@
|
||||
// Opus ⇄ PCM through CoreAudio's built-in codec (kAudioFormatOpus, macOS 10.13+ / iOS
|
||||
// 11+) — no bundled libopus. The host's audio plane is raw Opus packets (48 kHz stereo,
|
||||
// one frame per packet); AVAudioConverter handles them as single-packet
|
||||
// AVAudioCompressedBuffers with explicit packet descriptions.
|
||||
// one frame per packet); the mic uplink is 48 kHz MONO packets (one microphone bus —
|
||||
// the host's decoder upmixes, so duplicating it into a second channel only cost bits).
|
||||
// AVAudioConverter handles both as single-packet AVAudioCompressedBuffers with explicit
|
||||
// packet descriptions.
|
||||
//
|
||||
// Both classes are single-threaded by contract (one per direction, owned by their
|
||||
// drain/capture pipelines).
|
||||
@@ -14,16 +16,16 @@ enum OpusCodecError: Error {
|
||||
case convertFailed(String)
|
||||
}
|
||||
|
||||
/// 48 kHz stereo float32 interleaved — the PCM side of both converters and the layout
|
||||
/// of the playback ring buffer.
|
||||
/// 48 kHz stereo float32 interleaved — the decoder's PCM side (the host plane's shape).
|
||||
func opusPCMFormat() -> AVAudioFormat? {
|
||||
AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32, sampleRate: 48_000, channels: 2, interleaved: true)
|
||||
}
|
||||
|
||||
/// The compressed side: raw Opus, `framesPerPacket` nominal samples per packet at 48 kHz
|
||||
/// (240 = the host's 5 ms audio plane; 960 = the 20 ms packets the encoder emits).
|
||||
private func opusFormat(framesPerPacket: UInt32) -> AVAudioFormat? {
|
||||
/// (240 = the host's 5 ms audio plane; 480 = the 10 ms packets the encoder emits) and
|
||||
/// `channels` (2 = the host plane, 1 = the mic uplink).
|
||||
private func opusFormat(framesPerPacket: UInt32, channels: UInt32) -> AVAudioFormat? {
|
||||
var desc = AudioStreamBasicDescription(
|
||||
mSampleRate: 48_000,
|
||||
mFormatID: kAudioFormatOpus,
|
||||
@@ -31,7 +33,7 @@ private func opusFormat(framesPerPacket: UInt32) -> AVAudioFormat? {
|
||||
mBytesPerPacket: 0,
|
||||
mFramesPerPacket: framesPerPacket,
|
||||
mBytesPerFrame: 0,
|
||||
mChannelsPerFrame: 2,
|
||||
mChannelsPerFrame: channels,
|
||||
mBitsPerChannel: 0,
|
||||
mReserved: 0)
|
||||
return AVAudioFormat(streamDescription: &desc)
|
||||
@@ -45,7 +47,8 @@ final class OpusDecoder {
|
||||
|
||||
/// `framesPerPacket`: the sender's packet duration in samples (host audio = 240).
|
||||
init(framesPerPacket: UInt32) throws {
|
||||
guard let pcm = opusPCMFormat(), let opus = opusFormat(framesPerPacket: framesPerPacket),
|
||||
guard let pcm = opusPCMFormat(),
|
||||
let opus = opusFormat(framesPerPacket: framesPerPacket, channels: 2),
|
||||
let converter = AVAudioConverter(from: opus, to: pcm)
|
||||
else { throw OpusCodecError.unavailable }
|
||||
self.converter = converter
|
||||
@@ -90,24 +93,43 @@ final class OpusDecoder {
|
||||
}
|
||||
|
||||
final class OpusEncoder {
|
||||
/// The encoder's packet duration: 960 samples = 20 ms, CoreAudio's default Opus
|
||||
/// framing. The host's mic service decodes any Opus frame size up to 120 ms.
|
||||
static let framesPerPacket: AVAudioFrameCount = 960
|
||||
/// The encoder's packet duration in samples: 480 = 10 ms, halving the packetization
|
||||
/// latency of the old 20 ms framing. CoreAudio honors it — mFramesPerPacket 480/mono
|
||||
/// creates a converter that truly emits 10 ms CELT packets (TOC config 30), one per
|
||||
/// 480-frame chunk, verified by inspection of the emitted TOC bytes and per-packet
|
||||
/// frame accounting. 960 stays as the fallback should an older codec refuse 480.
|
||||
/// The host's mic service decodes any Opus frame size up to 120 ms, so either is
|
||||
/// wire-compatible.
|
||||
let framesPerPacket: AVAudioFrameCount
|
||||
|
||||
/// 48 kHz MONO float32 interleaved — the uplink carries one microphone bus.
|
||||
let pcmFormat: AVAudioFormat
|
||||
|
||||
private let converter: AVAudioConverter
|
||||
private let outBuf: AVAudioCompressedBuffer
|
||||
let pcmFormat: AVAudioFormat
|
||||
|
||||
init() throws {
|
||||
guard let pcm = opusPCMFormat(),
|
||||
let opus = opusFormat(framesPerPacket: UInt32(Self.framesPerPacket)),
|
||||
let converter = AVAudioConverter(from: pcm, to: opus)
|
||||
guard let pcm = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32, sampleRate: 48_000, channels: 1,
|
||||
interleaved: true)
|
||||
else { throw OpusCodecError.unavailable }
|
||||
converter.bitRate = 96_000
|
||||
self.converter = converter
|
||||
self.pcmFormat = pcm
|
||||
var made: (converter: AVAudioConverter, fpp: AVAudioFrameCount)?
|
||||
for fpp: AVAudioFrameCount in [480, 960] {
|
||||
if let opus = opusFormat(framesPerPacket: UInt32(fpp), channels: 1),
|
||||
let converter = AVAudioConverter(from: pcm, to: opus) {
|
||||
made = (converter, fpp)
|
||||
break
|
||||
}
|
||||
}
|
||||
guard let made else { throw OpusCodecError.unavailable }
|
||||
// 48 kbps: transparent for mono voice — the old 96 kbps budget was sized for
|
||||
// the duplicated-stereo framing this encoder no longer emits.
|
||||
made.converter.bitRate = 48_000
|
||||
converter = made.converter
|
||||
framesPerPacket = made.fpp
|
||||
pcmFormat = pcm
|
||||
outBuf = AVAudioCompressedBuffer(
|
||||
format: opus, packetCapacity: 4, maximumPacketSize: 1500)
|
||||
format: made.converter.outputFormat, packetCapacity: 4, maximumPacketSize: 1500)
|
||||
}
|
||||
|
||||
/// Encode exactly `framesPerPacket` frames of `pcmFormat` audio; returns the encoded
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
//
|
||||
// mic → host: a second AVAudioEngine taps the input device, folds it to one mono bus (the
|
||||
// chosen channel of a multi-channel interface, or a sum of all channels), resamples to 48 kHz
|
||||
// stereo, slices 20 ms chunks, Opus-encodes, and sendMic()s each packet — the host feeds them
|
||||
// mono, slices 10 ms chunks, Opus-encodes, and sendMic()s each packet — the host feeds them
|
||||
// into a virtual PipeWire source.
|
||||
//
|
||||
// Devices are chosen by UID ("" = system default: the engine is then never pinned to a
|
||||
@@ -104,6 +104,12 @@ public final class SessionAudio {
|
||||
try session.setCategory(
|
||||
.playAndRecord, mode: .default,
|
||||
options: [.allowBluetoothA2DP, .defaultToSpeaker])
|
||||
// Uplink latency: ask for 5 ms IO quanta at the wire rate (the default ~10-23 ms
|
||||
// quantum is most of the mic path's burst latency). Best-effort — the hardware
|
||||
// has the final word (a Bluetooth route will ignore both), and whatever quantum
|
||||
// is actually granted, the capture tap handles the buffers it gets.
|
||||
try? session.setPreferredIOBufferDuration(0.005)
|
||||
try? session.setPreferredSampleRate(48_000)
|
||||
} else {
|
||||
try session.setCategory(.playback, mode: .default)
|
||||
}
|
||||
@@ -378,22 +384,38 @@ public final class SessionAudio {
|
||||
#endif
|
||||
|
||||
// Encode a single mono bus (folded from `inFormat` in the tap): the resampler goes
|
||||
// mono@inputSR → the encoder's 48 kHz stereo, so it handles both the rate change and the
|
||||
// mono→stereo duplication, and the wrong-channel downmix never happens.
|
||||
// mono@inputSR → the encoder's 48 kHz mono, so it handles the rate change and the
|
||||
// wrong-channel downmix never happens. Mono end to end — the host's decoder upmixes,
|
||||
// so the old duplicate-into-stereo step only cost bits and cycles.
|
||||
//
|
||||
// `mono`/`staging` are the per-callback scratch buffers, preallocated HERE (grown only
|
||||
// if a larger-than-expected device quantum ever arrives) — the steady-state tap path
|
||||
// allocates nothing.
|
||||
let scratchFrames: AVAudioFrameCount = 8192
|
||||
let stagingCapacity = { (frames: AVAudioFrameCount) -> AVAudioFrameCount in
|
||||
AVAudioFrameCount(
|
||||
(Double(frames) * 48_000 / inFormat.sampleRate).rounded(.up)) + 64
|
||||
}
|
||||
guard let monoFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32, sampleRate: inFormat.sampleRate,
|
||||
channels: 1, interleaved: false),
|
||||
let encoder = try? OpusEncoder(),
|
||||
let resampler = AVAudioConverter(from: monoFormat, to: encoder.pcmFormat),
|
||||
let chunk = AVAudioPCMBuffer(
|
||||
pcmFormat: encoder.pcmFormat, frameCapacity: OpusEncoder.framesPerPacket)
|
||||
pcmFormat: encoder.pcmFormat, frameCapacity: encoder.framesPerPacket),
|
||||
let monoScratch = AVAudioPCMBuffer(
|
||||
pcmFormat: monoFormat, frameCapacity: scratchFrames),
|
||||
let stagingScratch = AVAudioPCMBuffer(
|
||||
pcmFormat: encoder.pcmFormat, frameCapacity: stagingCapacity(scratchFrames))
|
||||
else {
|
||||
log.error("Opus encoder unavailable — mic uplink disabled")
|
||||
return
|
||||
}
|
||||
|
||||
// Tap-thread-confined state: resample into `staging`, accumulate in `fifo`,
|
||||
// slice 960-frame chunks for the encoder.
|
||||
// Tap-thread-confined state: fold into `mono`, resample into `staging`, accumulate in
|
||||
// `fifo`, slice `framesPerPacket` (10 ms) chunks for the encoder.
|
||||
var mono = monoScratch
|
||||
var staging = stagingScratch
|
||||
var fifo: [Float] = []
|
||||
fifo.reserveCapacity(48_000)
|
||||
var seq: UInt32 = 0
|
||||
@@ -412,14 +434,26 @@ public final class SessionAudio {
|
||||
var inputPeak: Float = 0
|
||||
var levelReported = false
|
||||
|
||||
input.installTap(onBus: 0, bufferSize: 2048, format: inFormat) { buffer, _ in
|
||||
// 480 frames = 10 ms, matching the packet duration. Advisory — CoreAudio delivers the
|
||||
// device quantum whatever we ask (the old 2048 request came back as 42.7 ms bursts, most
|
||||
// of the uplink's latency) — but where the system honors it, the tap fires per-packet.
|
||||
input.installTap(onBus: 0, bufferSize: 480, format: inFormat) { buffer, _ in
|
||||
if flag.isStopped { return }
|
||||
let frames = Int(buffer.frameLength)
|
||||
guard frames > 0, let src = buffer.floatChannelData,
|
||||
let mono = AVAudioPCMBuffer(
|
||||
pcmFormat: monoFormat, frameCapacity: buffer.frameLength),
|
||||
let dst = mono.floatChannelData?[0]
|
||||
else { return }
|
||||
guard frames > 0, let src = buffer.floatChannelData else { return }
|
||||
if frames > Int(mono.frameCapacity) {
|
||||
// A quantum larger than the scratch (bufferSize is advisory both ways) — regrow
|
||||
// once to the new high-water mark; the steady state stays allocation-free.
|
||||
guard let biggerMono = AVAudioPCMBuffer(
|
||||
pcmFormat: monoFormat, frameCapacity: buffer.frameLength),
|
||||
let biggerStaging = AVAudioPCMBuffer(
|
||||
pcmFormat: encoder.pcmFormat,
|
||||
frameCapacity: stagingCapacity(buffer.frameLength))
|
||||
else { return }
|
||||
mono = biggerMono
|
||||
staging = biggerStaging
|
||||
}
|
||||
guard let dst = mono.floatChannelData?[0] else { return }
|
||||
mono.frameLength = buffer.frameLength
|
||||
|
||||
// Fold the multi-channel input down to the one mono bus we encode.
|
||||
@@ -451,11 +485,6 @@ public final class SessionAudio {
|
||||
}
|
||||
}
|
||||
|
||||
let ratio = 48_000 / inFormat.sampleRate
|
||||
let outCapacity = AVAudioFrameCount((Double(frames) * ratio).rounded(.up) + 64)
|
||||
guard let staging = AVAudioPCMBuffer(
|
||||
pcmFormat: encoder.pcmFormat, frameCapacity: outCapacity)
|
||||
else { return }
|
||||
var fed = false
|
||||
var convError: NSError?
|
||||
let status = resampler.convert(to: staging, error: &convError) { _, outStatus in
|
||||
@@ -469,16 +498,20 @@ public final class SessionAudio {
|
||||
}
|
||||
guard status != .error, let p = staging.floatChannelData?[0] else { return }
|
||||
fifo.append(contentsOf: UnsafeBufferPointer(
|
||||
start: p, count: Int(staging.frameLength) * 2))
|
||||
start: p, count: Int(staging.frameLength)))
|
||||
|
||||
let samplesPerChunk = Int(OpusEncoder.framesPerPacket) * 2
|
||||
while fifo.count >= samplesPerChunk {
|
||||
chunk.frameLength = OpusEncoder.framesPerPacket
|
||||
// Consume whole chunks through a head index, then drop the eaten prefix in ONE
|
||||
// move of the sub-chunk remainder. The old per-chunk removeFirst memmoved the
|
||||
// entire backlog for every packet — O(n) on the render-adjacent tap thread.
|
||||
let samplesPerChunk = Int(encoder.framesPerPacket)
|
||||
var head = 0
|
||||
while fifo.count - head >= samplesPerChunk {
|
||||
chunk.frameLength = encoder.framesPerPacket
|
||||
fifo.withUnsafeBufferPointer { src in
|
||||
chunk.floatChannelData![0].update(
|
||||
from: src.baseAddress!, count: samplesPerChunk)
|
||||
from: src.baseAddress! + head, count: samplesPerChunk)
|
||||
}
|
||||
fifo.removeFirst(samplesPerChunk)
|
||||
head += samplesPerChunk
|
||||
guard let packets = try? encoder.encode(chunk) else { continue }
|
||||
for packet in packets {
|
||||
connection.sendMic(
|
||||
@@ -486,6 +519,7 @@ public final class SessionAudio {
|
||||
seq &+= 1
|
||||
}
|
||||
}
|
||||
if head > 0 { fifo.removeFirst(head) } // keeps capacity — no realloc
|
||||
}
|
||||
|
||||
engine.prepare()
|
||||
|
||||
@@ -9,32 +9,34 @@ import XCTest
|
||||
@testable import PunktfunkKit
|
||||
|
||||
final class OpusCodecTests: XCTestCase {
|
||||
/// Encode a 440 Hz stereo tone, decode it back, and require the result to be
|
||||
/// recognizably the same signal (Opus is lossy — check correlation, not bytes).
|
||||
/// Encode a 440 Hz mono tone (the uplink's shape), decode it back through the
|
||||
/// STEREO-configured decoder (the host-plane shape — Opus upmixes mono packets), and
|
||||
/// require the result to be recognizably the same signal (Opus is lossy — check
|
||||
/// correlation, not bytes).
|
||||
func testEncodeDecodeRoundTripPreservesTone() throws {
|
||||
let encoder = try OpusEncoder()
|
||||
let decoder = try OpusDecoder(framesPerPacket: UInt32(OpusEncoder.framesPerPacket))
|
||||
let decoder = try OpusDecoder(framesPerPacket: UInt32(encoder.framesPerPacket))
|
||||
let pcmFormat = encoder.pcmFormat
|
||||
|
||||
let frames = OpusEncoder.framesPerPacket
|
||||
let frames = encoder.framesPerPacket
|
||||
var packets: [Data] = []
|
||||
var phase: Float = 0
|
||||
let step = 2 * Float.pi * 440 / 48_000
|
||||
|
||||
// 50 packets = 1 s of tone.
|
||||
for _ in 0..<50 {
|
||||
// 1 s of tone, whatever packet duration the encoder chose (10 ms → 100 chunks).
|
||||
let chunks = Int(48_000 / frames)
|
||||
for _ in 0..<chunks {
|
||||
let buf = AVAudioPCMBuffer(pcmFormat: pcmFormat, frameCapacity: frames)!
|
||||
buf.frameLength = frames
|
||||
let p = buf.floatChannelData![0] // interleaved: one plane, L R L R …
|
||||
let p = buf.floatChannelData![0] // mono: one plane
|
||||
for f in 0..<Int(frames) {
|
||||
let s = sin(phase) * 0.5
|
||||
p[f] = sin(phase) * 0.5
|
||||
phase += step
|
||||
p[f * 2] = s
|
||||
p[f * 2 + 1] = s
|
||||
}
|
||||
packets.append(contentsOf: try encoder.encode(buf))
|
||||
}
|
||||
XCTAssertGreaterThanOrEqual(packets.count, 45, "encoder must emit ~one packet per buffer")
|
||||
XCTAssertGreaterThanOrEqual(
|
||||
packets.count, chunks - 5, "encoder must emit ~one packet per buffer")
|
||||
XCTAssertTrue(packets.allSatisfy { !$0.isEmpty })
|
||||
|
||||
var decoded: [Float] = []
|
||||
|
||||
@@ -62,29 +62,30 @@ final class RemoteFirstLightTests: XCTestCase {
|
||||
host: host, port: port, width: 1280, height: 720, refreshHz: 60)
|
||||
defer { conn.close() }
|
||||
|
||||
// Mic uplink: 2 s of 440 Hz tone (the host's mic service opens its virtual
|
||||
// Mic uplink: 2 s of 440 Hz mono tone (the host's mic service opens its virtual
|
||||
// source on the first frame — check its log).
|
||||
let encoder = try OpusEncoder()
|
||||
let chunk = AVAudioPCMBuffer(
|
||||
pcmFormat: encoder.pcmFormat, frameCapacity: OpusEncoder.framesPerPacket)!
|
||||
pcmFormat: encoder.pcmFormat, frameCapacity: encoder.framesPerPacket)!
|
||||
var phase: Float = 0
|
||||
let step = 2 * Float.pi * 440 / 48_000
|
||||
var seq: UInt32 = 0
|
||||
for _ in 0..<100 {
|
||||
chunk.frameLength = OpusEncoder.framesPerPacket
|
||||
let p = chunk.floatChannelData![0]
|
||||
for f in 0..<Int(OpusEncoder.framesPerPacket) {
|
||||
let s = sin(phase) * 0.25
|
||||
let chunks = 2 * 48_000 / Int(encoder.framesPerPacket)
|
||||
let packetNs = UInt64(encoder.framesPerPacket) * 1_000_000_000 / 48_000
|
||||
for _ in 0..<chunks {
|
||||
chunk.frameLength = encoder.framesPerPacket
|
||||
let p = chunk.floatChannelData![0] // mono: one plane
|
||||
for f in 0..<Int(encoder.framesPerPacket) {
|
||||
p[f] = sin(phase) * 0.25
|
||||
phase += step
|
||||
p[f * 2] = s
|
||||
p[f * 2 + 1] = s
|
||||
}
|
||||
for packet in try encoder.encode(chunk) {
|
||||
conn.sendMic(packet, seq: seq, ptsNs: UInt64(seq) * 20_000_000)
|
||||
conn.sendMic(packet, seq: seq, ptsNs: UInt64(seq) * packetNs)
|
||||
seq &+= 1
|
||||
}
|
||||
}
|
||||
XCTAssertGreaterThanOrEqual(seq, 95, "mic encoder must emit ~one packet per chunk")
|
||||
XCTAssertGreaterThanOrEqual(
|
||||
seq, UInt32(chunks - 5), "mic encoder must emit ~one packet per chunk")
|
||||
|
||||
// Downlink: pull host audio packets and decode them (the host streams its sink
|
||||
// monitor — silence still produces packets).
|
||||
|
||||
Reference in New Issue
Block a user