From a7d42137786d0cf6fde0af60716f26fadbe5ad6c Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 31 Jul 2026 21:46:44 +0200 Subject: [PATCH] feat(android): the mic uplink drops to mono 10 ms frames, and stops hoarding stale audio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Latency, three ways, all inside mic.rs: - 48 kHz stereo 20 ms becomes mono 10 ms: speech gains nothing from a second channel, the shorter frame shaves a buffering interval off the uplink, and the host already decodes any Opus frame <= 120 ms with its stereo decoder (mono packets upmix) — no protocol change. The encoder follows: 48 kbps, complexity 5, in-band FEC at an assumed 10% loss so a dropped datagram reconstructs from its successor instead of a hole. - The latency ratchet is gone: the capture callback drops the NEWEST chunk when the hand-off channel fills, so an encode-side stall used to convert into standing mic delay that never drained. The encode loop now drains the whole backlog in one lump and, past ~60 ms, jumps to the newest ~20 ms (one audible blip, live again), counting what it shed in the periodic log line. The realtime callback stays exactly as allocation-free as it was. - The encode thread registers with the client's hot-thread set, so the ADPF session keeps mic encode on a fast core alongside audio decode. No .frames_per_data_callback() pin: AAudio's own docs say leaving it unset is the lowest-latency path (the callback runs at the device's optimal burst), and the encode side re-chunks to 10 ms frames anyway. Co-Authored-By: Claude Fable 5 --- clients/android/native/src/mic.rs | 85 ++++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 23 deletions(-) diff --git a/clients/android/native/src/mic.rs b/clients/android/native/src/mic.rs index 6ae164e1..23cb2172 100644 --- a/clients/android/native/src/mic.rs +++ b/clients/android/native/src/mic.rs @@ -1,12 +1,14 @@ //! Android microphone uplink (android-only): capture mic PCM via AAudio (LowLatency **input**), -//! Opus-encode 20 ms stereo frames, and push them to the host over the connector's mic plane +//! Opus-encode 10 ms mono frames, and push them to the host over the connector's mic plane //! (`send_mic` → 0xCB datagram). The mirror of [`crate::audio`] in reverse: AAudio's realtime input -//! callback hands captured interleaved f32 to a channel; a worker thread we own does the Opus -//! encode + send (encoding is too heavy for the realtime callback, exactly as decode is on the -//! playback side). Like the playback path, the realtime callback is allocation-free: captured -//! bursts are copied into pre-allocated buffers from a recycle free-list (pool empty = drop the -//! chunk, never allocate on the capture thread). Format matches the host decoder + the Linux -//! client: 48 kHz **stereo**, 20 ms, Opus VOIP. +//! callback hands captured f32 to a channel; a worker thread we own does the Opus encode + send +//! (encoding is too heavy for the realtime callback, exactly as decode is on the playback side). +//! Like the playback path, the realtime callback is allocation-free: captured bursts are copied +//! into pre-allocated buffers from a recycle free-list (pool empty = drop the chunk, never +//! allocate on the capture thread). Format: 48 kHz **mono**, 10 ms, Opus VOIP with in-band FEC — +//! the host decodes any Opus frame ≤ 120 ms with its stereo decoder (mono packets upmix), so this +//! needs no protocol change; speech gains nothing from stereo, and the shorter frame shaves a +//! buffering interval off the uplink. use ndk::audio::{ AudioCallbackResult, AudioDirection, AudioFormat, AudioPerformanceMode, AudioSharingMode, @@ -20,18 +22,28 @@ use std::sync::mpsc::{sync_channel, Receiver, RecvTimeoutError, SyncSender, TryS use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -const CHANNELS: usize = 2; +const CHANNELS: usize = 1; const SAMPLE_RATE: i32 = 48_000; -/// 20 ms per channel @ 48 kHz — the Linux client's frame; the host accepts ≤ 120 ms. -const FRAME_SAMPLES: usize = 960; +/// 10 ms per channel @ 48 kHz — half the desktop clients' 20 ms frame, trading a little Opus +/// header overhead for one less buffered interval; the host accepts ≤ 120 ms. +const FRAME_SAMPLES: usize = 480; /// Captured-chunk hand-off depth (each ~ one burst); drops on overflow (best-effort uplink). +/// Bursts are sized in frames, so the wall-time depth is unchanged by the stereo→mono move. const RING_CHUNKS: usize = 64; /// Free-list buffer capacity, in interleaved f32 samples: comfortably above a LowLatency input -/// burst (typically ≤ ~480 frames). A device with larger bursts costs each buffer a one-time grow -/// on the capture thread, after which the steady state is allocation-free again. -const CHUNK_CAP_SAMPLES: usize = 1920; // 20 ms stereo -/// Opus VOIP target bitrate (speech; tunable). -const MIC_BITRATE: i32 = 64_000; +/// burst (typically ≤ ~480 frames — mono, so samples = frames). A device with larger bursts costs +/// each buffer a one-time grow on the capture thread, after which the steady state is +/// allocation-free again. +const CHUNK_CAP_SAMPLES: usize = 960; // 20 ms mono — the same wall-time as the old stereo value +/// Opus VOIP target bitrate (mono speech; tunable). +const MIC_BITRATE: i32 = 48_000; +/// Encode-side self-heal threshold, in queued 10 ms frames (~60 ms): waking to more than this +/// means the uplink stalled — and because the capture callback drops the NEWEST chunk when the +/// channel is full, a stall otherwise converts to standing mic delay that never drains (real-time +/// playback host-side never makes time back up). Skip to the newest few frames instead. +const BACKLOG_MAX_FRAMES: usize = 6; +/// What a self-heal keeps: ~20 ms of the freshest audio (one audible blip, live again). +const BACKLOG_KEEP_FRAMES: usize = 2; /// Owned by [`crate::session::SessionHandle`]: the live AAudio input stream + the encode thread. pub struct MicCapture { @@ -41,7 +53,7 @@ pub struct MicCapture { } impl MicCapture { - /// Open AAudio (LowLatency, 48 kHz/stereo/f32) for **input** with a realtime callback that + /// Open AAudio (LowLatency, 48 kHz/mono/f32) for **input** with a realtime callback that /// forwards captured PCM to a channel, then spawn the Opus encode + uplink thread. `None` on /// failure (the caller leaves the rest of the session streaming). pub fn start(client: Arc) -> Option { @@ -168,7 +180,7 @@ impl Drop for MicCapture { } } -/// Consumer: drain captured f32 → accumulate → Opus `encode_float` 20 ms stereo frames → `send_mic`. +/// Consumer: drain captured f32 → accumulate → Opus `encode_float` 10 ms mono frames → `send_mic`. /// Drained chunk buffers go back to the callback's free-list; the encode scratch is reused across /// frames (only the packet Vec handed to `send_mic` is allocated per frame — it's sent away owned). fn encode_loop( @@ -179,9 +191,13 @@ fn encode_loop( captured: Arc, dropped: Arc, ) { + // Fold this Opus-encode/uplink thread into the client's hot-thread set so the ADPF session the + // decode thread opens keeps mic encode on a fast core too (the playback side's decode_loop + // does the same). No-op below API 33. + client.register_hot_thread(); let mut enc = match opus::Encoder::new( SAMPLE_RATE as u32, - opus::Channels::Stereo, + opus::Channels::Mono, opus::Application::Voip, ) { Ok(e) => e, @@ -191,13 +207,20 @@ fn encode_loop( } }; let _ = enc.set_bitrate(opus::Bitrate::Bits(MIC_BITRATE)); + // Speech tuning: complexity 5 roughly halves encode cost for no audible loss at this rate, + // and in-band FEC at an assumed 10% loss lets the host's decoder reconstruct a dropped + // datagram from its successor instead of playing a hole (the uplink is fire-and-forget). + let _ = enc.set_complexity(5); + let _ = enc.set_inband_fec(true); + let _ = enc.set_packet_loss_perc(10); let frame = FRAME_SAMPLES * CHANNELS; let mut ring: VecDeque = VecDeque::with_capacity(frame * 4); - let mut pcm = vec![0f32; frame]; // reusable encode scratch (one 20 ms frame) - let mut out = vec![0u8; 4000]; // max Opus packet for a 20 ms frame fits easily + let mut pcm = vec![0f32; frame]; // reusable encode scratch (one 10 ms frame) + let mut out = vec![0u8; 4000]; // max Opus packet for a 10 ms frame fits easily let mut seq: u32 = 0; let mut sent: u64 = 0; + let mut stale: u64 = 0; // frames shed by the backlog self-heal (see BACKLOG_MAX_FRAMES) let mut peak = 0f32; // loudest |sample| since the last log — tells speech from silence while !shutdown.load(Ordering::Relaxed) { @@ -207,10 +230,25 @@ fn encode_loop( // callback's free-list (dropped only if the pool is momentarily full). ring.extend(chunk.drain(..)); let _ = free_tx.try_send(chunk); + // Drain whatever else queued while we were away, so a post-stall backlog lands as + // ONE lump the self-heal below can size up — chunk-at-a-time it would be encoded + // (and inflicted on the host as standing delay) before it ever looked deep. + while let Ok(mut chunk) = rx.try_recv() { + ring.extend(chunk.drain(..)); + let _ = free_tx.try_send(chunk); + } } Err(RecvTimeoutError::Timeout) => continue, // wake to re-check shutdown Err(RecvTimeoutError::Disconnected) => break, } + // Self-heal the latency ratchet: a stall (scheduler hiccup, a slow send) queues stale + // audio, and every ms of it would ride the stream as mic delay for the rest of the + // session. Jump to the newest ~20 ms (one audible blip), counting the shed. + if ring.len() > BACKLOG_MAX_FRAMES * frame { + let excess = ring.len() - BACKLOG_KEEP_FRAMES * frame; + ring.drain(..excess); + stale += (excess / frame) as u64; + } while ring.len() >= frame { for (dst, src) in pcm.iter_mut().zip(ring.drain(..frame)) { *dst = src; @@ -227,9 +265,10 @@ fn encode_loop( let _ = client.send_mic(seq, pts, out[..len].to_vec()); seq = seq.wrapping_add(1); sent += 1; - if sent % 250 == 0 { + if sent % 500 == 0 { log::info!( - "mic: sent={sent} captured_frames={} dropped_chunks={} peak={peak:.3}", + "mic: sent={sent} captured_frames={} dropped_chunks={} \ + stale_frames={stale} peak={peak:.3}", captured.load(Ordering::Relaxed), dropped.load(Ordering::Relaxed), ); @@ -241,7 +280,7 @@ fn encode_loop( } } log::info!( - "mic: stopped (sent={sent} captured_frames={} dropped_chunks={})", + "mic: stopped (sent={sent} captured_frames={} dropped_chunks={} stale_frames={stale})", captured.load(Ordering::Relaxed), dropped.load(Ordering::Relaxed), );