diff --git a/clients/android/native/src/audio.rs b/clients/android/native/src/audio.rs index 5a538f6f..2a76dcba 100644 --- a/clients/android/native/src/audio.rs +++ b/clients/android/native/src/audio.rs @@ -355,45 +355,70 @@ fn decode_loop( }; let mut pcm = vec![0f32; pcm_scratch]; let mut window_peak = 0f32; // loudest |sample| since the last log — tells a tone from silence - while !shutdown.load(Ordering::Relaxed) { + let mut gaps = punktfunk_core::audio::AudioGapTracker::new(); + let mut frame_samples = 0usize; // per-channel samples of the last decoded frame — the PLC unit + 'pump: while !shutdown.load(Ordering::Relaxed) { match client.next_audio(Duration::from_millis(5)) { - Ok(pkt) => match dec.decode_float(&pkt.data, &mut pcm, false) { - Ok(samples) => { - let n = samples * channels; - for &s in &pcm[..n] { - window_peak = window_peak.max(s.abs()); + Ok(pkt) => { + // Conceal lost packets (a seq gap) with libopus PLC before decoding the one that + // arrived: empty input synthesizes `frame_samples` of interpolation per missing + // packet — an inaudible fade instead of the click a hard gap makes in the ring. + for _ in 0..gaps.missing_before(pkt.seq) { + let plc = frame_samples * channels; + if plc == 0 { + break; // no decoded frame yet to size the concealment from } - // The ring's pre-reservation in `start` assumes the protocol's 5 ms (≤480-f32/ch) - // frames; a larger frame would force a one-time realloc on the RT thread. Catch a - // future host frame-size change here in debug, not as a silent audio glitch. - debug_assert!( - n <= 5 * ms, - "audio frame {n} f32 exceeds the 5 ms ring reserve" - ); - let count = counters.opus_decoded.fetch_add(1, Ordering::Relaxed) + 1; - // Reuse a recycled buffer if the callback handed one back; only allocate when the - // free-list is momentarily empty (startup / after a backpressure drop). - let mut buf = free_rx - .try_recv() - .unwrap_or_else(|_| Vec::with_capacity(pcm_scratch)); - buf.clear(); - buf.extend_from_slice(&pcm[..n]); - match tx.try_send(buf) { - Ok(()) | Err(TrySendError::Full(_)) => {} // drop-newest under backpressure - Err(TrySendError::Disconnected(_)) => break, - } - if count % 600 == 0 { - log::info!( - "audio: opus={count} pcm_frames={} underruns={} ring={} peak={window_peak:.3}", - counters.pcm_written.load(Ordering::Relaxed), - counters.underruns.load(Ordering::Relaxed), - counters.ring_depth.load(Ordering::Relaxed), - ); - window_peak = 0.0; + if let Ok(samples) = dec.decode_float(&[], &mut pcm[..plc], false) { + let mut buf = free_rx + .try_recv() + .unwrap_or_else(|_| Vec::with_capacity(pcm_scratch)); + buf.clear(); + buf.extend_from_slice(&pcm[..samples * channels]); + match tx.try_send(buf) { + Ok(()) | Err(TrySendError::Full(_)) => {} + Err(TrySendError::Disconnected(_)) => break 'pump, + } } } - Err(e) => log::debug!("audio: opus decode: {e}"), - }, + match dec.decode_float(&pkt.data, &mut pcm, false) { + Ok(samples) => { + frame_samples = samples; + let n = samples * channels; + for &s in &pcm[..n] { + window_peak = window_peak.max(s.abs()); + } + // The ring's pre-reservation in `start` assumes the protocol's 5 ms (≤480-f32/ch) + // frames; a larger frame would force a one-time realloc on the RT thread. Catch a + // future host frame-size change here in debug, not as a silent audio glitch. + debug_assert!( + n <= 5 * ms, + "audio frame {n} f32 exceeds the 5 ms ring reserve" + ); + let count = counters.opus_decoded.fetch_add(1, Ordering::Relaxed) + 1; + // Reuse a recycled buffer if the callback handed one back; only allocate when the + // free-list is momentarily empty (startup / after a backpressure drop). + let mut buf = free_rx + .try_recv() + .unwrap_or_else(|_| Vec::with_capacity(pcm_scratch)); + buf.clear(); + buf.extend_from_slice(&pcm[..n]); + match tx.try_send(buf) { + Ok(()) | Err(TrySendError::Full(_)) => {} // drop-newest under backpressure + Err(TrySendError::Disconnected(_)) => break, + } + if count % 600 == 0 { + log::info!( + "audio: opus={count} pcm_frames={} underruns={} ring={} peak={window_peak:.3}", + counters.pcm_written.load(Ordering::Relaxed), + counters.underruns.load(Ordering::Relaxed), + counters.ring_depth.load(Ordering::Relaxed), + ); + window_peak = 0.0; + } + } + Err(e) => log::debug!("audio: opus decode: {e}"), + } + } Err(PunktfunkError::NoFrame) => {} // timeout Err(_) => break, // session closed } diff --git a/crates/pf-client-core/src/session.rs b/crates/pf-client-core/src/session.rs index 367267b3..08e9aedc 100644 --- a/crates/pf-client-core/src/session.rs +++ b/crates/pf-client-core/src/session.rs @@ -562,18 +562,37 @@ fn spawn_audio( .name("punktfunk-audio-rx".into()) .spawn(move || { let mut pcm = vec![0f32; 5760 * channels as usize]; // scratch: max Opus frame (120 ms) × channels + let mut gaps = punktfunk_core::audio::AudioGapTracker::new(); + let mut frame_samples = 0usize; // per-channel samples of the last decoded frame — the PLC unit while !stop.load(Ordering::SeqCst) { match connector.next_audio(Duration::from_millis(100)) { - Ok(pkt) => match dec.decode_float(&pkt.data, &mut pcm, false) { - // `samples` is per-channel; the interleaved frame is `samples * channels`. - Ok(samples) => { - let n = samples * channels as usize; - let mut buf = player.take_buffer(); - buf.extend_from_slice(&pcm[..n]); - player.push(buf); + Ok(pkt) => { + // Conceal lost packets (a seq gap) with libopus PLC before decoding the one + // that arrived: empty input synthesizes `frame_samples` of interpolation per + // missing packet — an inaudible fade instead of the click a hard gap makes. + for _ in 0..gaps.missing_before(pkt.seq) { + let plc = frame_samples * channels as usize; + if plc == 0 { + break; // no decoded frame yet to size the concealment from + } + if let Ok(samples) = dec.decode_float(&[], &mut pcm[..plc], false) { + let mut buf = player.take_buffer(); + buf.extend_from_slice(&pcm[..samples * channels as usize]); + player.push(buf); + } } - Err(e) => tracing::debug!(error = %e, "opus decode"), - }, + match dec.decode_float(&pkt.data, &mut pcm, false) { + // `samples` is per-channel; the interleaved frame is `samples * channels`. + Ok(samples) => { + frame_samples = samples; + let n = samples * channels as usize; + let mut buf = player.take_buffer(); + buf.extend_from_slice(&pcm[..n]); + player.push(buf); + } + Err(e) => tracing::debug!(error = %e, "opus decode"), + } + } Err(PunktfunkError::NoFrame) => {} Err(_) => break, // plane closed — the session is ending } diff --git a/crates/punktfunk-core/src/audio.rs b/crates/punktfunk-core/src/audio.rs index 057250fe..88270e6b 100644 --- a/crates/punktfunk-core/src/audio.rs +++ b/crates/punktfunk-core/src/audio.rs @@ -125,6 +125,54 @@ pub fn normalize_channels(requested: u8) -> u8 { } } +/// Loss detector for the client audio plane, shared by every platform decoder. +/// +/// The `0xC9` audio datagrams carry a per-packet sequence the host advances by 1 (wrapping), but +/// ride the lossy datagram plane with no FEC — a lost 5 ms Opus packet used to play out as a hard +/// gap (a click/pop; the jitter rings just emit silence). Feeding this tracker each received +/// packet's sequence tells the decoder how many packets went missing *immediately before it*, so +/// it can synthesize that many frames of libopus packet-loss concealment (`decode` with empty +/// input) before decoding the real one — turning clicks into an inaudible interpolation. +/// +/// Reorders and duplicates conceal nothing (the plane has no reorder buffer; playing a late +/// packet where it lands is the existing behaviour), and a gap is capped at +/// [`MAX_CONCEAL_PACKETS`] (50 ms at the protocol's 5 ms frames) — libopus PLC fades to silence +/// after a few frames anyway, so past the cap the ring's underrun/re-prime path takes over as +/// before. +#[derive(Debug, Default)] +pub struct AudioGapTracker { + /// Sequence of the newest packet seen (`None` until the first). + last_seq: Option, +} + +/// Most packets a single gap will ask concealment for (50 ms at the protocol's 5 ms frames). +/// Crate-internal: callers only ever see `missing_before`'s already-capped count (and cbindgen +/// must not export it — it's not part of the C ABI). +const MAX_CONCEAL_PACKETS: u32 = 10; + +impl AudioGapTracker { + pub fn new() -> Self { + Self::default() + } + + /// Feed the next received packet's sequence; returns how many packets are missing immediately + /// before it (`0` for in-order, the first packet, duplicates, and reorders), capped at + /// [`MAX_CONCEAL_PACKETS`]. Wrapping-safe: a sequence in the backward half of the u32 space is + /// a reorder, not a 2³¹-packet gap. + pub fn missing_before(&mut self, seq: u32) -> u32 { + let Some(last) = self.last_seq else { + self.last_seq = Some(seq); + return 0; + }; + let delta = seq.wrapping_sub(last); + if delta == 0 || delta > u32::MAX / 2 { + return 0; // duplicate, or a reorder older than the newest — nothing to conceal + } + self.last_seq = Some(seq); + (delta - 1).min(MAX_CONCEAL_PACKETS) + } +} + // ---- per-platform channel-layout helpers (pure data; no platform deps) -------------------- /// Windows `WAVEFORMATEXTENSIBLE.dwChannelMask` for the wire layout. @@ -215,6 +263,29 @@ mod tests { } } + #[test] + fn gap_tracker_counts_only_forward_gaps() { + let mut t = AudioGapTracker::new(); + assert_eq!(t.missing_before(100), 0, "first packet"); + assert_eq!(t.missing_before(101), 0, "in order"); + assert_eq!(t.missing_before(104), 2, "102+103 lost"); + assert_eq!(t.missing_before(104), 0, "duplicate"); + assert_eq!(t.missing_before(103), 0, "late reorder conceals nothing"); + assert_eq!(t.missing_before(105), 0, "reorder didn't move the anchor"); + // A huge gap is capped; the stream continues from the new anchor. + assert_eq!(t.missing_before(105 + 1000), MAX_CONCEAL_PACKETS); + assert_eq!(t.missing_before(105 + 1001), 0); + } + + #[test] + fn gap_tracker_survives_seq_wraparound() { + let mut t = AudioGapTracker::new(); + assert_eq!(t.missing_before(u32::MAX - 1), 0); + assert_eq!(t.missing_before(u32::MAX), 0, "in order at the edge"); + assert_eq!(t.missing_before(1), 1, "seq 0 lost across the wrap"); + assert_eq!(t.missing_before(0), 0, "pre-wrap reorder, not a 2^31 gap"); + } + #[test] fn wasapi_masks_are_correct() { assert_eq!(wasapi_channel_mask(2), 0x3);