feat(audio): libopus packet-loss concealment on the client audio plane
The 0xC9 audio datagrams ride the lossy plane with no FEC, and no client ever consulted the per-packet sequence: a lost 5 ms Opus packet played out as a hard gap in the ring — an audible click/pop on every drop, i.e. constantly on the Wi-Fi links where video loss is already being FEC-absorbed. Now a shared `AudioGapTracker` (punktfunk-core::audio — pure data, wrap-safe, unit-tested incl. u32 wraparound / reorder / duplicate cases) tells the decoder how many packets went missing immediately before each received one, and both native clients (pf-client-core PipeWire path, Android AAudio path) synthesize that many frames of libopus packet-loss concealment first: `decode` with empty input (the opus crate maps it to a NULL data pointer = PLC), sized by the last real frame's sample count. Interpolated fade instead of a click. Bounds: a gap is capped at 10 packets (50 ms) — libopus PLC fades to silence after a few frames anyway, so past the cap the rings' existing underrun/re-prime path takes over. Reorders and duplicates conceal nothing (the plane has no reorder buffer; playing a late packet where it lands is the existing behaviour). In-band Opus FEC (LBRR) is deliberately NOT used: the host sends 5 ms frames and LBRR needs ≥10 ms frames to carry anything. The cap is a crate-private const so cbindgen keeps it out of the C ABI header. Host cargo tests + clippy green; android crate verified via cargo ndk check. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -355,10 +355,34 @@ fn decode_loop(
|
|||||||
};
|
};
|
||||||
let mut pcm = vec![0f32; pcm_scratch];
|
let mut pcm = vec![0f32; pcm_scratch];
|
||||||
let mut window_peak = 0f32; // loudest |sample| since the last log — tells a tone from silence
|
let mut window_peak = 0f32; // loudest |sample| since the last log — tells a tone from silence
|
||||||
while !shutdown.load(Ordering::Relaxed) {
|
let mut gaps = punktfunk_core::audio::AudioGapTracker::new();
|
||||||
|
let mut frame_samples = 0usize; // per-channel samples of the last decoded frame — the PLC unit
|
||||||
|
'pump: while !shutdown.load(Ordering::Relaxed) {
|
||||||
match client.next_audio(Duration::from_millis(5)) {
|
match client.next_audio(Duration::from_millis(5)) {
|
||||||
Ok(pkt) => match dec.decode_float(&pkt.data, &mut pcm, false) {
|
Ok(pkt) => {
|
||||||
|
// Conceal lost packets (a seq gap) with libopus PLC before decoding the one that
|
||||||
|
// arrived: empty input synthesizes `frame_samples` of interpolation per missing
|
||||||
|
// packet — an inaudible fade instead of the click a hard gap makes in the ring.
|
||||||
|
for _ in 0..gaps.missing_before(pkt.seq) {
|
||||||
|
let plc = frame_samples * channels;
|
||||||
|
if plc == 0 {
|
||||||
|
break; // no decoded frame yet to size the concealment from
|
||||||
|
}
|
||||||
|
if let Ok(samples) = dec.decode_float(&[], &mut pcm[..plc], false) {
|
||||||
|
let mut buf = free_rx
|
||||||
|
.try_recv()
|
||||||
|
.unwrap_or_else(|_| Vec::with_capacity(pcm_scratch));
|
||||||
|
buf.clear();
|
||||||
|
buf.extend_from_slice(&pcm[..samples * channels]);
|
||||||
|
match tx.try_send(buf) {
|
||||||
|
Ok(()) | Err(TrySendError::Full(_)) => {}
|
||||||
|
Err(TrySendError::Disconnected(_)) => break 'pump,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match dec.decode_float(&pkt.data, &mut pcm, false) {
|
||||||
Ok(samples) => {
|
Ok(samples) => {
|
||||||
|
frame_samples = samples;
|
||||||
let n = samples * channels;
|
let n = samples * channels;
|
||||||
for &s in &pcm[..n] {
|
for &s in &pcm[..n] {
|
||||||
window_peak = window_peak.max(s.abs());
|
window_peak = window_peak.max(s.abs());
|
||||||
@@ -393,7 +417,8 @@ fn decode_loop(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => log::debug!("audio: opus decode: {e}"),
|
Err(e) => log::debug!("audio: opus decode: {e}"),
|
||||||
},
|
}
|
||||||
|
}
|
||||||
Err(PunktfunkError::NoFrame) => {} // timeout
|
Err(PunktfunkError::NoFrame) => {} // timeout
|
||||||
Err(_) => break, // session closed
|
Err(_) => break, // session closed
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -562,18 +562,37 @@ fn spawn_audio(
|
|||||||
.name("punktfunk-audio-rx".into())
|
.name("punktfunk-audio-rx".into())
|
||||||
.spawn(move || {
|
.spawn(move || {
|
||||||
let mut pcm = vec![0f32; 5760 * channels as usize]; // scratch: max Opus frame (120 ms) × channels
|
let mut pcm = vec![0f32; 5760 * channels as usize]; // scratch: max Opus frame (120 ms) × channels
|
||||||
|
let mut gaps = punktfunk_core::audio::AudioGapTracker::new();
|
||||||
|
let mut frame_samples = 0usize; // per-channel samples of the last decoded frame — the PLC unit
|
||||||
while !stop.load(Ordering::SeqCst) {
|
while !stop.load(Ordering::SeqCst) {
|
||||||
match connector.next_audio(Duration::from_millis(100)) {
|
match connector.next_audio(Duration::from_millis(100)) {
|
||||||
Ok(pkt) => match dec.decode_float(&pkt.data, &mut pcm, false) {
|
Ok(pkt) => {
|
||||||
|
// Conceal lost packets (a seq gap) with libopus PLC before decoding the one
|
||||||
|
// that arrived: empty input synthesizes `frame_samples` of interpolation per
|
||||||
|
// missing packet — an inaudible fade instead of the click a hard gap makes.
|
||||||
|
for _ in 0..gaps.missing_before(pkt.seq) {
|
||||||
|
let plc = frame_samples * channels as usize;
|
||||||
|
if plc == 0 {
|
||||||
|
break; // no decoded frame yet to size the concealment from
|
||||||
|
}
|
||||||
|
if let Ok(samples) = dec.decode_float(&[], &mut pcm[..plc], false) {
|
||||||
|
let mut buf = player.take_buffer();
|
||||||
|
buf.extend_from_slice(&pcm[..samples * channels as usize]);
|
||||||
|
player.push(buf);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match dec.decode_float(&pkt.data, &mut pcm, false) {
|
||||||
// `samples` is per-channel; the interleaved frame is `samples * channels`.
|
// `samples` is per-channel; the interleaved frame is `samples * channels`.
|
||||||
Ok(samples) => {
|
Ok(samples) => {
|
||||||
|
frame_samples = samples;
|
||||||
let n = samples * channels as usize;
|
let n = samples * channels as usize;
|
||||||
let mut buf = player.take_buffer();
|
let mut buf = player.take_buffer();
|
||||||
buf.extend_from_slice(&pcm[..n]);
|
buf.extend_from_slice(&pcm[..n]);
|
||||||
player.push(buf);
|
player.push(buf);
|
||||||
}
|
}
|
||||||
Err(e) => tracing::debug!(error = %e, "opus decode"),
|
Err(e) => tracing::debug!(error = %e, "opus decode"),
|
||||||
},
|
}
|
||||||
|
}
|
||||||
Err(PunktfunkError::NoFrame) => {}
|
Err(PunktfunkError::NoFrame) => {}
|
||||||
Err(_) => break, // plane closed — the session is ending
|
Err(_) => break, // plane closed — the session is ending
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -125,6 +125,54 @@ pub fn normalize_channels(requested: u8) -> u8 {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Loss detector for the client audio plane, shared by every platform decoder.
|
||||||
|
///
|
||||||
|
/// The `0xC9` audio datagrams carry a per-packet sequence the host advances by 1 (wrapping), but
|
||||||
|
/// ride the lossy datagram plane with no FEC — a lost 5 ms Opus packet used to play out as a hard
|
||||||
|
/// gap (a click/pop; the jitter rings just emit silence). Feeding this tracker each received
|
||||||
|
/// packet's sequence tells the decoder how many packets went missing *immediately before it*, so
|
||||||
|
/// it can synthesize that many frames of libopus packet-loss concealment (`decode` with empty
|
||||||
|
/// input) before decoding the real one — turning clicks into an inaudible interpolation.
|
||||||
|
///
|
||||||
|
/// Reorders and duplicates conceal nothing (the plane has no reorder buffer; playing a late
|
||||||
|
/// packet where it lands is the existing behaviour), and a gap is capped at
|
||||||
|
/// [`MAX_CONCEAL_PACKETS`] (50 ms at the protocol's 5 ms frames) — libopus PLC fades to silence
|
||||||
|
/// after a few frames anyway, so past the cap the ring's underrun/re-prime path takes over as
|
||||||
|
/// before.
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub struct AudioGapTracker {
|
||||||
|
/// Sequence of the newest packet seen (`None` until the first).
|
||||||
|
last_seq: Option<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Most packets a single gap will ask concealment for (50 ms at the protocol's 5 ms frames).
|
||||||
|
/// Crate-internal: callers only ever see `missing_before`'s already-capped count (and cbindgen
|
||||||
|
/// must not export it — it's not part of the C ABI).
|
||||||
|
const MAX_CONCEAL_PACKETS: u32 = 10;
|
||||||
|
|
||||||
|
impl AudioGapTracker {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Feed the next received packet's sequence; returns how many packets are missing immediately
|
||||||
|
/// before it (`0` for in-order, the first packet, duplicates, and reorders), capped at
|
||||||
|
/// [`MAX_CONCEAL_PACKETS`]. Wrapping-safe: a sequence in the backward half of the u32 space is
|
||||||
|
/// a reorder, not a 2³¹-packet gap.
|
||||||
|
pub fn missing_before(&mut self, seq: u32) -> u32 {
|
||||||
|
let Some(last) = self.last_seq else {
|
||||||
|
self.last_seq = Some(seq);
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
let delta = seq.wrapping_sub(last);
|
||||||
|
if delta == 0 || delta > u32::MAX / 2 {
|
||||||
|
return 0; // duplicate, or a reorder older than the newest — nothing to conceal
|
||||||
|
}
|
||||||
|
self.last_seq = Some(seq);
|
||||||
|
(delta - 1).min(MAX_CONCEAL_PACKETS)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---- per-platform channel-layout helpers (pure data; no platform deps) --------------------
|
// ---- per-platform channel-layout helpers (pure data; no platform deps) --------------------
|
||||||
|
|
||||||
/// Windows `WAVEFORMATEXTENSIBLE.dwChannelMask` for the wire layout.
|
/// Windows `WAVEFORMATEXTENSIBLE.dwChannelMask` for the wire layout.
|
||||||
@@ -215,6 +263,29 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gap_tracker_counts_only_forward_gaps() {
|
||||||
|
let mut t = AudioGapTracker::new();
|
||||||
|
assert_eq!(t.missing_before(100), 0, "first packet");
|
||||||
|
assert_eq!(t.missing_before(101), 0, "in order");
|
||||||
|
assert_eq!(t.missing_before(104), 2, "102+103 lost");
|
||||||
|
assert_eq!(t.missing_before(104), 0, "duplicate");
|
||||||
|
assert_eq!(t.missing_before(103), 0, "late reorder conceals nothing");
|
||||||
|
assert_eq!(t.missing_before(105), 0, "reorder didn't move the anchor");
|
||||||
|
// A huge gap is capped; the stream continues from the new anchor.
|
||||||
|
assert_eq!(t.missing_before(105 + 1000), MAX_CONCEAL_PACKETS);
|
||||||
|
assert_eq!(t.missing_before(105 + 1001), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gap_tracker_survives_seq_wraparound() {
|
||||||
|
let mut t = AudioGapTracker::new();
|
||||||
|
assert_eq!(t.missing_before(u32::MAX - 1), 0);
|
||||||
|
assert_eq!(t.missing_before(u32::MAX), 0, "in order at the edge");
|
||||||
|
assert_eq!(t.missing_before(1), 1, "seq 0 lost across the wrap");
|
||||||
|
assert_eq!(t.missing_before(0), 0, "pre-wrap reorder, not a 2^31 gap");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn wasapi_masks_are_correct() {
|
fn wasapi_masks_are_correct() {
|
||||||
assert_eq!(wasapi_channel_mask(2), 0x3);
|
assert_eq!(wasapi_channel_mask(2), 0x3);
|
||||||
|
|||||||
Reference in New Issue
Block a user