diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt index 926606ce..bb28db5d 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt @@ -18,12 +18,13 @@ import kotlin.math.roundToInt * The live stats overlay — the unified HUD (`design/stats-unification.md`): headline is * `capture→displayed` tiled by `host+network` + `decode` + `display` when the platform delivered * OnFrameRendered render callbacks this window (`dispValid`), falling back to the v1 - * `capture→decoded` headline without the `display` term when it didn't. Reads the 33-double + * `capture→decoded` headline without the `display` term when it didn't. Reads the 35-double * layout from [NativeBridge.nativeVideoStats] (that KDoc is the authoritative index list): * `[fps, mbps, e2eP50Ms, e2eP95Ms, latValid, skew, w, h, hz, lostTotal, bitDepth, colorPrimaries, * colorTransfer, chromaFormatIdc, hostNetP50Ms, decodeP50Ms, hostP50Ms, netP50Ms, lost, skipped, * fec, frames, dispValid, displayP50Ms, e2eDispP50Ms, e2eDispP95Ms, paceP50Ms, latchP50Ms, - * presentsWindow, presenterActive, feedP50Ms, codecP50Ms, skippedOverflowWindow]`. Every read + * presentsWindow, presenterActive, feedP50Ms, codecP50Ms, skippedOverflowWindow, audioBufferMs, + * audioAvOffsetMs]`. Every read * is length-guarded, so an older native lib simply omits the lines it can't feed. * * The shown `display` and `end-to-end` numbers EXCLUDE the OS present floor (see [osFloorMs]) at @@ -44,7 +45,7 @@ import kotlin.math.roundToInt * reliability counters (18–21) when nonzero. * - [StatsVerbosity.DETAILED] — also the decoder label, the video-feed descriptor (10–13), the * stage equation (14/15, split into `host + network` when the Phase-2 terms at 16/17 are nonzero), - * and the excluded-floor line when one was measured. + * the excluded-floor line when one was measured, and the audio plane's own latency (33/34). * [StatsVerbosity.OFF] renders nothing. Older native layouts simply omit the lines they lack (the * counter line falls back to the cumulative `lostTotal` at index 9 on a pre-window lib). */ @@ -178,10 +179,42 @@ internal fun StatsOverlay( } } } + if (detailed) { + audioLine(s)?.let { statLine(it, Color.White) } + } counterLine(s, lost)?.let { statLine(it, Color(0xFFFFB0B0)) } } } +/** + * The audio plane's own latency from the live gauges at 33/34 — `audio buffer 42 ms · a/v +18 ms`, + * the same wording the desktop HUD uses. `buffer` is how much decoded audio is queued ahead of the + * speaker; `a/v` is where that PUTS it relative to the picture (positive = audio behind). `null` + * before any audio has been queued (buffer 0 — audio off, or the ring not yet primed) and on an + * older native layout. + * + * Both terms, not just the depth: a deep ring on a jittery link is correct behaviour — the + * underrun-driven floor earned that buffer — and only the offset distinguishes it from a ring that + * is simply holding audio late. The offset term is dropped at zero, which is both "aligned" and + * "no measurement yet"; the depth alone is still the triage number, and it is the one that did not + * exist at all before (the plane published nothing any surface could render, so a "the audio delay + * is way too high" report had no instrument behind it). + * + * NOT shaved by [osFloorMs], unlike every video figure above. That shave is a reporting policy — + * metrics report what Punktfunk controls — but sound has to reach the ear when the light reaches + * the eye, so the sync loop aligns against the RAW capture→displayed time (see the native + * `DisplayTracker`) and this offset is stated in those same terms. Subtracting the floor here would + * report an alignment the listener is not getting. + */ +private fun audioLine(s: DoubleArray): String? { + if (s.size < 35) return null + val bufferMs = s[33].roundToInt() + if (bufferMs <= 0) return null + val avOffset = s[34].roundToInt() + val avTerm = if (avOffset != 0) " · a/v ${if (avOffset > 0) "+" else ""}$avOffset ms" else "" + return "audio buffer $bufferMs ms$avTerm" +} + /** One monospace HUD line — the shared type ramp so every tier's rows line up. */ @Composable private fun statLine(text: String, color: Color) { diff --git a/clients/android/app/src/test/kotlin/io/unom/punktfunk/StatsOverlayAudioTest.kt b/clients/android/app/src/test/kotlin/io/unom/punktfunk/StatsOverlayAudioTest.kt new file mode 100644 index 00000000..171c13fc --- /dev/null +++ b/clients/android/app/src/test/kotlin/io/unom/punktfunk/StatsOverlayAudioTest.kt @@ -0,0 +1,94 @@ +package io.unom.punktfunk + +import androidx.activity.ComponentActivity +import androidx.compose.ui.test.junit4.createAndroidComposeRule +import androidx.compose.ui.test.onNodeWithText +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +/** + * The stats HUD's audio line — `audio buffer N ms · a/v ±N ms`, from the live gauges at indexes + * 33/34 (`design/audio-latency-overhaul.md`). + * + * Worth pinning because the whole point of the overhaul's stats half is that the audio plane became + * OBSERVABLE. Before it, ring depth and A/V offset existed only as a log line, and on a device + * launched by a game launcher that goes to a pipe nobody can read — so the single number that + * identifies a deep ring was unobtainable on the exact device reporting the latency, and a field + * investigation ran to its conclusion without it. A measurement that never reaches a surface is + * indistinguishable from no measurement, which is what this asserts. + * + * `sdk = [36]` for the same reason as the screenshot tests: Robolectric ships android-all jars only + * up to API 36 while the app's compileSdk is 37. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [36]) +class StatsOverlayAudioTest { + @get:Rule + val compose = createAndroidComposeRule() + + /** + * A plausible 35-double window with the audio gauges dialled in. Everything before 33 is the + * DETAILED-renderable shape the ShotScenes fixture uses; only the last two matter here. + */ + private fun stats(bufferMs: Double, avOffsetMs: Double, size: Int = 35): DoubleArray { + val full = doubleArrayOf( + 238.0, 921.4, 1.3, 2.1, 1.0, 1.0, 5120.0, 1440.0, 240.0, 2.0, + 10.0, 9.0, 16.0, 1.0, 0.9, 0.4, 0.6, 0.3, + 2.0, 1.0, 5.0, 238.0, + 1.0, 0.5, 1.8, 2.6, + 0.2, 0.3, 236.0, 1.0, + 0.1, 0.3, 0.0, + bufferMs, avOffsetMs, + ) + return full.copyOf(size) + } + + private fun show(s: DoubleArray, verbosity: StatsVerbosity = StatsVerbosity.DETAILED) { + compose.setContent { StatsOverlay(s, verbosity = verbosity) } + } + + @Test + fun detailedShowsDepthAndOffset() { + show(stats(bufferMs = 42.0, avOffsetMs = 18.0)) + // Positive = audio playing BEHIND the picture, and the sign is explicit so a glance tells + // which way the loop still has to move. + compose.onNodeWithText("audio buffer 42 ms · a/v +18 ms").assertExists() + } + + @Test + fun audioAheadOfThePictureReadsNegative() { + show(stats(bufferMs = 42.0, avOffsetMs = -12.0)) + compose.onNodeWithText("audio buffer 42 ms · a/v -12 ms").assertExists() + } + + /** Aligned (or not yet measured) drops the offset term; the depth alone is still the triage number. */ + @Test + fun alignedShowsDepthAlone() { + show(stats(bufferMs = 42.0, avOffsetMs = 0.0)) + compose.onNodeWithText("audio buffer 42 ms").assertExists() + } + + /** Nothing queued (audio off, or the ring not yet primed) — the line has nothing to say. */ + @Test + fun silentPlaneRendersNoLine() { + show(stats(bufferMs = 0.0, avOffsetMs = 0.0)) + compose.onNodeWithText("audio buffer", substring = true).assertDoesNotExist() + } + + /** The line is DETAILED-only, like every other per-stage figure. */ + @Test + fun normalTierOmitsTheLine() { + show(stats(bufferMs = 42.0, avOffsetMs = 18.0), verbosity = StatsVerbosity.NORMAL) + compose.onNodeWithText("audio buffer", substring = true).assertDoesNotExist() + } + + /** An older native lib emits 33 doubles; the overlay must omit the line, not index past the end. */ + @Test + fun olderNativeLayoutOmitsTheLine() { + show(stats(bufferMs = 42.0, avOffsetMs = 18.0, size = 33)) + compose.onNodeWithText("audio buffer", substring = true).assertDoesNotExist() + } +} diff --git a/clients/android/app/src/test/kotlin/io/unom/punktfunk/screenshots/ShotScenes.kt b/clients/android/app/src/test/kotlin/io/unom/punktfunk/screenshots/ShotScenes.kt index de69a24e..31123c98 100644 --- a/clients/android/app/src/test/kotlin/io/unom/punktfunk/screenshots/ShotScenes.kt +++ b/clients/android/app/src/test/kotlin/io/unom/punktfunk/screenshots/ShotScenes.kt @@ -355,10 +355,12 @@ internal fun StreamScene(verbosity: StatsVerbosity = StatsVerbosity.DETAILED) { Brush.linearGradient(listOf(Color(0xFF2A1E5C), Color(0xFF0E1B3D), Color(0xFF06122B))), ), ) { - // The full 26-double unified layout (design/stats-unification.md): [fps, mbps, e2eP50, - // e2eP95, latValid, skew, w, h, hz, lostTotal, bitDepth, colorPrimaries, colorTransfer, - // chromaFormatIdc, hostNetP50, decodeP50, hostP50, netP50, lost, skipped, fec, frames, - // dispValid, displayP50, e2eDispP50, e2eDispP95]. + // The full 35-double unified layout — NativeBridge.nativeVideoStats' KDoc is the + // authoritative index list: [fps, mbps, e2eP50, e2eP95, latValid, skew, w, h, hz, + // lostTotal, bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc, hostNetP50, + // decodeP50, hostP50, netP50, lost, skipped, fec, frames, dispValid, displayP50, + // e2eDispP50, e2eDispP95, paceP50, latchP50, presents, presenterActive, feedP50, codecP50, + // skippedOverflow, audioBufferMs, audioAvOffsetMs]. // 10/9/16/1 = a 10-bit BT.2020 PQ (HDR) 4:2:0 feed so the DETAILED HUD renders its // video-feed line; the display stage is valid (dispValid 1) so the headline is the // directly-measured capture→displayed pair, less the excluded OS present floor (the 0.3 @@ -376,6 +378,12 @@ internal fun StreamScene(verbosity: StatsVerbosity = StatsVerbosity.DETAILED) { 1.0, 0.5, 1.8, 2.6, // Timeline-presenter split: pace + latch tile the display term; presents ≈ fps. 0.2, 0.3, 236.0, 1.0, + // The decode term's own split (feed + codec = 0.4), and no overflow — the one + // `skipped` above is benign newest-wins pacing, not a decoder falling behind. + 0.1, 0.3, 0.0, + // The audio plane: a 28 ms ring placed 4 ms behind the picture — a converged sync + // loop, i.e. inside the deadband it deliberately leaves alone. + 28.0, 4.0, ), verbosity = verbosity, decoderLabel = "c2.qti.hevc.decoder · low-latency", diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index 77513e1f..cd439ef5 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -264,12 +264,12 @@ object NativeBridge { /** * Drain ~1 s of live decode stats for the on-stream HUD, or `null` when no decode thread runs. - * Returns 33 doubles (unified stats spec, `design/stats-unification.md`): + * Returns 35 doubles (unified stats spec, `design/stats-unification.md`): * `[fps, mbps, e2eP50Ms, e2eP95Ms, latValid, skewCorrected, width, height, refreshHz, framesLost, * bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc, hostNetP50Ms, decodeP50Ms, hostP50Ms, * netP50Ms, lostWindow, skippedWindow, fecWindow, framesWindow, dispValid, displayP50Ms, * e2eDispP50Ms, e2eDispP95Ms, paceP50Ms, latchP50Ms, presentsWindow, presenterActive, - * feedP50Ms, codecP50Ms, skippedOverflowWindow]` + * feedP50Ms, codecP50Ms, skippedOverflowWindow, audioBufferMs, audioAvOffsetMs]` * (the flags are 1.0/0.0; indexes 2/3 are the end-to-end capture→decoded headline; 10–13 * describe the negotiated video feed — bit depth 8/10, CICP primaries/transfer, and the HEVC * chroma_format_idc 1=4:2:0 / 3=4:4:4; 14/15 are the stage p50s tiling the headline — @@ -285,7 +285,10 @@ object NativeBridge { * the window's on-glass confirm count, and whether the presenter is active at all; 30/31 * split `decode` (15) the same way — `feed` = received→queued (hand-off + input-slot wait), * `codec` = queued→decoded, the decoder's own time; 32 is the parked-AU overflow subset of - * `skipped` (19), i.e. the decoder falling behind rather than benign newest-wins pacing). + * `skipped` (19), i.e. the decoder falling behind rather than benign newest-wins pacing; + * 33/34 are the AUDIO plane — the playback ring's live depth in ms and the A/V sync loop's + * smoothed offset in ms, positive meaning audio plays BEHIND the picture. Those two are live + * gauges, not windowed samples, and the offset reads 0 until the loop has a video reference). * Poll ~1 Hz; each call resets the measurement window. */ external fun nativeVideoStats(handle: Long): DoubleArray? diff --git a/clients/android/native/src/audio.rs b/clients/android/native/src/audio.rs index 6dbec5cc..7cf53722 100644 --- a/clients/android/native/src/audio.rs +++ b/clients/android/native/src/audio.rs @@ -20,6 +20,16 @@ //! (2) is now the SHARED `punktfunk_core::audio::JitterPolicy` at `JitterTuning::AAUDIO`, which also //! fixed what this ring was missing: it had a hard cap but nothing that walked the depth back down, //! so drift and arrival bursts raised latency permanently and Android settled on its ceiling. +//! +//! It is also **A/V synchronised** (`design/audio-latency-overhaul.md`): the decode thread reads the +//! host capture `pts_ns` every `AudioPacket` has always carried, compares where this frame will +//! actually play against where the picture it belongs with reached glass +//! (`decode::DisplayTracker` publishes that), and asks the ring for a depth that closes the gap. +//! Only ASKS — `JitterPolicy` clamps the request between its own underrun-driven floor and the hard +//! cap, so continuity outranks sync and a link whose jitter genuinely needs more buffer than the +//! picture is away keeps its buffer, with the residual reported on the HUD instead of taken out of +//! the listener's stream. With no video reference (below API 33 there are no render callbacks, so +//! nothing confirms a present) the target stays `None` and the ring behaves exactly as it did. use ndk::audio::{ AudioCallbackResult, AudioContentType, AudioDirection, AudioFormat, AudioPerformanceMode, @@ -94,15 +104,45 @@ impl AudioDec { /// Diagnostics — written by the decode thread + the realtime callback, logged periodically. The /// audio analogue of the video `fed`/`rendered` counters (we can't "screenshot" sound). +/// +/// The ring's DEPTH is not here: the A/V sync loop needs the same number in the same units, so it +/// is published once through [`punktfunk_core::audio::AudioSyncCell`] and read from there by the +/// log line below. One publisher, one reading — a second copy is a second thing to go stale. #[derive(Default)] struct Counters { opus_decoded: AtomicU64, // Opus packets decoded OK (~200/s at 5 ms frames) pcm_written: AtomicU64, // PCM frames copied out to AAudio (device clock is pulling) underruns: AtomicU64, // callbacks that emitted silence (ring not primed / drained) - ring_depth: AtomicU64, // ring sample count at the last callback target_ms: AtomicU64, // the policy's LIVE target depth (it grows on this device's underruns) } +/// Whether the A/V sync loop runs this session. `false` leaves `JitterPolicy`'s sync target at +/// `None`, which reproduces the pre-overhaul ring behaviour exactly — the point of the hatch. +/// +/// Two levers because Android has neither of the other clients' launch surfaces. `PUNKTFUNK_NO_AV_SYNC` +/// keeps the contract the desktop clients document (and works when the client is driven from a +/// shell), but an app started from the launcher inherits no such environment, so the one a field +/// tester can actually reach is the sysprop — `adb shell setprop debug.punktfunk.no_av_sync 1`, +/// no rebuild, exactly like `debug.punktfunk.presenter`. A loop that steers PLAYBACK has to be +/// bisectable on the device that reports the regression, not only on the bench. +fn av_sync_enabled() -> bool { + if matches!( + std::env::var("PUNKTFUNK_NO_AV_SYNC").as_deref(), + Ok("1") | Ok("true") + ) { + return false; + } + let mut buf = [0u8; 92]; // PROP_VALUE_MAX + // SAFETY: __system_property_get with a valid name + PROP_VALUE_MAX buffer is always safe. + let n = unsafe { + libc::__system_property_get( + c"debug.punktfunk.no_av_sync".as_ptr(), + buf.as_mut_ptr().cast(), + ) + }; + !(n > 0 && matches!(&buf[..n as usize], b"1" | b"true")) +} + /// Owned by [`crate::session::SessionHandle`]: the live AAudio stream + the decode thread. pub struct AudioPlayback { _stream: AudioStream, // dropping it stops + closes the AAudio stream @@ -127,6 +167,10 @@ impl AudioPlayback { // Worst transient the ring can hold before the policy trims it. let hard_cap_max = tuning.hard_cap_ms as usize * ms; let counters = Arc::new(Counters::default()); + // The A/V sync hand-off: the realtime callback owns the ring (so it publishes the depth and + // consumes the target), the decode thread owns the timestamps (so it computes the target). + // Two atomics, because the callback must not block on the thread that decodes Opus. + let sync: Arc = Arc::default(); // One open attempt at a given sharing mode. Everything the realtime callback captures // (channels, ring, prime state) is rebuilt per attempt — `open_stream` consumes the builder @@ -146,6 +190,7 @@ impl AudioPlayback { // Realtime consumer state, owned by the callback (FnMut) — no lock: AAudio calls it from // a single high-priority thread, and the decode thread only touches `tx`/`free_rx`. let cb_counters = counters.clone(); + let cb_sync = sync.clone(); // Pre-reserve the ring so `extend` never reallocates on the realtime thread. Worst // transient before the trim below = the hard cap plus one full channel of 5 ms (480-f32) // frames — the punktfunk protocol always sends 5 ms Opus frames (host `audio_thread`); a @@ -171,6 +216,13 @@ impl AudioPlayback { ring.extend(chunk.drain(..)); let _ = free_tx.try_send(chunk); } + // A/V sync: take whatever depth the decode thread's sync loop last asked for, and + // publish where the ring actually is so it can measure the result. The policy + // clamps the request between its own underrun floor and the hard cap — continuity + // outranks sync, always (see `JitterPolicy::set_sync_target`). Read AFTER the + // drain, so the depth is everything a frame queued right now must wait behind. + policy.set_sync_target(cb_sync.target()); + cb_sync.publish_depth(ring.len()); // Jitter buffer: the shared policy decides prime/silence, trims a burst, and — // new here — sheds ONE crossfaded 5 ms frame when the depth average has sat above // target long enough to be drift rather than jitter. Without that shed this ring @@ -201,9 +253,6 @@ impl AudioPlayback { // No-op while un-primed, so a deliberate priming silence is never counted as an // underrun (which would otherwise drive the adaptive floor up for no reason). policy.note_read(ran_short); - cb_counters - .ring_depth - .store(ring.len() as u64, Ordering::Relaxed); cb_counters .target_ms .store(policy.target_ms() as u64, Ordering::Relaxed); @@ -303,7 +352,7 @@ impl AudioPlayback { let sd = shutdown.clone(); let join = std::thread::Builder::new() .name("pf-audio".into()) - .spawn(move || decode_loop(client, tx, free_rx, sd, counters, channels)) + .spawn(move || decode_loop(client, tx, free_rx, sd, counters, channels, sync)) .ok(); Some(AudioPlayback { @@ -334,6 +383,7 @@ fn decode_loop( shutdown: Arc, counters: Arc, channels: usize, + sync: Arc, ) { // Fold this Opus→AAudio thread into the client's hot-thread set so the ADPF session the decode // thread opens also keeps audio decode on a fast core (registered before the video pump's first @@ -354,9 +404,44 @@ fn decode_loop( let mut window_peak = 0f32; // loudest |sample| since the last log — tells a tone from silence let mut gaps = punktfunk_core::audio::AudioGapTracker::new(); let mut frame_samples = 0usize; // per-channel samples of the last decoded frame — the PLC unit + + // A/V sync (audio latency overhaul). This thread is the only place holding all three + // ingredients at once: the packet's host capture `pts_ns`, the ring depth (via the sync cell) + // and the video plane's end-to-end figure. `pts_ns` arrived in every `AudioPacket` and was + // dropped on the floor here for the plane's whole existence, which is why audio ran at whatever + // depth its jitter ring settled at with nothing ever placing it against the picture. + let av_sync_enabled = av_sync_enabled(); + let mut av = punktfunk_core::audio::AvSync::new(channels as u8); + let video_e2e = client.video_e2e_shared(); + let av_offset_out = client.audio_av_offset_shared(); + let buffer_ms_out = client.audio_buffer_ms_shared(); + if !av_sync_enabled { + log::info!("audio: A/V sync disabled (PUNKTFUNK_NO_AV_SYNC / debug.punktfunk.no_av_sync)"); + } 'pump: while !shutdown.load(Ordering::Relaxed) { match client.next_audio(Duration::from_millis(5)) { Ok(pkt) => { + // Place this frame against the picture it belongs with, BEFORE it is queued: + // `buffered_ahead` is everything that must still play first, so the depth read here + // is exactly what delays it. + let depth = sync.depth(); + // Published unconditionally — the ring's depth is worth seeing even with sync off, + // and it is what makes a "the audio delay is way too high" report triageable at all. + buffer_ms_out.store((depth / ms.max(1)) as u32, Ordering::Relaxed); + if av_sync_enabled { + let ve2e = video_e2e.load(Ordering::Relaxed); + av.observe(punktfunk_core::audio::AvSyncObservation { + pts_ns: pkt.pts_ns, + now_local_ns: punktfunk_core::client::now_realtime_ns(), + clock_offset_ns: client.clock_offset_now_ns(), + buffered_ahead: depth, + // 0 = nothing confirmed on the glass yet (no render callback below API 33, + // or the stream has not presented a frame); no reference, no correction. + video_e2e_ns: (ve2e > 0).then_some(ve2e), + }); + sync.set_target(av.desired_depth(depth)); + av_offset_out.store(av.offset_ms() as i64, Ordering::Relaxed); + } // Conceal lost packets (a seq gap) with libopus PLC before decoding the one that // arrived: empty input synthesizes `frame_samples` of interpolation per missing // packet — an inaudible fade instead of the click a hard gap makes in the ring. @@ -404,12 +489,17 @@ fn decode_loop( Err(TrySendError::Disconnected(_)) => break, } if count % 600 == 0 { + // `av_ms` is the sync loop's smoothed placement error (+ = audio behind + // the picture); 0 with sync off, or before it has a video reference. + // Logged next to the depth because a deep ring on a jittery link is + // correct and only the offset separates that from audio held late. log::info!( - "audio: opus={count} pcm_frames={} underruns={} buffer_ms={} target_ms={} peak={window_peak:.3}", + "audio: opus={count} pcm_frames={} underruns={} buffer_ms={} target_ms={} av_ms={} peak={window_peak:.3}", counters.pcm_written.load(Ordering::Relaxed), counters.underruns.load(Ordering::Relaxed), - counters.ring_depth.load(Ordering::Relaxed) / ms.max(1) as u64, + (depth / ms.max(1)) as u64, counters.target_ms.load(Ordering::Relaxed), + av.offset_ms(), ); window_peak = 0.0; } diff --git a/clients/android/native/src/decode/async_loop.rs b/clients/android/native/src/decode/async_loop.rs index dc9cb30d..69ace83c 100644 --- a/clients/android/native/src/decode/async_loop.rs +++ b/clients/android/native/src/decode/async_loop.rs @@ -204,7 +204,15 @@ pub(super) fn run_async( // SurfaceFlinger's render timestamp. `render_cb` is the callback's leaked Arc refcount, // reclaimed after the codec is dropped below. let meter = Arc::new(PresentMeter::new()); - let tracker = DisplayTracker::new(stats.clone(), clock_offset.clone(), meter.clone()); + // The tracker also publishes each confirmed present's end-to-end into the shared cell the audio + // plane steers its jitter ring by (`design/audio-latency-overhaul.md`) — video is the master, + // and this is the only point that knows when a frame actually reached glass. + let tracker = DisplayTracker::new( + stats.clone(), + clock_offset.clone(), + client.video_e2e_shared(), + meter.clone(), + ); let render_cb = install_render_callback(&codec, &tracker); // The timeline presenter (see `presenter.rs`): newest-wins / smoothing store, one-in-flight diff --git a/clients/android/native/src/decode/display.rs b/clients/android/native/src/decode/display.rs index b5df8b27..47d5e985 100644 --- a/clients/android/native/src/decode/display.rs +++ b/clients/android/native/src/decode/display.rs @@ -5,7 +5,7 @@ use ndk::media::media_codec::MediaCodec; use ndk::native_window::NativeWindow; use std::collections::VecDeque; use std::ffi::c_void; -use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use super::latency::now_realtime_ns; @@ -35,6 +35,16 @@ pub(super) struct DisplayTracker { /// loaded per callback so mid-stream re-syncs apply. Holding the handle (not the client) /// keeps the leaked render-callback refcount from pinning the whole session alive. clock_offset: Arc, + /// Where the AUDIO plane reads the video leg it has to land with (ns) — `displayed + + /// clock_offset − pts`, published on every confirmed present. Written here, read by + /// [`crate::audio`]'s sync loop; the two planes never touch each other directly (the presenter + /// must not know about audio, and the audio thread cannot see the glass). + /// + /// Published RAW. The HUD shaves the OS present floor off its shown display / end-to-end + /// numbers (`StatsOverlay.osFloorMs` — metrics report what Punktfunk controls), but sound has + /// to reach the ear when the light reaches the eye, and a floor-shaved reference would place + /// audio a whole latch period early on every device. Presentation policy, not physics. + video_e2e: Arc, /// Always-on latch/display accumulator for the presenter's 1 Hz `pf-present` line — /// independent of the HUD gate, so a HUD-off A/B stays measurable from logcat. meter: Arc, @@ -48,11 +58,13 @@ impl DisplayTracker { pub(super) fn new( stats: Arc, clock_offset: Arc, + video_e2e: Arc, meter: Arc, ) -> Arc { Arc::new(DisplayTracker { stats, clock_offset, + video_e2e, meter, rendered: Mutex::new(VecDeque::new()), }) @@ -105,7 +117,14 @@ pub(super) fn install_render_callback( } let sym = libc::dlsym(lib, c"AMediaCodec_setOnFrameRenderedCallback".as_ptr()); if sym.is_null() { - log::info!("decode: no render callback on this API level (<33) — no display stage"); + // No confirmed present ⇒ no `display` stage AND no reference for the audio plane's A/V + // sync, which then stays inert and leaves the ring exactly as it was. The release + // instant is NOT substituted: releases target a future vsync, so it runs a whole latch + // period (8-21 ms measured) ahead of glass — well outside the loop's deadband, i.e. it + // would place audio early on every frame while looking like it was working. + log::info!( + "decode: no render callback on this API level (<33) — no display stage, no A/V sync" + ); return None; } std::mem::transmute::<*mut c_void, SetOnFrameRenderedFn>(sym) @@ -145,8 +164,10 @@ pub(super) unsafe fn release_render_callback(ud: *const DisplayTracker) { /// between the frame rendering and the (batchable) callback delivery — to subtract against the /// receipt/decode stamps and the host capture pts. Records the HUD's `displayed` point: /// `end-to-end` = capture→displayed (skew-corrected) and `display` = decoded→displayed -/// (single-clock local). Panic-free by construction (poison-proof lock, saturating math) — an -/// unwind out of an `extern "C"` fn would abort the process. +/// (single-clock local) — and publishes that end-to-end figure for the audio plane to align +/// against, which is the only place in the client that knows when a frame truly reached glass. +/// Panic-free by construction (poison-proof lock, saturating math) — an unwind out of an +/// `extern "C"` fn would abort the process. unsafe extern "C" fn on_frame_rendered( _codec: *mut ndk_sys::AMediaCodec, userdata: *mut c_void, @@ -186,13 +207,28 @@ unsafe extern "C" fn on_frame_rendered( let latch_us = paired.and_then(|(_, r)| clamp(displayed_ns - r)); // Always-on half: the presenter's pf-present line reads these with the HUD off. t.meter.note_latch(latch_us); - if !t.stats.enabled() { - return; // HUD hidden — skip the skew math + the stats lock - } + // The glass-to-glass figure, computed ABOVE the HUD gate: the audio plane steers its ring by it + // (see `video_e2e`), and a sync loop that only worked while the overlay was up would be off on + // the exact devices that report latency — on a Deck-class report the overlay is precisely what + // the field cannot reach. The cost is one relaxed load and some integer arithmetic per confirmed + // present (≤ the panel rate); the stats LOCK stays behind the gate, which is what that + // early-return was really protecting. let e2e_ns = displayed_ns + t.clock_offset.load(Ordering::Relaxed) as i128 - pts_us as i128 * 1000; - let e2e_us = (e2e_ns > 0 && e2e_ns < 10_000_000_000).then_some((e2e_ns / 1000) as u64); - t.stats.note_displayed(e2e_us, display_us, latch_us); + // Same (0, 10 s) clamp as every other e2e sample — a vendor's first render callbacks can carry + // a garbage `system_nano`, and here that would step the audio ring rather than just a p95. + let e2e_valid = e2e_ns > 0 && e2e_ns < 10_000_000_000; + if e2e_valid { + t.video_e2e.store(e2e_ns as u64, Ordering::Relaxed); + } + if !t.stats.enabled() { + return; // HUD hidden — skip the stats lock + } + t.stats.note_displayed( + e2e_valid.then_some((e2e_ns / 1000) as u64), + display_us, + latch_us, + ); } /// React to an output-format change by signalling the stream's HDR dataspace on the Surface (SDR diff --git a/clients/android/native/src/decode/sync_loop.rs b/clients/android/native/src/decode/sync_loop.rs index 400054c4..9885adc8 100644 --- a/clients/android/native/src/decode/sync_loop.rs +++ b/clients/android/native/src/decode/sync_loop.rs @@ -185,9 +185,12 @@ pub(super) fn run_sync( // render = true are parked in the tracker; the OnFrameRendered callback pairs them with // SurfaceFlinger's render timestamp. `render_cb` is the callback's leaked Arc refcount, // reclaimed after the codec is dropped below. + // The `video_e2e` cell is the audio plane's alignment reference (see `DisplayTracker`): this + // legacy loop feeds it too, so A/V sync works with "Low-latency mode" off as well. let tracker = DisplayTracker::new( stats.clone(), clock_offset.clone(), + client.video_e2e_shared(), std::sync::Arc::new(super::presenter::PresentMeter::new()), ); let render_cb = install_render_callback(&codec, &tracker); diff --git a/clients/android/native/src/session/planes.rs b/clients/android/native/src/session/planes.rs index 2fef14ac..198ea461 100644 --- a/clients/android/native/src/session/planes.rs +++ b/clients/android/native/src/session/planes.rs @@ -177,12 +177,12 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopVideo( } /// `NativeBridge.nativeVideoStats(handle): DoubleArray?` — drain ~1 s of decode stats for the HUD -/// (unified stats spec, `design/stats-unification.md`). Returns 33 doubles +/// (unified stats spec, `design/stats-unification.md`). Returns 35 doubles /// `[fps, mbps, e2eP50Ms, e2eP95Ms, latValid, skewCorrected, width, height, refreshHz, framesLost, /// bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc, hostNetP50Ms, decodeP50Ms, hostP50Ms, /// netP50Ms, lostWindow, skippedWindow, fecWindow, framesWindow, dispValid, displayP50Ms, /// e2eDispP50Ms, e2eDispP95Ms, paceP50Ms, latchP50Ms, presentsWindow, presenterActive, -/// feedP50Ms, codecP50Ms, skippedOverflowWindow]` +/// feedP50Ms, codecP50Ms, skippedOverflowWindow, audioBufferMs, audioAvOffsetMs]` /// (the flags are 1.0/0.0; indexes 0–21 match the previous 22-double layout — 0–13 the original /// 14-double one with the latency pair re-based to the end-to-end capture→decoded headline, 14/15 /// the stage p50s tiling it: `host+network` = capture→received, `decode` = received→decoded; 16/17 @@ -203,7 +203,10 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopVideo( /// received→queued (hand-off + input-slot wait) at 30 and `codec` = queued→decoded (codec-pure, /// from the AU's last piece) at 31, both 0.0 when no sample landed (sync loop); 32 is the /// parked-AU overflow subset of the window's `skipped` at 19 (decoder fell behind, vs benign -/// newest-wins pacing)), or `null` when no decode thread is running. +/// newest-wins pacing); 33/34 are the AUDIO plane's latency — the playback ring's live depth in ms +/// and the A/V sync loop's smoothed offset in ms (positive = audio behind the picture) — both live +/// gauges rather than windowed samples, like the cumulative drop total at 9), or `null` when no +/// decode thread is running. /// Poll ~1 Hz from the UI; each call /// resets the measurement window. Not android-gated — pure `jni` + connector reads, so it links on /// the host build too (Kotlin only ever calls it on device). @@ -227,7 +230,7 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats( .drain(h.client.frames_dropped(), h.client.fec_recovered_shards()); let mode = h.client.mode(); let color = h.client.color; - let buf: [f64; 33] = [ + let buf: [f64; 35] = [ snap.fps, snap.mbps, snap.e2e_p50_ms, @@ -281,6 +284,15 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats( snap.feed_p50_ms, snap.codec_p50_ms, snap.skipped_overflow as f64, + // The audio plane's own latency (`design/audio-latency-overhaul.md`): how much decoded + // audio is queued ahead of the speaker, and where the A/V sync loop measures that + // PUTS it relative to the picture (+ = audio behind). Both, because a deep ring on a + // jittery link is correct behaviour and only the offset tells that apart from audio + // simply held late. Live gauges written by the audio thread — before this the whole + // plane published nothing any surface could render, so a "the audio delay is way too + // high" report had no instrument behind it at all. + h.client.audio_buffer_ms() as f64, + h.client.audio_av_offset_ms() as f64, ]; let arr = match env.new_double_array(buf.len() as jsize) { Ok(a) => a, diff --git a/docs-site/content/docs/stats.md b/docs-site/content/docs/stats.md index dce3cccb..2617c238 100644 --- a/docs-site/content/docs/stats.md +++ b/docs-site/content/docs/stats.md @@ -73,6 +73,7 @@ e2e 14.2/19.8 ms (p50/p95) · host 3.1 · net 6.7 · decode 2.1 · display 2.3 m host: queue 0.6 · encode 1.8 · xfer 0.2 · pace 0.5 ms present: mailbox lost 3 (2.4%) +audio buffer 28 ms · a/v +4 ms ``` Android (headline and `display` both floor-shaved, like the Apple clients — the raw @@ -85,6 +86,7 @@ HEVC · 10-bit · HDR (BT.2020 PQ) · 4:2:0 end-to-end 14.2 ms p50 · 19.8 p95 · capture→displayed = host 3.1 + network 6.7 + decode 2.1 + display 2.3 · presents 119 os present +16.7 excluded (display pipeline minimum) +audio buffer 28 ms · a/v +4 ms lost 3 (2.4%) · skipped 1 · FEC 12 ``` @@ -185,6 +187,16 @@ lost 3 (2.4%) (frames your client chose not to display because a newer one had already arrived) and `FEC` (packet shards the error correction recovered this second — loss you *didn't* feel) are reported by the **Android client only**; the other clients show `lost` alone. +- **The audio line** — Detailed only, on Linux · Windows · Steam Deck · Android, and shown + once sound is actually playing. `audio buffer` is how much decoded audio is queued ahead + of your speakers; `a/v` is where that *puts* it relative to the picture — **positive means + audio is playing behind the picture**, negative means ahead of it. The client steers the + buffer to drive `a/v` toward zero, but never below the depth your link's jitter needs, so + on a rough connection you may see the buffer hold and a small `a/v` remain: that is the + client choosing an unbroken stream over perfect lip-sync, and it is the honest reading + rather than a hidden compromise. The `a/v` term is omitted when it is zero — aligned, or + not yet measured (it needs a frame on screen to compare against, and a few seconds to + settle). The Apple clients do not report it yet. All values refresh once per second over the last second of frames.