diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/Profiles.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/Profiles.kt index cf6fb2fb..5cb6c3c8 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/Profiles.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/Profiles.kt @@ -48,6 +48,9 @@ data class SettingsOverlay( * else, but here it is the one knob a marginal link wants turned off per host. */ val lowLatencyMode: Boolean? = null, + /** The timeline presenter's intent pair — cross-client keys, see [Settings.presentPriority]. */ + val presentPriority: String? = null, + val smoothBuffer: Int? = null, /** * Overlay keys a newer build wrote and this one doesn't model — carried through a load→save * round-trip untouched. The don't-clobber rule: opening and saving a profile on an older client @@ -73,6 +76,8 @@ data class SettingsOverlay( gamepad = gamepad ?: base.gamepad, statsVerbosity = statsVerbosity ?: base.statsVerbosity, lowLatencyMode = lowLatencyMode ?: base.lowLatencyMode, + presentPriority = presentPriority ?: base.presentPriority, + smoothBuffer = smoothBuffer ?: base.smoothBuffer, ) /** @@ -104,6 +109,8 @@ data class SettingsOverlay( gamepad = if (after.gamepad != before.gamepad) after.gamepad else gamepad, statsVerbosity = if (after.statsVerbosity != before.statsVerbosity) after.statsVerbosity else statsVerbosity, lowLatencyMode = if (after.lowLatencyMode != before.lowLatencyMode) after.lowLatencyMode else lowLatencyMode, + presentPriority = if (after.presentPriority != before.presentPriority) after.presentPriority else presentPriority, + smoothBuffer = if (after.smoothBuffer != before.smoothBuffer) after.smoothBuffer else smoothBuffer, ) /** @@ -127,6 +134,8 @@ data class SettingsOverlay( "gamepad" -> copy(gamepad = null) "stats_verbosity" -> copy(statsVerbosity = null) "low_latency_mode" -> copy(lowLatencyMode = null) + "present_priority" -> copy(presentPriority = null) + "smooth_buffer" -> copy(smoothBuffer = null) else -> this } @@ -147,6 +156,8 @@ data class SettingsOverlay( if (gamepad != null) add("gamepad") if (statsVerbosity != null) add("stats_verbosity") if (lowLatencyMode != null) add("low_latency_mode") + if (presentPriority != null) add("present_priority") + if (smoothBuffer != null) add("smooth_buffer") } /** @@ -175,6 +186,8 @@ data class SettingsOverlay( gamepad?.let { j.put("gamepad", it) } statsVerbosity?.let { j.put("stats_verbosity", it.name) } lowLatencyMode?.let { j.put("low_latency_mode", it) } + presentPriority?.let { j.put("present_priority", it) } + smoothBuffer?.let { j.put("smooth_buffer", it) } return j } @@ -187,6 +200,7 @@ data class SettingsOverlay( "width", "height", "refresh_hz", "bitrate_kbps", "render_scale", "codec", "hdr_enabled", "compositor", "audio_channels", "mic_enabled", "touch_mode", "mouse_mode", "invert_scroll", "gamepad", "stats_verbosity", "low_latency_mode", + "present_priority", "smooth_buffer", ) internal fun fromJson(j: JSONObject): SettingsOverlay = SettingsOverlay( @@ -209,6 +223,8 @@ data class SettingsOverlay( statsVerbosity = j.optStringOrNull("stats_verbosity") ?.let { n -> StatsVerbosity.entries.firstOrNull { it.name == n } }, lowLatencyMode = j.optBooleanOrNull("low_latency_mode"), + presentPriority = j.optStringOrNull("present_priority"), + smoothBuffer = j.optIntOrNull("smooth_buffer"), extra = j.keys().asSequence().filter { it !in KNOWN }.associateWith { j.get(it) }, ) } diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt index 13950ae7..7ab44805 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt @@ -83,6 +83,19 @@ data class Settings( * feeds a queue that only grows. */ val lowLatencyMode: Boolean = true, + /** + * The timeline presenter's intent — the cross-client `present_priority` pair (the Apple + * client's "Prioritize" picker, same stored values): `"latency"` (default) = newest-wins, + * a frame reaches glass the instant the glass budget opens; `"smooth"` = a small FIFO + * drained one frame per vsync, absorbing network/decode jitter at one refresh of added + * display latency per buffered frame. Anything unrecognized resolves to latency. + */ + val presentPriority: String = "latency", + /** + * The smoothness buffer depth (`smooth_buffer`): 0 = Automatic (2 frames), else 1..3. + * Only meaningful when [presentPriority] is `"smooth"`. + */ + val smoothBuffer: Int = 0, /** * Wake-on-LAN a saved host before connecting when it isn't currently seen on mDNS. On (default): * a connect to a host with a learned MAC that isn't advertising sends a magic packet and waits @@ -213,6 +226,8 @@ class SettingsStore(context: Context) { gamepadUiEnabled = prefs.getBoolean(K_GAMEPAD_UI, true), libraryEnabled = prefs.getBoolean(K_LIBRARY, true), lowLatencyMode = prefs.getBoolean(K_LOW_LATENCY, true), + presentPriority = prefs.getString(K_PRESENT_PRIORITY, "latency") ?: "latency", + smoothBuffer = prefs.getInt(K_SMOOTH_BUFFER, 0), autoWakeEnabled = prefs.getBoolean(K_AUTO_WAKE, true), rumbleOnPhone = prefs.getBoolean(K_RUMBLE_ON_PHONE, false), sc2Capture = prefs.getBoolean(K_SC2_CAPTURE, true), @@ -244,6 +259,8 @@ class SettingsStore(context: Context) { .putBoolean(K_GAMEPAD_UI, s.gamepadUiEnabled) .putBoolean(K_LIBRARY, s.libraryEnabled) .putBoolean(K_LOW_LATENCY, s.lowLatencyMode) + .putString(K_PRESENT_PRIORITY, s.presentPriority) + .putInt(K_SMOOTH_BUFFER, s.smoothBuffer) .putBoolean(K_AUTO_WAKE, s.autoWakeEnabled) .putBoolean(K_RUMBLE_ON_PHONE, s.rumbleOnPhone) .putBoolean(K_SC2_CAPTURE, s.sc2Capture) @@ -285,6 +302,8 @@ class SettingsStore(context: Context) { * on; both stale keys are abandoned unread. The toggle stays as a per-device escape hatch. */ const val K_LOW_LATENCY = "low_latency_mode_v2" + const val K_PRESENT_PRIORITY = "present_priority" + const val K_SMOOTH_BUFFER = "smooth_buffer" const val K_AUTO_WAKE = "auto_wake_enabled" const val K_RUMBLE_ON_PHONE = "rumble_on_phone" const val K_SC2_CAPTURE = "sc2_capture" @@ -506,6 +525,29 @@ val COMPOSITOR_OPTIONS = listOf( /** (verbosity, label) for the stats-overlay detail picker. Order = the live 3-finger-tap cycle. */ val STATS_VERBOSITY_OPTIONS = StatsVerbosity.entries.map { it to it.label } +/** [Settings.presentPriority] as the wire int `nativeStartVideo` takes (0 = latency, 1 = smooth). + * Unrecognized values resolve to latency — same rule as the Apple client. */ +fun Settings.presentPriorityWire(): Int = if (presentPriority == "smooth") 1 else 0 + +/** (stored value, label) for the presenter-intent picker — the Apple client's table verbatim. */ +val PRESENT_PRIORITY_OPTIONS = listOf( + "latency" to "Lowest latency", + "smooth" to "Smoothness", +) + +/** (frames, label) for the smoothness-buffer picker; each buffered frame ≈ one refresh interval + * of jitter absorbed for one interval of added display latency ([hz] labels the cost). */ +fun smoothBufferOptions(hz: Int): List> { + val periodMs = 1000.0 / maxOf(24, hz) + fun cost(frames: Int) = "+%.0f ms".format(periodMs * frames) + return listOf( + 0 to "Automatic", + 1 to "1 frame (${cost(1)})", + 2 to "2 frames (${cost(2)})", + 3 to "3 frames (${cost(3)})", + ) +} + /** (mode, label) for the touch-input model. */ val TOUCH_MODE_OPTIONS = listOf( TouchMode.TRACKPAD to "Trackpad", diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt index 76c1495f..20ae8660 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt @@ -711,6 +711,26 @@ private fun DisplaySettings(s: Settings, update: (Settings) -> Unit, context: an field = "low_latency_mode", onCheckedChange = { on -> update(s.copy(lowLatencyMode = on)) }, ) + // The timeline presenter's intent — the Apple client's "Prioritize" pair, same stored + // values, so a profile written on one platform means the same thing here. + SettingDropdown( + label = "Prioritize", + options = PRESENT_PRIORITY_OPTIONS, + selected = if (s.presentPriority == "smooth") "smooth" else "latency", + field = "present_priority", + caption = "Lowest latency shows each frame the moment it can reach the panel; " + + "Smoothness buffers a little to absorb network jitter.", + ) { v -> update(s.copy(presentPriority = v)) } + if (s.presentPriority == "smooth") { + SettingDropdown( + label = "Smoothness buffer", + options = smoothBufferOptions(if (s.hz > 0) s.hz else nhz), + selected = if (s.smoothBuffer in 1..3) s.smoothBuffer else 0, + field = "smooth_buffer", + caption = "Each buffered frame absorbs one refresh of jitter and adds one of " + + "display latency — the cost shown is at the session's refresh rate.", + ) { v -> update(s.copy(smoothBuffer = v)) } + } } SettingsGroup("Host output", footer = "Display changes apply from the next session.") { diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt index ba23c882..1b909097 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt @@ -112,8 +112,27 @@ internal fun StatsOverlay( } else { "host+network ${"%.1f".format(s[14])}" } - val displayTerm = if (dispValid) " + display ${"%.1f".format(s[23])}" else "" - statLine("= $hostTerms + decode ${"%.1f".format(s[15])}$displayTerm", Color.White) + // Timeline-presenter split (s[26]/s[27], when s[29] flags it active): the display + // term decomposes into pace (store + glass budget) + latch (SurfaceFlinger), and + // s[28] is the on-glass confirm count — presents ≪ fps means the presenter is + // dropping/serializing, an fps deficit is upstream. + val split = s.size >= 30 && s[29] != 0.0 && (s[26] > 0 || s[27] > 0) + val displayTerm = when { + dispValid && split -> + " + display ${"%.1f".format(s[23])} " + + "(pace ${"%.1f".format(s[26])} + latch ${"%.1f".format(s[27])})" + dispValid -> " + display ${"%.1f".format(s[23])}" + else -> "" + } + val presents = if (s.size >= 30 && s[29] != 0.0) { + " · presents ${s[28].toInt()}" + } else { + "" + } + statLine( + "= $hostTerms + decode ${"%.1f".format(s[15])}$displayTerm$presents", + Color.White, + ) } } counterLine(s, lost)?.let { statLine(it, Color(0xFFFFB0B0)) } diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt index d1c6a1e6..00493892 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt @@ -584,6 +584,8 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { lowLatencyMode, choice?.lowLatencyFeature ?: false, isTv, + initialSettings.presentPriorityWire(), + initialSettings.smoothBuffer, ) NativeBridge.nativeStartAudio(handle, lowLatencyMode) if (micWanted) NativeBridge.nativeStartMic(handle) diff --git a/clients/android/app/src/test/kotlin/io/unom/punktfunk/screenshots/ShotScenes.kt b/clients/android/app/src/test/kotlin/io/unom/punktfunk/screenshots/ShotScenes.kt index 0f6b12c6..b012a453 100644 --- a/clients/android/app/src/test/kotlin/io/unom/punktfunk/screenshots/ShotScenes.kt +++ b/clients/android/app/src/test/kotlin/io/unom/punktfunk/screenshots/ShotScenes.kt @@ -366,6 +366,8 @@ internal fun StreamScene(verbosity: StatsVerbosity = StatsVerbosity.DETAILED) { 10.0, 9.0, 16.0, 1.0, 0.9, 0.4, 0.6, 0.3, 2.0, 1.0, 5.0, 238.0, 1.0, 0.5, 1.8, 2.6, + // Timeline-presenter split: pace + latch tile the display term; presents ≈ fps. + 0.2, 0.3, 236.0, 1.0, ), verbosity = verbosity, decoderLabel = "c2.qti.hevc.decoder · low-latency", diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index 677a41ec..bf084384 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -210,7 +210,9 @@ object NativeBridge { * original synchronous pipeline as the per-device escape hatch); [lowLatencyFeature] is whether * [decoderName] advertised `FEATURE_LowLatency` (HUD label only). [isTv] drives an active HDMI * mode switch to the stream refresh on TV boxes when the toggle is on (vs. the softer seamless - * hint otherwise). No-op if already started. + * hint otherwise). [presentPriority]/[smoothBuffer] are the timeline presenter's intent + * (0 = lowest latency / 1 = smoothness; buffer 0 = automatic, else 1..3 frames) — the Apple + * client's `present_priority`/`smooth_buffer` pair. No-op if already started. */ external fun nativeStartVideo( handle: Long, @@ -219,6 +221,8 @@ object NativeBridge { lowLatencyMode: Boolean, lowLatencyFeature: Boolean, isTv: Boolean, + presentPriority: Int, + smoothBuffer: Int, ) /** Stop + join the decode thread without closing the session. No-op on `0`. */ @@ -233,11 +237,11 @@ object NativeBridge { /** * Drain ~1 s of live decode stats for the on-stream HUD, or `null` when no decode thread runs. - * Returns 26 doubles (unified stats spec, `design/stats-unification.md`): + * Returns 30 doubles (unified stats spec, `design/stats-unification.md`): * `[fps, mbps, e2eP50Ms, e2eP95Ms, latValid, skewCorrected, width, height, refreshHz, framesLost, * bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc, hostNetP50Ms, decodeP50Ms, hostP50Ms, * netP50Ms, lostWindow, skippedWindow, fecWindow, framesWindow, dispValid, displayP50Ms, - * e2eDispP50Ms, e2eDispP95Ms]` + * e2eDispP50Ms, e2eDispP95Ms, paceP50Ms, latchP50Ms, presentsWindow, presenterActive]` * (the flags are 1.0/0.0; indexes 2/3 are the end-to-end capture→decoded headline; 10–13 * describe the negotiated video feed — bit depth 8/10, CICP primaries/transfer, and the HEVC * chroma_format_idc 1=4:2:0 / 3=4:4:4; 14/15 are the stage p50s tiling the headline — diff --git a/clients/android/native/src/decode/async_loop.rs b/clients/android/native/src/decode/async_loop.rs index 9164b299..ed5390ee 100644 --- a/clients/android/native/src/decode/async_loop.rs +++ b/clients/android/native/src/decode/async_loop.rs @@ -17,10 +17,12 @@ use super::display::{ apply_hdr_dataspace, install_render_callback, release_render_callback, DisplayTracker, }; use super::latency::{note_decoded_pts, now_realtime_ns, take_flags}; +use super::presenter::{presenter_disabled_by_sysprop, PresentMeter, PresentPriority, Presenter}; use super::setup::{ android_hdr_static_info, boost_hot_threads, boost_thread_priority, codec_mime, configure_low_latency, create_codec, try_set_frame_rate, }; +use super::vsync::{now_monotonic_ns, VsyncClock}; use super::{ DecodeOptions, FRAME_PARK_CAP, IN_FLIGHT_CAP, NO_OUTPUT_PATIENCE, NO_VIDEO_PATIENCE, NO_VIDEO_RETRY, PENDING_SPLIT_CAP, @@ -55,6 +57,8 @@ enum DecodeEvent { }, /// The output format changed — re-check the stream's colour signalling (HDR DataSpace). FormatChanged, + /// A panel vsync (from the [`VsyncClock`] thread) — the presenter's retry/pacing tick. + Vsync, /// The codec reported an error; `fatal` when neither recoverable nor transient. Error { fatal: bool }, } @@ -78,6 +82,8 @@ pub(super) fn run_async( ll_feature, low_latency_mode, is_tv, + present_priority, + smooth_buffer, } = opts; boost_thread_priority(); let mode = client.mode(); @@ -196,9 +202,33 @@ pub(super) fn run_async( // parked in the tracker at release; the OnFrameRendered callback pairs it with // SurfaceFlinger's render timestamp. `render_cb` is the callback's leaked Arc refcount, // reclaimed after the codec is dropped below. - let tracker = DisplayTracker::new(stats.clone(), clock_offset.clone()); + let meter = Arc::new(PresentMeter::new()); + let tracker = DisplayTracker::new(stats.clone(), clock_offset.clone(), meter.clone()); let render_cb = install_render_callback(&codec, &tracker); + // The timeline presenter (see `presenter.rs`): newest-wins / smoothing store, one-in-flight + // glass budget, timeline-timed release. `debug.punktfunk.presenter = arrival` selects the + // legacy release-immediately path for a rebuild-free on-device A/B. + let mut presenter = if presenter_disabled_by_sysprop() { + log::info!("decode: presenter = arrival (sysprop) — legacy immediate release"); + None + } else { + let priority = PresentPriority::resolve(present_priority, smooth_buffer); + log::info!( + "decode: presenter = timeline ({})", + match priority { + PresentPriority::Latency => "lowest latency".to_string(), + PresentPriority::Smooth { buffer } => format!("smoothness, buffer {buffer}"), + } + ); + Some(Presenter::new(priority)) + }; + stats.set_presenter_active(presenter.is_some()); + // The vsync clock, started LAZILY on the first decoded frame (see `vsync.rs`); its ticks ride + // the same event channel. The Sender parks here until that moment. + let mut vsync: Option = None; + let mut vsync_tx = presenter.is_some().then(|| ev_tx.clone()); + // Feeder thread: block on the network so this loop doesn't (an AU's arrival becomes an event that // wakes us immediately, with no input-side poll latency). It also records the `received` HUD stat. let feeder = { @@ -277,6 +307,7 @@ pub(super) fn run_async( }; let work_t0 = Instant::now(); let mut fmt_dirty = false; + let mut vsync_tick = false; let mut aus_dropped: u64 = 0; if let Some(ev) = ev0 { aus_dropped += u64::from(dispatch_event( @@ -285,6 +316,7 @@ pub(super) fn run_async( &mut free_inputs, &mut ready, &mut fmt_dirty, + &mut vsync_tick, &mut fatal, &mut gate, &mut recovery_flags, @@ -299,11 +331,17 @@ pub(super) fn run_async( &mut free_inputs, &mut ready, &mut fmt_dirty, + &mut vsync_tick, &mut fatal, &mut gate, &mut recovery_flags, )); } + if vsync_tick { + if let Some(p) = presenter.as_mut() { + p.on_vsync(); + } + } stats.note_skipped(aus_dropped); // parked-AU overflow drops are client-side skips too if fmt_dirty { apply_hdr_dataspace(&codec, &window, &mut applied_ds); @@ -317,6 +355,7 @@ pub(super) fn run_async( &mut oversized_dropped, ); let had_output = !ready.is_empty(); + let rendered_before = rendered; present_ready( &codec, &client, @@ -326,14 +365,39 @@ pub(super) fn run_async( &in_flight, clock_offset.load(Ordering::Relaxed), &tracker, + &mut presenter, &mut rendered, &mut discarded, &mut gate, &mut recovery_flags, ); + // The presenter's decision point runs EVERY pass — frame arrivals, vsync ticks and the + // 5 ms housekeeping wake all land here, which is what reopens the glass budget on time + // even when the choreographer clock is absent. + if let Some(p) = presenter.as_mut() { + let clock = vsync.as_ref().map(|v| v.shared().as_ref()); + if p.pump(&codec, clock, &tracker, &stats, now_monotonic_ns()) { + rendered += 1; + } + p.flush_log(&meter, clock); + } + let presented_now = rendered > rendered_before; + // Start the vsync clock LAZILY on the first decoded output (eager, it ticks the panel + // rate into a session that has no frame yet — the Apple deadline presenter's bootstrap + // lesson). A `None` from start (no choreographer surface) simply leaves ASAP targets. + if had_output && vsync.is_none() { + if let Some(tx) = vsync_tx.take() { + vsync = VsyncClock::start(Box::new(move || { + let _ = tx.send(DecodeEvent::Vsync); + })); + if vsync.is_none() { + log::info!("decode: no choreographer clock — presenter uses ASAP targets"); + } + } + } work_accum_ns += work_t0.elapsed().as_nanos() as i64; - if had_output { + if presented_now { if !hint_tried { hint_tried = true; let tids = client.hot_thread_ids(); @@ -420,6 +484,10 @@ pub(super) fn run_async( } } + if let Some(p) = presenter.as_mut() { + p.release_all(&codec); // hand every held output buffer back before the codec stops + } + drop(vsync); // stop + join the choreographer thread; its channel sends are harmless after let _ = codec.stop(); shutdown.store(true, Ordering::SeqCst); // ensure the feeder wakes and exits, then join it if let Some(j) = feeder { @@ -520,6 +588,7 @@ fn dispatch_event( free_inputs: &mut VecDeque, ready: &mut Vec, fmt_dirty: &mut bool, + vsync_tick: &mut bool, fatal: &mut bool, gate: &mut ReanchorGate, recovery_flags: &mut VecDeque<(u64, u32)>, @@ -552,6 +621,7 @@ fn dispatch_event( decoded_ns, }), DecodeEvent::FormatChanged => *fmt_dirty = true, + DecodeEvent::Vsync => *vsync_tick = true, DecodeEvent::Error { fatal: f } => { if f { *fatal = true; @@ -613,13 +683,14 @@ fn feed_ready( } } -/// Present only the NEWEST ready output (render = true) and release the rest without rendering — a -/// burst of stale frames on glass is worse than skipping to the freshest (the sync loop's newest-ready -/// policy, callback-driven). Every dequeued buffer, rendered or not, is the HUD's `decoded` -/// measurement point (it finished decoding either way); samples are recorded in pts order so the -/// receipt-map eviction stays monotonic. The presented frame's `(pts, decoded stamp)` is parked in -/// `tracker` for the OnFrameRendered callback — the `display` stage's other endpoint. `ready` is -/// drained. +/// Route the ready outputs toward glass. With the timeline presenter (default): fold each output +/// through the re-anchor gate in pts order, hand the approved ones to the presenter's store +/// (newest-wins / smoothing FIFO — the actual release happens in `Presenter::pump`, budgeted and +/// timeline-timed), and release withheld concealment unrendered. Legacy (`arrival` sysprop): +/// present only the NEWEST ready output immediately and release the rest unrendered — the +/// original policy. Every dequeued buffer, rendered or not, is the HUD's `decoded` measurement +/// point (it finished decoding either way); samples are recorded in pts order so the receipt-map +/// eviction stays monotonic. `ready` is drained. #[allow(clippy::too_many_arguments)] // one call site; mirrors the sync loop's drain fn present_ready( codec: &MediaCodec, @@ -630,6 +701,7 @@ fn present_ready( in_flight: &Mutex>, clock_offset: i64, tracker: &DisplayTracker, + presenter: &mut Option, rendered: &mut u64, discarded: &mut u64, gate: &mut ReanchorGate, @@ -657,31 +729,46 @@ fn present_ready( } } // Fold EVERY output through the gate in pts (== decode) order — even the ones newest-wins discards — - // so the two-mark re-anchor count stays correct; the newest's verdict decides whether it reaches - // glass (`false` = withheld concealment; the SurfaceView keeps the last rendered frame frozen on). + // so the two-mark re-anchor count stays correct; a `false` verdict is withheld concealment (the + // SurfaceView keeps the last rendered frame frozen on). let now = Instant::now(); - let last = ready.len() - 1; let mut skipped: u64 = 0; - for (i, o) in ready.drain(..).enumerate() { - let flags = take_flags(recovery_flags, o.pts_us); - let present = gate.on_decoded(flags, false, now) == GateVerdict::Present; - let render = i == last && present; - match codec.release_output_buffer_by_index(o.index, render) { - Ok(()) if render => { - *rendered += 1; - if stats.enabled() { - tracker.note_rendered(o.pts_us, o.decoded_ns); + if let Some(p) = presenter.as_mut() { + for o in ready.drain(..) { + let flags = take_flags(recovery_flags, o.pts_us); + if gate.on_decoded(flags, false, now) == GateVerdict::Present { + let dropped = p.submit(codec, o.index, o.pts_us, o.decoded_ns); + skipped += dropped; + *discarded += dropped; + } else { + if let Err(e) = codec.release_output_buffer_by_index(o.index, false) { + log::warn!("decode: release_output_buffer_by_index({}): {e}", o.index); } - } - Ok(()) => { *discarded += 1; skipped += 1; } - Err(e) => { - log::warn!( - "decode: release_output_buffer_by_index({}, {render}): {e}", - o.index - ) + } + } else { + let last = ready.len() - 1; + for (i, o) in ready.drain(..).enumerate() { + let flags = take_flags(recovery_flags, o.pts_us); + let present = gate.on_decoded(flags, false, now) == GateVerdict::Present; + let render = i == last && present; + match codec.release_output_buffer_by_index(o.index, render) { + Ok(()) if render => { + *rendered += 1; + tracker.note_rendered(o.pts_us, o.decoded_ns, now_realtime_ns()); + } + Ok(()) => { + *discarded += 1; + skipped += 1; + } + Err(e) => { + log::warn!( + "decode: release_output_buffer_by_index({}, {render}): {e}", + o.index + ) + } } } } diff --git a/clients/android/native/src/decode/display.rs b/clients/android/native/src/decode/display.rs index 80c67649..adb83c70 100644 --- a/clients/android/native/src/decode/display.rs +++ b/clients/android/native/src/decode/display.rs @@ -35,32 +35,37 @@ pub(super) struct DisplayTracker { /// loaded per callback so mid-stream re-syncs apply. Holding the handle (not the client) /// keeps the leaked render-callback refcount from pinning the whole session alive. clock_offset: Arc, - /// `(pts_us, decoded_real_ns)` of frames released with `render = true`, in release order, - /// awaiting their callback. Pushes are HUD-gated by the caller, so this stays empty (and the - /// callback early-outs) while the overlay is hidden. - rendered: Mutex>, + /// Always-on latch/display accumulator for the presenter's 1 Hz `pf-present` line — + /// independent of the HUD gate, so a HUD-off A/B stays measurable from logcat. + meter: Arc, + /// `(pts_us, decoded_real_ns, released_real_ns)` of frames released with `render = true`, in + /// release order, awaiting their callback. Pushed on EVERY render (no HUD gate — the ring is + /// a 64-tuple bound and the latch metric wants to exist when nobody is watching). + rendered: Mutex>, } impl DisplayTracker { pub(super) fn new( stats: Arc, clock_offset: Arc, + meter: Arc, ) -> Arc { Arc::new(DisplayTracker { stats, clock_offset, + meter, rendered: Mutex::new(VecDeque::new()), }) } - /// Park one just-rendered frame's `(pts, decoded stamp)` for the render callback to pair. - /// Caller gates on the HUD being visible. - pub(super) fn note_rendered(&self, pts_us: u64, decoded_ns: i128) { + /// Park one just-rendered frame's `(pts, decoded stamp, release stamp)` for the render + /// callback to pair — the release stamp is the latch metric's start (release→displayed). + pub(super) fn note_rendered(&self, pts_us: u64, decoded_ns: i128, released_ns: i128) { let mut g = self .rendered .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - g.push_back((pts_us, decoded_ns)); + g.push_back((pts_us, decoded_ns, released_ns)); if g.len() > RENDERED_CAP { g.pop_front(); // render callbacks stopped coming (allowed under load) — evict } @@ -152,36 +157,38 @@ unsafe extern "C" fn on_frame_rendered( // `Arc::into_raw` pointer from `install_render_callback`, whose refcount is held for as long as // the codec exists, and the codec is what delivers this call. let t = unsafe { &*(userdata as *const DisplayTracker) }; - if !t.stats.enabled() { - return; // HUD hidden — the ring is empty too (pushes are caller-gated) - } let displayed_ns = now_realtime_ns() - (now_monotonic_ns() - system_nano as i128); let pts_us = media_time_us.max(0) as u64; // Pair the frame back to its release record, evicting older entries (their callbacks were - // dropped by the platform, or the entry predates a HUD toggle) — same monotonic-eviction - // discipline as `note_decoded_pts`. - let mut decoded_ns = None; + // dropped by the platform) — same monotonic-eviction discipline as `note_decoded_pts`. + let mut paired = None; { let mut g = t .rendered .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - while let Some(&(p, d)) = g.front() { + while let Some(&(p, d, r)) = g.front() { if p > pts_us { break; // future frame — leave it for its own callback } g.pop_front(); if p == pts_us { - decoded_ns = Some(d); + paired = Some((d, r)); break; } } } + let display_us = paired.map(|(d, _)| ((displayed_ns - d).max(0) / 1000) as u64); + let latch_us = paired.map(|(_, r)| ((displayed_ns - r).max(0) / 1000) as u64); + // Always-on half: the presenter's pf-present line reads these with the HUD off. + t.meter.note_latch(latch_us); + if !t.stats.enabled() { + return; // HUD hidden — skip the skew math + the stats lock + } let e2e_ns = displayed_ns + t.clock_offset.load(Ordering::Relaxed) as i128 - pts_us as i128 * 1000; let e2e_us = (e2e_ns > 0 && e2e_ns < 10_000_000_000).then_some((e2e_ns / 1000) as u64); - let display_us = decoded_ns.map(|d| ((displayed_ns - d).max(0) / 1000) as u64); - t.stats.note_displayed(e2e_us, display_us); + t.stats.note_displayed(e2e_us, display_us, latch_us); } /// React to an output-format change by signalling the stream's HDR dataspace on the Surface (SDR diff --git a/clients/android/native/src/decode/mod.rs b/clients/android/native/src/decode/mod.rs index 55cf2bce..18d411d7 100644 --- a/clients/android/native/src/decode/mod.rs +++ b/clients/android/native/src/decode/mod.rs @@ -9,8 +9,10 @@ mod async_loop; mod display; mod latency; +mod presenter; mod setup; mod sync_loop; +mod vsync; use async_loop::run_async; pub(crate) use setup::{codec_label, codec_mime}; @@ -106,6 +108,13 @@ pub(crate) struct DecodeOptions { /// TV form factor (Kotlin's `UiModeManager`): actively drive the HDMI output into the stream's /// refresh mode, vs. the softer seamless hint on a phone/tablet. pub is_tv: bool, + /// The user's presentation intent (`present_priority` setting): 0 = lowest latency + /// (newest-wins), 1 = smoothness (a small FIFO). Resolved by + /// [`presenter::PresentPriority::resolve`]; anything else = latency. + pub present_priority: i32, + /// The smoothness buffer depth (`smooth_buffer` setting): 0 = automatic (2), else 1..=3. + /// Only meaningful with `present_priority` = smooth. + pub smooth_buffer: i32, } /// The decode entry point on the `pf-decode` thread: dispatches to the async or synchronous loop. diff --git a/clients/android/native/src/decode/presenter.rs b/clients/android/native/src/decode/presenter.rs new file mode 100644 index 00000000..1766c8f6 --- /dev/null +++ b/clients/android/native/src/decode/presenter.rs @@ -0,0 +1,385 @@ +//! The timeline presenter — Android's port of the Apple client's stage-4 deadline discipline +//! (`clients/apple/.../Stage2Pipeline.swift`, the `.deadline` pacing): +//! +//! * a **newest-wins slot** (or a small smoothing FIFO, by user intent) between decode and +//! release, so a burst coalesces in the app — as an explicit, counted drop — instead of +//! queueing behind the display; +//! * a **glass budget of exactly one**: at most one undisplayed release in flight to +//! SurfaceFlinger, reopened on the clock-predicted latch (with a 100 ms stale force-open as +//! the liveness backstop, mirroring Apple's `PresentGate.staleAfter`). The BufferQueue can +//! hold at most the frame being scanned out plus one — a standing queue is unconstructible; +//! * a **timed release**: `AMediaCodec_releaseOutputBufferAtTime` targeting the platform's own +//! frame timeline (API 33+, via [`super::vsync`]), so the latch phase is deterministic instead +//! of inheriting network + decode jitter. On the 31/32 fallback the release is ASAP — +//! identical to the legacy path — and only the budget prediction uses the measured period. +//! +//! The legacy behaviour (release the newest ready buffer immediately, unbudgeted) remains +//! selectable at runtime: `adb shell setprop debug.punktfunk.presenter arrival` — the on-device +//! A/B needs no rebuild. The user-facing escape hatch stays the "Low-latency mode" master toggle +//! (off = the synchronous pre-overhaul loop, no presenter at all). + +use ndk::media::media_codec::MediaCodec; +use std::collections::VecDeque; +use std::sync::Mutex; +use std::time::Instant; + +use super::display::DisplayTracker; +use super::latency::now_realtime_ns; +use super::vsync::VsyncShared; + +/// Submit-margin ahead of a timeline's deadline: the released buffer still has to travel +/// codec → BufferQueue → SurfaceFlinger's latch, so a deadline closer than this is treated as +/// missed and the next timeline is targeted. +const DEADLINE_MARGIN_NS: i64 = 1_000_000; + +/// The budget's liveness backstop: a release whose predicted latch never seems to arrive +/// (clock glitch, mode switch) force-reopens the budget this long after the release, counted in +/// `forced` — reads 0 on healthy systems (Apple's `PresentGate.staleAfter`, same value). +const STALE_REOPEN_NS: i64 = 100_000_000; + +/// Fallback latch-prediction period while the vsync clock is unmeasured/absent: one 120 Hz frame. +const FALLBACK_PERIOD_NS: i64 = 8_333_333; + +/// The user's presentation intent — the Apple client's `PresentPriority`, same resolution rules: +/// anything but an explicit "smooth" is latency; a smooth buffer outside 1..=3 becomes 2. +#[derive(Clone, Copy, PartialEq)] +pub(crate) enum PresentPriority { + /// Newest-wins, release the instant the budget opens. The default. + Latency, + /// A small FIFO (1..=3 frames) drained one per vsync: jitter absorbed at one refresh of + /// added display latency per slot, which the metrics show rather than hide. + Smooth { buffer: usize }, +} + +impl PresentPriority { + /// From the JNI ints (`presentPriority` 0 = latency / 1 = smooth; `smoothBuffer` 0 = auto). + pub(crate) fn resolve(priority: i32, buffer: i32) -> PresentPriority { + if priority != 1 { + return PresentPriority::Latency; + } + let b = if (1..=3).contains(&buffer) { + buffer as usize + } else { + 2 + }; + PresentPriority::Smooth { buffer: b } + } +} + +/// One decoded output buffer held for presentation. +struct HeldFrame { + index: usize, + pts_us: u64, + /// The output callback's `CLOCK_REALTIME` stamp — the pace metric's start (decoded→release). + decoded_ns: i128, +} + +/// The one-in-flight glass budget. +struct InFlight { + /// Monotonic instant the budget reopens: the release target's expected present (clock), or + /// `release + period` on the fallback path. + reopen_at_ns: i64, + released_at_ns: i64, +} + +/// Latch samples + display confirms recorded by the `OnFrameRendered` callback thread, drained by +/// the presenter's 1 Hz `pf-present` line. Always on (independent of the HUD) — this is what makes +/// a HUD-off wireless A/B readable from logcat. +pub(super) struct PresentMeter { + inner: Mutex, +} + +struct PresentMeterInner { + latch_us: Vec, + displays: u64, +} + +impl PresentMeter { + pub(super) fn new() -> PresentMeter { + PresentMeter { + inner: Mutex::new(PresentMeterInner { + latch_us: Vec::with_capacity(256), + displays: 0, + }), + } + } + + /// One displayed frame's release→displayed latch, µs. Callback thread; poison-proof. + pub(super) fn note_latch(&self, latch_us: Option) { + let mut g = self + .inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + g.displays += 1; + if let Some(l) = latch_us { + if g.latch_us.len() < 4096 { + g.latch_us.push(l); + } + } + } + + fn drain(&self) -> (Vec, u64) { + let mut g = self + .inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let displays = g.displays; + g.displays = 0; + (std::mem::take(&mut g.latch_us), displays) + } +} + +/// p50/max of an unsorted µs sample vec, in ms. (0, 0) when empty. +fn p50_max_ms(mut v: Vec) -> (f64, f64) { + if v.is_empty() { + return (0.0, 0.0); + } + v.sort_unstable(); + let p50 = v[v.len() / 2] as f64 / 1000.0; + let max = *v.last().unwrap() as f64 / 1000.0; + (p50, max) +} + +pub(super) struct Presenter { + /// 0 = newest-wins; 1..=3 = smoothing FIFO capacity. + fifo_capacity: usize, + frames: VecDeque, + /// FIFO preroll: `take` withholds until the buffer filled to capacity once, re-armed on a dry + /// run — the Apple `FrameStore` semantics (headroom never builds without it). + prerolled: bool, + inflight: Option, + /// A vsync arrived since the last release — the FIFO's one-per-refresh drain pace. + vsync_tick: bool, + // -- 1 Hz pf-present window, always on -- + released: u64, + paced_drops: u64, + no_budget: u64, + forced: u64, + dry: u64, + pace_us: Vec, + last_flush: Instant, +} + +impl Presenter { + pub(super) fn new(priority: PresentPriority) -> Presenter { + Presenter { + fifo_capacity: match priority { + PresentPriority::Latency => 0, + PresentPriority::Smooth { buffer } => buffer, + }, + frames: VecDeque::new(), + prerolled: false, + inflight: None, + vsync_tick: false, + released: 0, + paced_drops: 0, + no_budget: 0, + forced: 0, + dry: 0, + pace_us: Vec::with_capacity(256), + last_flush: Instant::now(), + } + } + + /// A vsync pulse from the clock thread's event — the retry tick for a parked frame and the + /// FIFO's drain pace. + pub(super) fn on_vsync(&mut self) { + self.vsync_tick = true; + } + + /// Accept one decoded, gate-approved output buffer. Newest-wins evicts everything older + /// (released unrendered — the explicit, counted drop); the FIFO evicts its oldest past + /// capacity. Returns how many frames were dropped by the policy (the HUD's `skipped`). + pub(super) fn submit( + &mut self, + codec: &MediaCodec, + index: usize, + pts_us: u64, + decoded_ns: i128, + ) -> u64 { + let mut dropped = 0u64; + if self.fifo_capacity == 0 { + while let Some(stale) = self.frames.pop_front() { + release_unrendered(codec, stale.index); + dropped += 1; + } + } + self.frames.push_back(HeldFrame { + index, + pts_us, + decoded_ns, + }); + if self.fifo_capacity > 0 && self.frames.len() > self.fifo_capacity { + if let Some(stale) = self.frames.pop_front() { + release_unrendered(codec, stale.index); + dropped += 1; + } + } + self.paced_drops += dropped; + dropped + } + + /// The present decision point — run on every loop pass (frame arrivals, vsync ticks, and the + /// 5 ms housekeeping wake all land here). Releases AT MOST one frame (the budget). Returns + /// `true` when a frame was released to glass this call. + #[allow(clippy::too_many_arguments)] // one call site; the seams are the point + pub(super) fn pump( + &mut self, + codec: &MediaCodec, + clock: Option<&VsyncShared>, + tracker: &DisplayTracker, + stats: &crate::stats::VideoStats, + now_mono_ns: i64, + ) -> bool { + // Budget bookkeeping first: reopen on the predicted latch, force-open on the backstop. + if let Some(f) = &self.inflight { + if now_mono_ns >= f.reopen_at_ns { + self.inflight = None; + } else if now_mono_ns - f.released_at_ns > STALE_REOPEN_NS { + self.forced += 1; + self.inflight = None; + } + } + // Pick the frame this pump may release. + let frame = if self.fifo_capacity == 0 { + self.frames.pop_back() // submit() kept it a single slot; back == the newest + } else { + // FIFO: drain exactly one frame per vsync tick, after preroll; a drain tick that + // finds the buffer dry re-arms preroll (the Apple `FrameStore` underflow semantics — + // the previous frame persists on glass, a repeat by omission, while headroom + // rebuilds). Everything is gated on the tick so an idle stream neither counts + // underflows nor churns the preroll flag 200×/s. + if !self.vsync_tick { + return false; + } + if !self.prerolled { + if self.frames.len() < self.fifo_capacity { + return false; + } + self.prerolled = true; + } + if self.frames.is_empty() { + self.prerolled = false; + self.dry += 1; + self.vsync_tick = false; // this tick's drain ran (and found nothing) + return false; + } + self.frames.pop_front() + }; + let Some(frame) = frame else { return false }; + if self.inflight.is_some() { + // Budget closed — park it back; a fresher submit replaces it (newest-wins), the next + // vsync tick / loop pass retries the pairing. + self.no_budget += 1; + match self.fifo_capacity { + 0 => self.frames.push_back(frame), + _ => self.frames.push_front(frame), + } + return false; + } + // Release: timeline-timed when the clock has one, ASAP otherwise. + let target = clock.and_then(|c| c.next_target(now_mono_ns, DEADLINE_MARGIN_NS)); + let released = match target { + Some(t) => codec + .release_output_buffer_at_time_by_index(frame.index, t.expected_present_ns) + .map_err(|e| log::warn!("presenter: release_at_time({}): {e}", frame.index)), + None => codec + .release_output_buffer_by_index(frame.index, true) + .map_err(|e| log::warn!("presenter: release({}): {e}", frame.index)), + }; + self.vsync_tick = false; + if released.is_err() { + return false; // the buffer is gone either way; nothing to book-keep + } + let period = clock.map(|c| c.period_ns()).filter(|&p| p > 0); + // Reopen at the target's DEADLINE, not its present time: SurfaceFlinger consumes the + // queued buffer at its latch (≈ the deadline) — that is when the queue slot frees and a + // new release can target the NEXT refresh. Reopening a period later (at present time) + // would cap the sustainable release rate at roughly half the panel rate. + let reopen_at_ns = target + .map(|t| t.deadline_ns) + .unwrap_or(now_mono_ns + period.unwrap_or(FALLBACK_PERIOD_NS)); + self.inflight = Some(InFlight { + reopen_at_ns, + released_at_ns: now_mono_ns, + }); + self.released += 1; + let release_real_ns = now_realtime_ns(); + let pace_us = ((release_real_ns - frame.decoded_ns).max(0) / 1000) as u64; + if self.pace_us.len() < 4096 { + self.pace_us.push(pace_us); + } + stats.note_release(pace_us); + tracker.note_rendered(frame.pts_us, frame.decoded_ns, release_real_ns); + true + } + + /// Release every held buffer unrendered — the teardown path, BEFORE `codec.stop()`. + pub(super) fn release_all(&mut self, codec: &MediaCodec) { + while let Some(f) = self.frames.pop_front() { + release_unrendered(codec, f.index); + } + self.inflight = None; + } + + /// The 1 Hz `pf-present` logcat mirror (target `pf.present`) — the Apple client's Console + /// `pf-present` line, so a HUD-off on-device A/B is readable wirelessly: + /// `released` (to glass) / `displays` (OnFrameRendered confirms) / `paced` (policy drops) / + /// `noBudget` (waits on the closed budget) / `forced` (stale force-opens — 0 when healthy) / + /// `qDry` (FIFO underflows) / `pace` (decoded→release) / `latch` (release→displayed) / + /// `vsync` (the measured panel period). + pub(super) fn flush_log(&mut self, meter: &PresentMeter, clock: Option<&VsyncShared>) { + if self.last_flush.elapsed() < std::time::Duration::from_secs(1) { + return; + } + self.last_flush = Instant::now(); + let (latch, displays) = meter.drain(); + if self.released == 0 && displays == 0 { + return; // idle stream — nothing worth a line + } + let (pace_p50, pace_max) = p50_max_ms(std::mem::take(&mut self.pace_us)); + let (latch_p50, latch_max) = p50_max_ms(latch); + let period_ms = clock.map(|c| c.period_ns() as f64 / 1e6).unwrap_or(0.0); + log::info!( + target: "pf.present", + "released={} displays={} paced={} noBudget={} forced={} qDry={} \ + paceMs p50={:.2} max={:.2} latchMs p50={:.2} max={:.2} vsyncMs={:.2}", + self.released, + displays, + self.paced_drops, + self.no_budget, + self.forced, + self.dry, + pace_p50, + pace_max, + latch_p50, + latch_max, + period_ms, + ); + self.released = 0; + self.paced_drops = 0; + self.no_budget = 0; + self.forced = 0; + self.dry = 0; + } +} + +fn release_unrendered(codec: &MediaCodec, index: usize) { + if let Err(e) = codec.release_output_buffer_by_index(index, false) { + log::warn!("presenter: release_output_buffer({index}, false): {e}"); + } +} + +/// `debug.punktfunk.presenter` sysprop: `arrival` = the legacy release-immediately path, +/// anything else / unset = the timeline presenter. The rebuild-free on-device A/B lever. +pub(super) fn presenter_disabled_by_sysprop() -> bool { + let mut buf = [0u8; 92]; // PROP_VALUE_MAX + // SAFETY: __system_property_get with a valid name + PROP_VALUE_MAX buffer is always safe. + let n = unsafe { + libc::__system_property_get( + c"debug.punktfunk.presenter".as_ptr(), + buf.as_mut_ptr().cast(), + ) + }; + n > 0 && &buf[..n as usize] == b"arrival" +} diff --git a/clients/android/native/src/decode/sync_loop.rs b/clients/android/native/src/decode/sync_loop.rs index d6bc130e..870239e2 100644 --- a/clients/android/native/src/decode/sync_loop.rs +++ b/clients/android/native/src/decode/sync_loop.rs @@ -44,6 +44,9 @@ pub(super) fn run_sync( ll_feature, low_latency_mode, is_tv, + // The timeline presenter lives in the async loop only; this loop IS the escape hatch. + present_priority: _, + smooth_buffer: _, } = opts; boost_thread_priority(); let mode = client.mode(); @@ -181,7 +184,11 @@ pub(super) fn run_sync( // render = true are parked in the tracker; the OnFrameRendered callback pairs them with // SurfaceFlinger's render timestamp. `render_cb` is the callback's leaked Arc refcount, // reclaimed after the codec is dropped below. - let tracker = DisplayTracker::new(stats.clone(), clock_offset.clone()); + let tracker = DisplayTracker::new( + stats.clone(), + clock_offset.clone(), + std::sync::Arc::new(super::presenter::PresentMeter::new()), + ); let render_cb = install_render_callback(&codec, &tracker); // Receipt timestamps keyed by the pts we queue into the codec, so the decoded point (output- // buffer dequeue — MediaCodec round-trips presentationTimeUs) can be paired back to its receipt @@ -598,7 +605,7 @@ fn drain( Ok(()) if held_present => { rendered = 1; if let Some((pts_us, decoded_ns)) = meta { - tracker.note_rendered(pts_us, decoded_ns); + tracker.note_rendered(pts_us, decoded_ns, super::latency::now_realtime_ns()); } } Ok(()) => discarded += 1, // held off the screen — awaiting a clean re-anchor diff --git a/clients/android/native/src/decode/vsync.rs b/clients/android/native/src/decode/vsync.rs new file mode 100644 index 00000000..23eb9b69 --- /dev/null +++ b/clients/android/native/src/decode/vsync.rs @@ -0,0 +1,357 @@ +//! The vsync clock behind the timeline presenter: an `AChoreographer` thread publishing the +//! panel's vsync grid + upcoming frame timelines, and pulsing the decode loop's event channel so +//! a frame parked on a closed glass budget gets its retry tick. +//! +//! On API 33+ the thread rides `AChoreographer_postVsyncCallback`, whose callback payload carries +//! the platform's FRAME TIMELINES — for each upcoming refresh, when SurfaceFlinger expects to +//! present and the deadline by which a frame must be submitted to make it. That pair is exactly +//! what `AMediaCodec_releaseOutputBufferAtTime` wants as its target. On 31/32 the older +//! `postFrameCallback64` supplies only the vsync instant; the presenter then releases ASAP +//! (identical to the legacy path) and uses the measured period purely to predict the latch for +//! its glass budget. +//! +//! Every `AChoreographer_*` symbol is dlsym-resolved from `libandroid.so` (mirrors +//! [`super::setup::try_set_frame_rate`]): several sit above the crate's API floor, and one hard +//! import of a too-new symbol fails `System.loadLibrary` on every older device. +//! +//! Started LAZILY on the first decoded frame (the Apple deadline presenter's bootstrap lesson: +//! an eagerly started clock ticks uselessly for the whole connect window), stopped + joined via +//! [`VsyncClock`]'s `Drop`. + +use std::ffi::c_void; +use std::sync::atomic::{AtomicBool, AtomicI64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +/// `CLOCK_MONOTONIC` now in nanoseconds — the clock AChoreographer stamps its timelines on and +/// the one `AMediaCodec_releaseOutputBufferAtTime` compares against (`System.nanoTime` basis). +/// Distinct from the stats path's `CLOCK_REALTIME`: presenter scheduling stays monotonic. +pub(super) fn now_monotonic_ns() -> i64 { + let mut ts = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + // SAFETY: `clock_gettime` with a valid out-pointer is an always-safe syscall. + unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) }; + // Explicit widening: timespec's fields are 32-bit on armv7 (time_t/c_long). + ts.tv_sec as i64 * 1_000_000_000 + ts.tv_nsec as i64 +} + +/// One upcoming frame timeline (API 33+ payload): when SurfaceFlinger expects to present the +/// frame, and the last instant it can be submitted to make that present. Monotonic ns. +#[derive(Clone, Copy)] +pub(super) struct FrameTimeline { + pub expected_present_ns: i64, + pub deadline_ns: i64, +} + +/// State the choreographer thread publishes and the decode loop reads. All monotonic ns. +pub(super) struct VsyncShared { + stop: AtomicBool, + /// The latest vsync callback's frame time (0 = no callback yet). + last_vsync_ns: AtomicI64, + /// Estimated vsync period (EMA over callback deltas / timeline spacing; 0 = unmeasured). + period_ns: AtomicI64, + /// The latest callback's upcoming timelines, soonest first. Empty on the 31/32 fallback. + timelines: Mutex>, +} + +impl VsyncShared { + /// The measured vsync period, or 0 while unmeasured. + pub(super) fn period_ns(&self) -> i64 { + self.period_ns.load(Ordering::Relaxed) + } + + /// The release target for a frame submitted at `now`: the earliest stored timeline whose + /// deadline is still `margin` away, extrapolated forward by whole periods once the stored + /// set has aged out (timelines refresh once per vsync callback; a frame can decode anywhere + /// inside that window). `None` on the 31/32 fallback — the caller releases ASAP. + pub(super) fn next_target(&self, now_ns: i64, margin_ns: i64) -> Option { + let g = self + .timelines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for t in g.iter() { + if t.deadline_ns > now_ns + margin_ns { + return Some(*t); + } + } + let last = g.last().copied()?; + let period = self.period_ns(); + if period <= 0 { + return None; + } + // All stored timelines have passed — step the last one forward whole periods until its + // deadline clears `now + margin` again. + let behind = (now_ns + margin_ns).saturating_sub(last.deadline_ns); + let k = behind / period + 1; + Some(FrameTimeline { + expected_present_ns: last.expected_present_ns + k * period, + deadline_ns: last.deadline_ns + k * period, + }) + } +} + +// ---- dlsym'd AChoreographer surface ---- + +type PostFrameCallback64 = + unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(i64, *mut c_void), *mut c_void); +type PostVsyncCallback = unsafe extern "C" fn( + *mut c_void, + unsafe extern "C" fn(*const c_void, *mut c_void), + *mut c_void, +); + +struct ChoreoApi { + get_instance: unsafe extern "C" fn() -> *mut c_void, + /// API 33: vsync callback with frame-timeline payload. Preferred. + post_vsync: Option, + /// API 29 fallback: frame callback with only the vsync instant. + post_frame64: Option, + // AChoreographerFrameCallbackData accessors (API 33; present iff `post_vsync` is). + fcd_frame_time: Option i64>, + fcd_timelines_len: Option usize>, + fcd_preferred_index: Option usize>, + fcd_expected_present: Option i64>, + fcd_deadline: Option i64>, +} + +impl ChoreoApi { + /// Resolve from `libandroid.so`. `None` when even the baseline symbols are missing. + fn resolve() -> Option { + // SAFETY: dlopen of the always-mapped libandroid.so (refcount bump, never closed); each + // dlsym is null-checked before the transmute to its fn-pointer type. + unsafe { + let lib = libc::dlopen(c"libandroid.so".as_ptr(), libc::RTLD_NOW); + if lib.is_null() { + return None; + } + let sym = |name: &std::ffi::CStr| { + let p = libc::dlsym(lib, name.as_ptr()); + (!p.is_null()).then_some(p) + }; + let get_instance = sym(c"AChoreographer_getInstance")?; + let post_vsync = sym(c"AChoreographer_postVsyncCallback"); + let post_frame64 = sym(c"AChoreographer_postFrameCallback64"); + post_vsync.or(post_frame64)?; // neither post entry point — no clock on this device + Some(ChoreoApi { + get_instance: std::mem::transmute::< + *mut c_void, + unsafe extern "C" fn() -> *mut c_void, + >(get_instance), + post_vsync: post_vsync.map(|p| std::mem::transmute::<*mut c_void, PostVsyncCallback>(p)), + post_frame64: post_frame64 + .map(|p| std::mem::transmute::<*mut c_void, PostFrameCallback64>(p)), + fcd_frame_time: sym(c"AChoreographerFrameCallbackData_getFrameTimeNanos").map(|p| { + std::mem::transmute::<*mut c_void, unsafe extern "C" fn(*const c_void) -> i64>(p) + }), + fcd_timelines_len: sym(c"AChoreographerFrameCallbackData_getFrameTimelinesLength") + .map(|p| { + std::mem::transmute::<*mut c_void, unsafe extern "C" fn(*const c_void) -> usize>( + p, + ) + }), + fcd_preferred_index: sym( + c"AChoreographerFrameCallbackData_getPreferredFrameTimelineIndex", + ) + .map(|p| { + std::mem::transmute::<*mut c_void, unsafe extern "C" fn(*const c_void) -> usize>(p) + }), + fcd_expected_present: sym( + c"AChoreographerFrameCallbackData_getFrameTimelineExpectedPresentationTimeNanos", + ) + .map(|p| { + std::mem::transmute::< + *mut c_void, + unsafe extern "C" fn(*const c_void, usize) -> i64, + >(p) + }), + fcd_deadline: sym(c"AChoreographerFrameCallbackData_getFrameTimelineDeadlineNanos") + .map(|p| { + std::mem::transmute::< + *mut c_void, + unsafe extern "C" fn(*const c_void, usize) -> i64, + >(p) + }), + }) + } + } +} + +/// Everything a callback invocation needs. Owned by the choreographer thread's stack; callbacks +/// only ever fire inside that thread's looper poll, so the borrow can't outlive the thread. +struct CallbackCtx { + api: ChoreoApi, + choreographer: *mut c_void, + shared: Arc, + on_tick: Box, +} + +impl CallbackCtx { + /// Common tail of both callback flavours: update the grid estimate, publish, pulse, re-arm. + fn tick(&self, frame_time_ns: i64, timelines: Vec) { + let prev = self + .shared + .last_vsync_ns + .swap(frame_time_ns, Ordering::Relaxed); + // Period: prefer timeline spacing (exact, straight from the platform), else the delta of + // successive callbacks (jittery — EMA'd), clamped to sane panel rates (24..500 Hz). + let mut period = 0i64; + if timelines.len() >= 2 { + period = timelines[1].expected_present_ns - timelines[0].expected_present_ns; + } else if prev > 0 { + period = frame_time_ns - prev; + } + if (2_000_000..=42_000_000).contains(&period) { + let old = self.shared.period_ns.load(Ordering::Relaxed); + let smoothed = if old > 0 { + (old * 7 + period) / 8 + } else { + period + }; + self.shared.period_ns.store(smoothed, Ordering::Relaxed); + } + if !timelines.is_empty() { + let mut g = self + .shared + .timelines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *g = timelines; + } + (self.on_tick)(); + if !self.shared.stop.load(Ordering::Relaxed) { + self.repost(); + } + } + + fn repost(&self) { + // SAFETY: `choreographer` is this thread's instance; the ctx pointer stays valid for the + // thread's life and callbacks only fire on this thread (see the struct doc). + unsafe { + let ud = self as *const CallbackCtx as *mut c_void; + if let Some(post) = self.api.post_vsync { + post(self.choreographer, on_vsync, ud); + } else if let Some(post) = self.api.post_frame64 { + post(self.choreographer, on_frame64, ud); + } + } + } +} + +/// API 33+ trampoline: harvest the frame timelines, then the common tick. Panic-free (an unwind +/// out of an `extern "C"` fn aborts). +unsafe extern "C" fn on_vsync(data: *const c_void, ud: *mut c_void) { + // SAFETY: `ud` is the thread's `CallbackCtx`, alive for the whole poll loop (see struct doc). + let ctx = unsafe { &*(ud as *const CallbackCtx) }; + let api = &ctx.api; + let (mut frame_time, mut timelines) = (now_monotonic_ns(), Vec::new()); + // SAFETY: `data` is the platform's callback payload, valid for this invocation; the accessors + // were resolved together with `post_vsync` (same API level) and are only called when present. + unsafe { + if let Some(f) = api.fcd_frame_time { + frame_time = f(data); + } + if let (Some(len_f), Some(pref_f), Some(exp_f), Some(dl_f)) = ( + api.fcd_timelines_len, + api.fcd_preferred_index, + api.fcd_expected_present, + api.fcd_deadline, + ) { + let len = len_f(data).min(8); + // From the PREFERRED index on: earlier timelines are ones the platform already + // considers missed for a frame starting now. + let start = pref_f(data).min(len); + timelines = (start..len) + .map(|i| FrameTimeline { + expected_present_ns: exp_f(data, i), + deadline_ns: dl_f(data, i), + }) + .collect(); + } + } + ctx.tick(frame_time, timelines); +} + +/// API 29 fallback trampoline: vsync instant only. +unsafe extern "C" fn on_frame64(frame_time_ns: i64, ud: *mut c_void) { + // SAFETY: `ud` is the thread's `CallbackCtx` (see `on_vsync`). + let ctx = unsafe { &*(ud as *const CallbackCtx) }; + ctx.tick(frame_time_ns, Vec::new()); +} + +/// The clock: a dedicated looper thread the choreographer calls back on. Dropping stops + joins. +pub(super) struct VsyncClock { + shared: Arc, + join: Option>, +} + +impl VsyncClock { + /// Spawn the choreographer thread. `on_tick` fires once per vsync ON THAT THREAD — it must + /// only do something cheap and `Send` (the decode loop passes an event-channel send). + /// `None` when the platform surface is missing (very old device) — the presenter then runs + /// clock-less (ASAP targets, predicted-latch budget). + pub(super) fn start(on_tick: Box) -> Option { + let api = ChoreoApi::resolve()?; + let timelines_live = api.post_vsync.is_some(); + let shared = Arc::new(VsyncShared { + stop: AtomicBool::new(false), + last_vsync_ns: AtomicI64::new(0), + period_ns: AtomicI64::new(0), + timelines: Mutex::new(Vec::new()), + }); + let thread_shared = shared.clone(); + let join = std::thread::Builder::new() + .name("pf-vsync".into()) + .spawn(move || { + let looper = ndk::looper::ThreadLooper::prepare(); + // SAFETY: getInstance on a thread with a prepared looper returns this thread's + // choreographer (never null once a looper exists). + let choreographer = unsafe { (api.get_instance)() }; + if choreographer.is_null() { + log::warn!("vsync: AChoreographer_getInstance returned null — no clock"); + return; + } + let ctx = CallbackCtx { + api, + choreographer, + shared: thread_shared, + on_tick, + }; + ctx.repost(); + // The bounded poll doubles as the stop check: no cross-thread wake needed, worst + // case teardown waits one timeout out. Callbacks fire inside poll_once_timeout. + while !ctx.shared.stop.load(Ordering::Relaxed) { + let _ = looper.poll_once_timeout(Duration::from_millis(250)); + } + // `ctx` drops here — after the loop, so no queued callback can outlive it (they + // only ever fire inside this thread's poll). + }) + .ok()?; + log::info!( + "vsync: choreographer clock started ({})", + if timelines_live { + "frame timelines" + } else { + "frame callback fallback" + } + ); + Some(VsyncClock { + shared, + join: Some(join), + }) + } + + pub(super) fn shared(&self) -> &Arc { + &self.shared + } +} + +impl Drop for VsyncClock { + fn drop(&mut self) { + self.shared.stop.store(true, Ordering::Relaxed); + if let Some(j) = self.join.take() { + let _ = j.join(); + } + } +} diff --git a/clients/android/native/src/session/planes.rs b/clients/android/native/src/session/planes.rs index 84a71ce6..23d486ce 100644 --- a/clients/android/native/src/session/planes.rs +++ b/clients/android/native/src/session/planes.rs @@ -10,11 +10,13 @@ use jni::JNIEnv; use super::{jni_guard, SessionHandle}; -/// `NativeBridge.nativeStartVideo(handle, surface, decoderName, lowLatencyMode, lowLatencyFeature)` -/// — wrap the SurfaceView's `Surface` as an `ANativeWindow` and start the decode thread rendering -/// onto it. `decoderName` is the codec Kotlin ranked from `MediaCodecList` (`""` = let the platform -/// resolve the default for the MIME); `lowLatencyMode` is the user's master toggle; -/// `lowLatencyFeature` is whether that decoder advertised `FEATURE_LowLatency` (HUD label only). +/// `NativeBridge.nativeStartVideo(handle, surface, decoderName, lowLatencyMode, lowLatencyFeature, +/// isTv, presentPriority, smoothBuffer)` — wrap the SurfaceView's `Surface` as an `ANativeWindow` +/// and start the decode thread rendering onto it. `decoderName` is the codec Kotlin ranked from +/// `MediaCodecList` (`""` = let the platform resolve the default for the MIME); `lowLatencyMode` +/// is the user's master toggle; `lowLatencyFeature` is whether that decoder advertised +/// `FEATURE_LowLatency` (HUD label only); `presentPriority`/`smoothBuffer` are the timeline +/// presenter's intent (0 = lowest latency / 1 = smoothness; buffer 0 = auto, 1..=3 frames). /// No-op if already started. #[cfg(target_os = "android")] #[no_mangle] @@ -27,6 +29,8 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartVideo( low_latency_mode: jboolean, ll_feature: jboolean, is_tv: jboolean, + present_priority: jni::sys::jint, + smooth_buffer: jni::sys::jint, ) { use super::VideoThread; use std::sync::atomic::AtomicBool; @@ -70,6 +74,8 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartVideo( ll_feature: ll_feature != 0, low_latency_mode: low_latency_mode != 0, is_tv: is_tv != 0, + present_priority, + smooth_buffer, }; let join = std::thread::Builder::new() .name("pf-decode".into()) @@ -173,7 +179,7 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopVideo( /// `[fps, mbps, e2eP50Ms, e2eP95Ms, latValid, skewCorrected, width, height, refreshHz, framesLost, /// bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc, hostNetP50Ms, decodeP50Ms, hostP50Ms, /// netP50Ms, lostWindow, skippedWindow, fecWindow, framesWindow, dispValid, displayP50Ms, -/// e2eDispP50Ms, e2eDispP95Ms]` +/// e2eDispP50Ms, e2eDispP95Ms, paceP50Ms, latchP50Ms, presentsWindow, presenterActive]` /// (the flags are 1.0/0.0; indexes 0–21 match the previous 22-double layout — 0–13 the original /// 14-double one with the latency pair re-based to the end-to-end capture→decoded headline, 14/15 /// the stage p50s tiling it: `host+network` = capture→received, `decode` = received→decoded; 16/17 @@ -186,7 +192,11 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopVideo( /// OnFrameRendered render timestamps — when `dispValid` is 1.0 the HUD headline becomes the /// directly-measured capture→displayed pair at 24/25 with `display` = decoded→displayed p50 at 23 /// closing the equation, and when 0.0 — no render callback landed this window — it falls back to -/// the capture→decoded headline at 2/3), or `null` when no decode thread is running. +/// the capture→decoded headline at 2/3; 26–29 are the timeline presenter's split of the `display` +/// term — `pace` = decoded→release (store + glass budget) p50 at 26, `latch` = +/// release→displayed (SurfaceFlinger) p50 at 27, the window's on-glass confirm count at 28 +/// (`presents` vs `fps` is the presenter-health pair), and 29 = 1.0 while the timeline presenter +/// is active this session), or `null` when no decode thread is running. /// Poll ~1 Hz from the UI; each call /// resets the measurement window. Not android-gated — pure `jni` + connector reads, so it links on /// the host build too (Kotlin only ever calls it on device). @@ -210,7 +220,7 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats( .drain(h.client.frames_dropped(), h.client.fec_recovered_shards()); let mode = h.client.mode(); let color = h.client.color; - let buf: [f64; 26] = [ + let buf: [f64; 30] = [ snap.fps, snap.mbps, snap.e2e_p50_ms, @@ -251,6 +261,13 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats( snap.display_p50_ms, snap.e2e_disp_p50_ms, snap.e2e_disp_p95_ms, + // Timeline-presenter split of the `display` term (pace = decoded→release, latch = + // release→displayed), the window's on-glass confirm count, and whether the presenter + // is active at all (0.0 = legacy release-immediately path — split reads 0 too). + snap.pace_p50_ms, + snap.latch_p50_ms, + snap.presents as f64, + if h.stats.presenter_active() { 1.0 } else { 0.0 }, ]; let arr = match env.new_double_array(buf.len() as jsize) { Ok(a) => a, diff --git a/clients/android/native/src/stats.rs b/clients/android/native/src/stats.rs index 7489a8dd..f89a9ba0 100644 --- a/clients/android/native/src/stats.rs +++ b/clients/android/native/src/stats.rs @@ -28,6 +28,9 @@ pub struct VideoStats { /// they (and the caller's latency computation — see `enabled`) early-out on this flag alone. /// Off until Kotlin shows the HUD. enabled: AtomicBool, + /// Whether the timeline presenter is active this session (async loop, not sysprop-disabled) — + /// stats index 29, so the HUD can label the display split it is (or isn't) getting. + presenter_active: AtomicBool, /// The resolved decoder identity for the HUD: the codec's actual `AMediaCodec` name (e.g. /// `c2.qti.avc.decoder`) and whether it advertised `FEATURE_LowLatency`. Set once when the /// decode thread creates the codec (`set_decoder`), read one-shot by `nativeVideoDecoderLabel`. @@ -67,6 +70,16 @@ struct Inner { /// `end-to-end` = capture→displayed samples, µs (skew-corrected) — the spec's headline, /// measured directly (not summed from stages). Empty under the same fallback as `display_us`. e2e_disp_us: Vec, + /// The `display` stage's presenter split, decoded→release (pace wait: store + glass budget), + /// µs. Empty on the legacy release-immediately path. + pace_us: Vec, + /// The other half of the split, release→displayed (SurfaceFlinger's latch + scanout), µs — + /// from the `OnFrameRendered` render timestamps. `pace + latch ≈ display` per frame. + latch_us: Vec, + /// Frames confirmed on glass this window (`OnFrameRendered` callbacks) — the `presents`-vs- + /// `fps` health pair: presents ≪ fps means the presenter is dropping/serializing; an fps + /// deficit is upstream. + presents: u64, /// Client-side newest-wins/pacing drops this window (decoded frames released without /// rendering, or parked AUs dropped on overflow) — the spec's `skipped` counter. skipped: u64, @@ -101,6 +114,13 @@ pub struct Snapshot { /// Whether any capture→displayed sample landed this window — gates the HUD's headline endpoint /// (`capture→displayed` vs the capture→decoded fallback) and the equation's `display` term. pub disp_valid: bool, + /// The `display` stage's presenter split p50s (ms): `pace` = decoded→release (store + glass + /// budget), `latch` = release→displayed (SurfaceFlinger). 0.0 when no sample landed (legacy + /// path / no render callbacks). + pub pace_p50_ms: f64, + pub latch_p50_ms: f64, + /// Frames confirmed on glass this window (`OnFrameRendered` callbacks). + pub presents: u64, /// Phase-2 `host` / `network` split p50s (ms) — 0.0 when no 0xCF timing matched this window /// (old host / no samples yet), in which case the HUD keeps the combined `host+network` term. pub host_p50_ms: f64, @@ -132,6 +152,7 @@ impl VideoStats { pub fn new() -> VideoStats { VideoStats { enabled: AtomicBool::new(false), + presenter_active: AtomicBool::new(false), decoder: Mutex::new(None), inner: Mutex::new(Inner { window_start: Instant::now(), @@ -144,6 +165,9 @@ impl VideoStats { decode_us: Vec::with_capacity(256), display_us: Vec::with_capacity(256), e2e_disp_us: Vec::with_capacity(256), + pace_us: Vec::with_capacity(256), + latch_us: Vec::with_capacity(256), + presents: 0, skipped: 0, last_dropped_total: 0, last_fec_total: 0, @@ -160,6 +184,18 @@ impl VideoStats { self.enabled.load(Ordering::Relaxed) } + /// Record whether the timeline presenter runs this session (decode thread, once at start). + // Set only by the android-only decode thread; unreferenced on the host build — expected. + #[cfg_attr(not(target_os = "android"), allow(dead_code))] + pub fn set_presenter_active(&self, on: bool) { + self.presenter_active.store(on, Ordering::Relaxed); + } + + /// Whether the timeline presenter is active (stats index 29). + pub fn presenter_active(&self) -> bool { + self.presenter_active.load(Ordering::Relaxed) + } + /// Toggle sampling. Enabling resets the window, so the first HUD poll after a show never mixes /// in counters (or a window start) from before the overlay was visible. `dropped_total` / /// `fec_total` are the connector's session-cumulative counters at this instant — they seed the @@ -181,6 +217,9 @@ impl VideoStats { g.decode_us.clear(); g.display_us.clear(); g.e2e_disp_us.clear(); + g.pace_us.clear(); + g.latch_us.clear(); + g.presents = 0; g.skipped = 0; g.last_dropped_total = dropped_total; g.last_fec_total = fec_total; @@ -298,13 +337,19 @@ impl VideoStats { } /// Record one displayed frame (the `OnFrameRendered` render timestamp, re-based to the - /// realtime clock): its capture→displayed `end-to-end` sample and its decoded→displayed - /// `display` stage sample (either may be absent — the e2e clamp rejected an out-of-range - /// value, or the decoded stamp for this pts was already evicted/pre-HUD). Fired from the - /// codec's render-callback thread, not the decode thread — the lock makes that safe. + /// realtime clock): its capture→displayed `end-to-end` sample, its decoded→displayed + /// `display` stage sample, and the presenter split's `latch` half (release→displayed) — any + /// may be absent (the e2e clamp rejected an out-of-range value, or the release record for + /// this pts was already evicted). Fired from the codec's render-callback thread, not the + /// decode thread — the lock makes that safe. // Driven only by the android-only decode path; unreferenced on the host build — expected. #[cfg_attr(not(target_os = "android"), allow(dead_code))] - pub fn note_displayed(&self, e2e_us: Option, display_us: Option) { + pub fn note_displayed( + &self, + e2e_us: Option, + display_us: Option, + latch_us: Option, + ) { if !self.enabled.load(Ordering::Relaxed) { return; // HUD hidden — skip the lock (the callback already skipped the clock reads) } @@ -313,12 +358,32 @@ impl VideoStats { .inner .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); + g.presents += 1; if let Some(l) = e2e_us { g.e2e_disp_us.push(l); } if let Some(l) = display_us { g.display_us.push(l); } + if let Some(l) = latch_us { + g.latch_us.push(l); + } + } + + /// Record one released frame's pace-wait (decoded→release: the presenter's store + glass + /// budget), µs — the `display` stage's other half. Decode-thread only. + // Driven only by the android-only decode thread; unreferenced on the host build — expected. + #[cfg_attr(not(target_os = "android"), allow(dead_code))] + pub fn note_release(&self, pace_us: u64) { + if !self.enabled.load(Ordering::Relaxed) { + return; // HUD hidden — skip the lock + } + // Poison-proof for the same reason as `note_received`. + let mut g = self + .inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + g.pace_us.push(pace_us); } /// Compute the window's rates + latency percentiles, then reset for the next window. @@ -341,6 +406,8 @@ impl VideoStats { g.decode_us.sort_unstable(); g.display_us.sort_unstable(); g.e2e_disp_us.sort_unstable(); + g.pace_us.sort_unstable(); + g.latch_us.sort_unstable(); let snap = Snapshot { fps, mbps, @@ -352,6 +419,9 @@ impl VideoStats { decode_p50_ms: pctl_ms(&g.decode_us, 0.50), display_p50_ms: pctl_ms(&g.display_us, 0.50), disp_valid: !g.e2e_disp_us.is_empty(), + pace_p50_ms: pctl_ms(&g.pace_us, 0.50), + latch_p50_ms: pctl_ms(&g.latch_us, 0.50), + presents: g.presents, host_p50_ms: pctl_ms(&g.host_us, 0.50), net_p50_ms: pctl_ms(&g.net_us, 0.50), lat_valid: !g.e2e_us.is_empty(), @@ -371,6 +441,9 @@ impl VideoStats { g.decode_us.clear(); g.display_us.clear(); g.e2e_disp_us.clear(); + g.pace_us.clear(); + g.latch_us.clear(); + g.presents = 0; g.skipped = 0; g.last_dropped_total = dropped_total; g.last_fec_total = fec_total;