feat(android): tiered stats HUD + windowed lost/skipped/FEC counters

Stats overlay verbosity tiers (Off/Compact/Normal/Detailed, 3-finger tap
cycles live, old Boolean pref migrated) and the unified-spec line-4
reliability counters: lost/FEC windowed from the connector's cumulative
totals, skipped from the client's own newest-wins drops. Adds the
fec_recovered_shards accessor to NativeClient, mirrored from the
data-plane pump like frames_dropped.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-08 12:29:14 +02:00
parent b12c7d3deb
commit 4c9c7e606e
12 changed files with 343 additions and 113 deletions
@@ -346,11 +346,12 @@ private fun buildSettingsRows(s: Settings, update: (Settings) -> Unit): List<GpR
GAMEPAD_OPTIONS.mapIndexed { i, lbl -> i to lbl }, s.gamepad,
) { update(s.copy(gamepad = it)) },
toggle(
choice(
"hud", "Interface", "Statistics overlay",
"Show FPS, throughput and latency while streaming.",
s.statsHudEnabled,
) { update(s.copy(statsHudEnabled = it)) },
"How much the overlay shows: Compact (one line) → Normal → Detailed (full HUD). " +
"A 3-finger tap cycles the tiers live.",
STATS_VERBOSITY_OPTIONS, s.statsVerbosity,
) { update(s.copy(statsVerbosity = it)) },
toggle(
"library", null, "Game library",
"Browse a paired host's games with Y (experimental).",
@@ -30,8 +30,13 @@ data class Settings(
* host emits it when it can, else falls back. AMediaCodec decodes whichever the host resolves. */
val codec: String = "auto",
val micEnabled: Boolean = false,
/** Show the live stats overlay (FPS / throughput / latency) during a stream. */
val statsHudEnabled: Boolean = true,
/**
* How much the in-stream stats overlay shows — see [StatsVerbosity]. Defaults to
* [StatsVerbosity.NORMAL] (the res/fps line + latency headline + reliability counters); the full
* decoder/feed/equation HUD is [StatsVerbosity.DETAILED], and a single terse line is
* [StatsVerbosity.COMPACT]. A 3-finger tap cycles through the tiers live.
*/
val statsVerbosity: StatsVerbosity = StatsVerbosity.NORMAL,
/**
* Touch input model — how touchscreen fingers drive the host. [TouchMode.TRACKPAD] (default):
* the cursor stays put on touch-down and moves by the finger's relative delta (swipe to nudge,
@@ -81,6 +86,27 @@ data class Settings(
/** [Settings.touchMode] values; persisted by name. */
enum class TouchMode { TRACKPAD, POINTER, TOUCH }
/**
* Stats-overlay detail tiers, in cycling order (persisted by name). Each tier is a strict superset
* of the previous one, so toning down never hides a number a lower tier keeps:
* - [OFF] — no overlay (and native sampling is gated off, one atomic load per frame).
* - [COMPACT] — one line: `fps · end-to-end ms · Mb/s` (+ a loss flag when frames drop).
* - [NORMAL] — adds the resolution/refresh line, the end-to-end p50/p95 headline, and the
* reliability counters (lost / skipped / FEC) when nonzero. The default.
* - [DETAILED] — the full HUD: also the decoder label, the video-feed descriptor, and the
* `host+network + decode` stage equation.
* A 3-finger tap in-stream cycles Off → Compact → Normal → Detailed → Off (see [next]).
*/
enum class StatsVerbosity(val label: String) {
OFF("Off"),
COMPACT("Compact"),
NORMAL("Normal"),
DETAILED("Detailed");
/** The next tier for the live 3-finger-tap cycle (wraps Detailed → Off). */
fun next(): StatsVerbosity = entries[(ordinal + 1) % entries.size]
}
/** Loads/saves [Settings] in the app-private `punktfunk_settings` prefs. */
class SettingsStore(context: Context) {
private val prefs =
@@ -97,7 +123,16 @@ class SettingsStore(context: Context) {
audioChannels = prefs.getInt(K_AUDIO_CH, 2),
codec = prefs.getString(K_CODEC, "auto") ?: "auto",
micEnabled = prefs.getBoolean(K_MIC, false),
statsHudEnabled = prefs.getBoolean(K_HUD, true),
statsVerbosity = prefs.getString(K_STATS_VERBOSITY, null)
?.let { name -> StatsVerbosity.entries.firstOrNull { it.name == name } }
// Migration from the pre-tier Boolean "stats_hud_enabled": an explicit OFF stays off;
// everyone else (incl. fresh installs) lands on NORMAL — the old always-full HUD toned
// down to the new default, which is the whole point of adding tiers.
?: if (prefs.contains(K_HUD) && !prefs.getBoolean(K_HUD, true)) {
StatsVerbosity.OFF
} else {
StatsVerbosity.NORMAL
},
touchMode = prefs.getString(K_TOUCH_MODE, null)
?.let { name -> TouchMode.entries.firstOrNull { it.name == name } }
// Migration: the pre-enum Boolean "trackpad_mode" (true = trackpad, false = direct).
@@ -120,7 +155,7 @@ class SettingsStore(context: Context) {
.putInt(K_AUDIO_CH, s.audioChannels)
.putString(K_CODEC, s.codec)
.putBoolean(K_MIC, s.micEnabled)
.putBoolean(K_HUD, s.statsHudEnabled)
.putString(K_STATS_VERBOSITY, s.statsVerbosity.name)
.putString(K_TOUCH_MODE, s.touchMode.name)
.putBoolean(K_GAMEPAD_UI, s.gamepadUiEnabled)
.putBoolean(K_LIBRARY, s.libraryEnabled)
@@ -140,6 +175,10 @@ class SettingsStore(context: Context) {
const val K_AUDIO_CH = "audio_channels"
const val K_CODEC = "codec"
const val K_MIC = "mic_enabled"
const val K_STATS_VERBOSITY = "stats_verbosity"
/** Pre-tier Boolean the [K_STATS_VERBOSITY] enum replaced — read once for migration, never
* written. */
const val K_HUD = "stats_hud_enabled"
const val K_TOUCH_MODE = "touch_mode"
const val K_GAMEPAD_UI = "gamepad_ui_enabled"
@@ -269,6 +308,9 @@ val COMPOSITOR_OPTIONS = listOf(
"gamescope",
)
/** (verbosity, label) for the stats-overlay detail picker. Order = the live 3-finger-tap cycle. */
val STATS_VERBOSITY_OPTIONS = StatsVerbosity.entries.map { it to it.label }
/** (mode, label) for the touch-input model. */
val TOUCH_MODE_OPTIONS = listOf(
TouchMode.TRACKPAD to "Trackpad",
@@ -403,11 +403,17 @@ private fun InterfaceSettings(s: Settings, update: (Settings) -> Unit) {
checked = s.autoWakeEnabled,
onCheckedChange = { on -> update(s.copy(autoWakeEnabled = on)) },
)
ToggleRow(
title = "Stats overlay",
subtitle = "Show FPS, throughput and latency while streaming (3-finger tap toggles it live)",
checked = s.statsHudEnabled,
onCheckedChange = { on -> update(s.copy(statsHudEnabled = on)) },
SettingDropdown(
label = "Stats overlay",
options = STATS_VERBOSITY_OPTIONS,
selected = s.statsVerbosity,
) { v -> update(s.copy(statsVerbosity = v)) }
Text(
"How much the in-stream overlay shows: Compact is a single fps · latency · bitrate " +
"line; Normal adds the resolution and reliability lines; Detailed adds the decoder, " +
"colour and latency-breakdown lines. A 3-finger tap cycles the tiers live.",
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
@@ -16,61 +16,63 @@ import kotlin.math.roundToInt
/**
* The live stats overlay — the unified HUD (`design/stats-unification.md`, Android v1: headline is
* `capture→decoded`, tiled by `host+network` + `decode`). Reads the 18-double layout from
* `capture→decoded`, tiled by `host+network` + `decode`). Reads the 22-double layout from
* [NativeBridge.nativeVideoStats]:
* `[fps, mbps, e2eP50Ms, e2eP95Ms, latValid, skew, w, h, hz, lost, bitDepth, colorPrimaries,
* colorTransfer, chromaFormatIdc, hostNetP50Ms, decodeP50Ms, hostP50Ms, netP50Ms]`. Indexes 1013
* (present on a current native lib) describe the negotiated video feed and render as a
* codec/depth/colour/chroma line; 14/15 render as the stage equation — split into
* `host + network + decode` when the Phase-2 terms at 16/17 are nonzero (a current host sends
* per-AU 0xCF timings; an old host leaves them 0 and the combined `host+network` term stands);
* older layouts just omit those lines.
* `[fps, mbps, e2eP50Ms, e2eP95Ms, latValid, skew, w, h, hz, lostTotal, bitDepth, colorPrimaries,
* colorTransfer, chromaFormatIdc, hostNetP50Ms, decodeP50Ms, hostP50Ms, netP50Ms, lost, skipped,
* fec, frames]`.
*
* [verbosity] selects how many lines render (each tier a superset of the last — see
* [StatsVerbosity]):
* - [StatsVerbosity.COMPACT] — one line, `fps · end-to-end ms · Mb/s` (+ a loss flag).
* - [StatsVerbosity.NORMAL] — the res/fps/Mb·s line, the end-to-end p50/p95 headline, and the
* reliability counters (1821) when nonzero.
* - [StatsVerbosity.DETAILED] — also the decoder label, the video-feed descriptor (1013), and the
* stage equation (14/15, split into `host + network` when the Phase-2 terms at 16/17 are nonzero).
* [StatsVerbosity.OFF] renders nothing. Older native layouts simply omit the lines they lack (the
* counter line falls back to the cumulative `lostTotal` at index 9 on a pre-window lib).
*/
@Composable
internal fun StatsOverlay(s: DoubleArray, decoderLabel: String = "", modifier: Modifier = Modifier) {
if (s.size < 10) return
internal fun StatsOverlay(
s: DoubleArray,
verbosity: StatsVerbosity,
decoderLabel: String = "",
modifier: Modifier = Modifier,
) {
if (verbosity == StatsVerbosity.OFF || s.size < 10) return
val w = s[6].toInt()
val h = s[7].toInt()
val hz = s[8].toInt()
val latValid = s[4] != 0.0
val skew = s[5] != 0.0
val lost = s[9].toLong()
val detailed = verbosity == StatsVerbosity.DETAILED
Column(
modifier = modifier
.background(Color.Black.copy(alpha = 0.45f), RoundedCornerShape(6.dp))
.padding(horizontal = 8.dp, vertical = 4.dp),
) {
Text(
"$w×$h@$hz ${s[0].roundToInt()} fps ${"%.1f".format(s[1])} Mb/s",
color = Color.White,
fontFamily = FontFamily.Monospace,
fontSize = 12.sp,
)
if (decoderLabel.isNotEmpty()) {
Text(
decoderLabel,
color = Color(0xFFB0D0FF),
fontFamily = FontFamily.Monospace,
fontSize = 12.sp,
)
// Compact: everything the glance-value needs on one line, nothing else.
if (verbosity == StatsVerbosity.COMPACT) {
statLine(compactLine(s, latValid), Color.White)
return@Column
}
videoFeedLine(s)?.let { feed ->
Text(
feed,
color = Color.White,
fontFamily = FontFamily.Monospace,
fontSize = 12.sp,
)
statLine("$w×$h@$hz ${s[0].roundToInt()} fps ${"%.1f".format(s[1])} Mb/s", Color.White)
if (detailed && decoderLabel.isNotEmpty()) {
statLine(decoderLabel, Color(0xFFB0D0FF))
}
if (detailed) {
videoFeedLine(s)?.let { statLine(it, Color.White) }
}
if (latValid) {
val tag = if (skew) "" else " (same-host clock)"
Text(
statLine(
"end-to-end ${"%.1f".format(s[2])} ms p50 · ${"%.1f".format(s[3])} p95 · capture→decoded$tag",
color = Color.White,
fontFamily = FontFamily.Monospace,
fontSize = 12.sp,
Color.White,
)
if (s.size >= 16) {
if (detailed && s.size >= 16) {
// Phase-2 split (s[16]/s[17]): render `host + network` separately when the host
// reported its share this window; otherwise the combined term (old host / no
// matched 0xCF timing).
@@ -79,25 +81,60 @@ internal fun StatsOverlay(s: DoubleArray, decoderLabel: String = "", modifier: M
} else {
"= host+network ${"%.1f".format(s[14])} + decode ${"%.1f".format(s[15])}"
}
Text(
equation,
color = Color.White,
fontFamily = FontFamily.Monospace,
fontSize = 12.sp,
)
statLine(equation, Color.White)
}
}
if (lost > 0) {
Text(
"lost $lost",
color = Color(0xFFFFB0B0),
fontFamily = FontFamily.Monospace,
fontSize = 12.sp,
)
}
counterLine(s, lost)?.let { statLine(it, Color(0xFFFFB0B0)) }
}
}
/** One monospace HUD line — the shared type ramp so every tier's rows line up. */
@Composable
private fun statLine(text: String, color: Color) {
Text(text, color = color, fontFamily = FontFamily.Monospace, fontSize = 12.sp)
}
/**
* The single [StatsVerbosity.COMPACT] line: `238 fps · 1.3 ms · 921 Mb/s`. The end-to-end p50 term
* is dropped when no in-range latency sample landed (`latValid` false), and a loss flag
* `· ⚠ lost {n}` is appended when the window (or, on an old lib, the session) dropped frames — the
* one reliability signal worth surfacing even at the tersest tier.
*/
private fun compactLine(s: DoubleArray, latValid: Boolean): String {
val parts = buildList {
add("${s[0].roundToInt()} fps")
if (latValid) add("${"%.1f".format(s[2])} ms")
add("${s[1].roundToInt()} Mb/s")
}
val lostWindow = if (s.size >= 22) s[18].toLong() else s[9].toLong()
val suffix = if (lostWindow > 0) " ⚠ lost $lostWindow" else ""
return parts.joinToString(" · ") + suffix
}
/**
* Format the spec's line-4 counters from the per-window doubles at 1821 —
* `lost {n} ({pct}%) · skipped {m} · FEC {k}`, each term only when nonzero, the whole line `null`
* when all are zero (spec: "only rendered when any value is nonzero"). `pct = lost/(frames+lost)`
* (the received count rides at index 21). A pre-window layout (< 22 doubles) falls back to the
* session-cumulative `lostTotal` so an older native lib still reports loss.
*/
private fun counterLine(s: DoubleArray, lostTotal: Long): String? {
if (s.size < 22) return if (lostTotal > 0) "lost $lostTotal" else null
val lost = s[18].toLong()
val skipped = s[19].toLong()
val fec = s[20].toLong()
val frames = s[21].toLong()
if (lost == 0L && skipped == 0L && fec == 0L) return null
return buildList {
if (lost > 0) {
val pct = 100.0 * lost / (frames + lost).coerceAtLeast(1)
add("lost $lost (${"%.1f".format(pct)}%)")
}
if (skipped > 0) add("skipped $skipped")
if (fec > 0) add("FEC $fec")
}.joinToString(" · ")
}
/**
* Format the negotiated video-feed descriptor from the trailing four stats doubles
* `[bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc]`, e.g.
@@ -54,15 +54,19 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) {
Manifest.permission.RECORD_AUDIO,
) == PackageManager.PERMISSION_GRANTED
// Live decode stats for the HUD. `showStats` gates the whole pipeline: the native per-frame
// sampling (nativeSetVideoStatsEnabled — hidden HUD costs one atomic load per frame) AND the
// 1 s poll loop, which only runs while the overlay is visible. Enabling resets the native
// window, so re-showing never renders stale data. A 3-finger tap toggles it live; the default
// comes from Settings.
// Live decode stats for the HUD. `statsOn` (verbosity != OFF) gates the whole native pipeline:
// the per-frame sampling (nativeSetVideoStatsEnabled — a hidden HUD costs one atomic load per
// frame) AND the 1 s poll loop, which only runs while the overlay is visible. Enabling resets
// the native window, so re-showing never renders stale data. A 3-finger tap cycles the
// verbosity tier live (Off → Compact → Normal → Detailed → Off); the default comes from
// Settings. The tier only changes how many lines `StatsOverlay` draws — switching between the
// visible tiers keeps sampling running (the effect keys on `statsOn`, not the tier) so it never
// blanks the numbers for a poll interval.
val initialSettings = remember { SettingsStore(context).load() }
var stats by remember { mutableStateOf<DoubleArray?>(null) }
var decoderLabel by remember { mutableStateOf("") }
var showStats by remember { mutableStateOf(initialSettings.statsHudEnabled) }
var statsVerbosity by remember { mutableStateOf(initialSettings.statsVerbosity) }
val statsOn = statsVerbosity != StatsVerbosity.OFF
// Touch model is fixed per session (re-keys the gesture handler below if it ever changes).
val touchMode = initialSettings.touchMode
// "Low-latency mode" master toggle, resolved once for the session. On (the default) enables the
@@ -73,9 +77,9 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) {
// TV form factor (leanback): the decoder actively switches the HDMI output mode to the stream
// refresh; a phone/tablet gets the softer seamless frame-rate hint instead.
val isTv = remember { context.packageManager.hasSystemFeature(PackageManager.FEATURE_LEANBACK) }
LaunchedEffect(handle, showStats) {
NativeBridge.nativeSetVideoStatsEnabled(handle, showStats)
if (showStats) {
LaunchedEffect(handle, statsOn) {
NativeBridge.nativeSetVideoStatsEnabled(handle, statsOn)
if (statsOn) {
while (true) {
delay(1000)
stats = NativeBridge.nativeVideoStats(handle)
@@ -250,8 +254,10 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) {
)
// Live stats HUD (FPS / throughput / capture→client latency), drawn over the video but
// BEFORE the transparent gesture layer below, so it shows through and never eats touches.
if (showStats) {
stats?.let { StatsOverlay(it, decoderLabel, Modifier.align(Alignment.TopStart).padding(12.dp)) }
if (statsOn) {
stats?.let {
StatsOverlay(it, statsVerbosity, decoderLabel, Modifier.align(Alignment.TopStart).padding(12.dp))
}
}
// Touch input per the Settings model: trackpad/direct-pointer mouse (the shared gesture
// vocabulary) or real multi-touch passthrough — see TouchInput.kt.
@@ -262,7 +268,7 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) {
else -> streamTouchInput(
handle,
trackpad = touchMode == TouchMode.TRACKPAD,
onToggleStats = { showStats = !showStats },
onCycleStats = { statsVerbosity = statsVerbosity.next() },
)
}
},
@@ -40,7 +40,7 @@ private const val ACCEL_MAX = 3.0f
*
* Both share the same gesture vocabulary: tap = left click; two-finger tap = right click;
* two-finger drag = scroll; tap-then-press-and-drag = left-drag (text selection / moving
* windows); three-finger tap = [onToggleStats] (the stats HUD).
* windows); three-finger tap = [onCycleStats] (cycle the stats-HUD verbosity tier).
*/
/**
* Real multi-touch passthrough ([TouchMode.TOUCH]): every finger forwards as a host touchscreen
@@ -93,7 +93,7 @@ internal suspend fun PointerInputScope.streamTouchPassthrough(handle: Long) {
internal suspend fun PointerInputScope.streamTouchInput(
handle: Long,
trackpad: Boolean,
onToggleStats: () -> Unit,
onCycleStats: () -> Unit,
) {
var lastTapUp = 0L
var lastTapX = 0f
@@ -218,7 +218,7 @@ internal suspend fun PointerInputScope.streamTouchInput(
NativeBridge.nativeSendPointerButton(handle, 1, false) // end the drag
} else if (!moved) {
when {
maxFingers >= 3 -> onToggleStats() // in-stream HUD toggle
maxFingers >= 3 -> onCycleStats() // in-stream HUD verbosity cycle
maxFingers == 2 -> { // two-finger tap → right click
NativeBridge.nativeSendPointerButton(handle, 3, true)
NativeBridge.nativeSendPointerButton(handle, 3, false)
@@ -58,7 +58,15 @@ class ScreenshotTest {
@Test
@Config(sdk = [36], qualifiers = "w800dp-h360dp-xxhdpi") // landscape — the stream is immersive
fun stream() = shootRoot("stream") { StreamScene() }
fun stream() = shootRoot("stream") { StreamScene(io.unom.punktfunk.StatsVerbosity.DETAILED) }
@Test
@Config(sdk = [36], qualifiers = "w800dp-h360dp-xxhdpi")
fun streamCompact() = shootRoot("stream-compact") { StreamScene(io.unom.punktfunk.StatsVerbosity.COMPACT) }
@Test
@Config(sdk = [36], qualifiers = "w800dp-h360dp-xxhdpi")
fun streamNormal() = shootRoot("stream-normal") { StreamScene(io.unom.punktfunk.StatsVerbosity.NORMAL) }
@Test
fun trust() = shootScreen("trust") {
@@ -30,6 +30,7 @@ import io.unom.punktfunk.Settings
import io.unom.punktfunk.TouchMode
import io.unom.punktfunk.SettingsScreen
import io.unom.punktfunk.StatsOverlay
import io.unom.punktfunk.StatsVerbosity
import io.unom.punktfunk.components.HostCard
import io.unom.punktfunk.components.SectionLabel
import io.unom.punktfunk.models.HostStatus
@@ -109,7 +110,7 @@ internal fun SettingsScene() {
compositor = 1,
gamepad = 2,
micEnabled = true,
statsHudEnabled = true,
statsVerbosity = StatsVerbosity.DETAILED,
touchMode = TouchMode.TRACKPAD,
),
onChange = {},
@@ -177,9 +178,12 @@ internal fun PairDialog() {
)
}
/** The live stats HUD (the real StatsOverlay) over a synthetic "streamed frame" gradient. */
/**
* The live stats HUD (the real StatsOverlay) over a synthetic "streamed frame" gradient, at the
* given [verbosity] tier — one scene per tier documents how far each tones the overlay down.
*/
@Composable
internal fun StreamScene() {
internal fun StreamScene(verbosity: StatsVerbosity = StatsVerbosity.DETAILED) {
Box(
Modifier
.fillMaxSize()
@@ -187,17 +191,21 @@ internal fun StreamScene() {
Brush.linearGradient(listOf(Color(0xFF2A1E5C), Color(0xFF0E1B3D), Color(0xFF06122B))),
),
) {
// The full 18-double unified layout (design/stats-unification.md): [fps, mbps, e2eP50,
// e2eP95, latValid, skew, w, h, hz, lost, bitDepth, colorPrimaries, colorTransfer,
// chromaFormatIdc, hostNetP50, decodeP50, hostP50, netP50]. 10/9/16/1 = a 10-bit BT.2020
// PQ (HDR) 4:2:0 feed so the HUD renders its video-feed line; the Phase-2 stage terms
// (host 0.6 + network 0.3 + decode 0.4) tile the 1.3 ms headline so it renders the full
// split equation, and the decoder label line shows the ranked low-latency decoder.
// The full 22-double unified layout (design/stats-unification.md): [fps, mbps, e2eP50,
// e2eP95, latValid, skew, w, h, hz, lostTotal, bitDepth, colorPrimaries, colorTransfer,
// chromaFormatIdc, hostNetP50, decodeP50, hostP50, netP50, lost, skipped, fec, frames].
// 10/9/16/1 = a 10-bit BT.2020 PQ (HDR) 4:2:0 feed so the DETAILED HUD renders its
// video-feed line; the Phase-2 stage terms (host 0.6 + network 0.3 + decode 0.4) tile the
// 1.3 ms headline so it renders the full split equation, and the decoder label shows the
// ranked low-latency decoder. Light per-window loss (lost 2 · skipped 1 · FEC 5 of 238) so
// the reliability line (NORMAL/DETAILED) and the compact loss flag both render.
StatsOverlay(
doubleArrayOf(
238.0, 921.4, 1.3, 2.1, 1.0, 1.0, 5120.0, 1440.0, 240.0, 0.0,
238.0, 921.4, 1.3, 2.1, 1.0, 1.0, 5120.0, 1440.0, 240.0, 2.0,
10.0, 9.0, 16.0, 1.0, 0.9, 0.4, 0.6, 0.3,
2.0, 1.0, 5.0, 238.0,
),
verbosity = verbosity,
decoderLabel = "c2.qti.hevc.decoder · low-latency",
modifier = Modifier.align(Alignment.TopStart).padding(12.dp),
)
+43 -14
View File
@@ -493,10 +493,14 @@ fn decoder_supports_max_operating_rate(name_lower: &str) -> bool {
}
/// One decoded output buffer ready to release: its codec buffer index + the pts the codec echoed
/// (from the output callback's `BufferInfo`), used to pair the `decode` HUD stat.
/// (from the output callback's `BufferInfo`), used to pair the `decode` HUD stat, and the
/// wall-clock instant the output callback fired — the spec's `decoded` point ("decoder output
/// frame available"), stamped at the callback so the event-channel hop + coalescing wait in the
/// loop never inflates the decode stage.
struct OutputReady {
index: usize,
pts_us: u64,
decoded_ns: i128,
}
/// Events the async decode loop reacts to. The codec's async-notify callbacks (which run on its
@@ -507,8 +511,12 @@ enum DecodeEvent {
Au(Frame),
/// An input buffer slot freed (index) — we can queue an AU into it.
InputAvailable(usize),
/// A decoded frame is ready (buffer index + echoed pts).
OutputAvailable { index: usize, pts_us: u64 },
/// A decoded frame is ready (buffer index + echoed pts + the callback-time `decoded` stamp).
OutputAvailable {
index: usize,
pts_us: u64,
decoded_ns: i128,
},
/// The output format changed — re-check the stream's colour signalling (HDR DataSpace).
FormatChanged,
/// The codec reported an error; `fatal` when neither recoverable nor transient.
@@ -569,6 +577,10 @@ fn run_async(
let _ = out_tx.send(DecodeEvent::OutputAvailable {
index: idx,
pts_us: info.presentation_time_us().max(0) as u64,
// The `decoded` HUD point: stamp HERE, on the codec's looper thread, so the
// decode stage ends when the frame actually became available — not after the
// channel hop + whatever work the loop coalesces in front of presenting it.
decoded_ns: now_realtime_ns(),
});
})),
on_format_changed: Some(Box::new(move |_fmt| {
@@ -697,29 +709,30 @@ fn run_async(
};
let work_t0 = Instant::now();
let mut fmt_dirty = false;
let mut au_dropped = false;
let mut aus_dropped: u64 = 0;
if let Some(ev) = ev0 {
au_dropped |= dispatch_event(
aus_dropped += u64::from(dispatch_event(
ev,
&mut pending_aus,
&mut free_inputs,
&mut ready,
&mut fmt_dirty,
&mut fatal,
);
));
}
// Coalesce every other event already queued into this one work pass — correct newest-only
// presentation across a decode burst, and batched feeding.
while let Ok(ev) = ev_rx.try_recv() {
au_dropped |= dispatch_event(
aus_dropped += u64::from(dispatch_event(
ev,
&mut pending_aus,
&mut free_inputs,
&mut ready,
&mut fmt_dirty,
&mut fatal,
);
));
}
stats.note_skipped(aus_dropped); // parked-AU overflow drops are client-side skips too
if fmt_dirty {
apply_hdr_dataspace(&codec, &window, &mut applied_ds);
}
@@ -768,7 +781,7 @@ fn run_async(
// dropped a parked AU on overflow), throttled so a multi-frame recovery gap doesn't flood the
// control stream.
let dropped = client.frames_dropped();
if dropped > last_dropped || au_dropped {
if dropped > last_dropped || aus_dropped > 0 {
last_dropped = dropped;
let now = Instant::now();
if last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) {
@@ -863,7 +876,15 @@ fn dispatch_event(
}
}
DecodeEvent::InputAvailable(i) => free_inputs.push_back(i),
DecodeEvent::OutputAvailable { index, pts_us } => ready.push(OutputReady { index, pts_us }),
DecodeEvent::OutputAvailable {
index,
pts_us,
decoded_ns,
} => ready.push(OutputReady {
index,
pts_us,
decoded_ns,
}),
DecodeEvent::FormatChanged => *fmt_dirty = true,
DecodeEvent::Error { fatal: f } => {
if f {
@@ -935,15 +956,19 @@ fn present_ready(
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for o in ready.iter() {
note_decoded_pts(stats, &mut g, clock_offset, o.pts_us);
note_decoded_pts(stats, &mut g, clock_offset, o.pts_us, o.decoded_ns);
}
}
let last = ready.len() - 1;
let mut skipped: u64 = 0;
for (i, o) in ready.drain(..).enumerate() {
let render = i == last;
match codec.release_output_buffer_by_index(o.index, render) {
Ok(()) if render => *rendered += 1,
Ok(()) => *discarded += 1,
Ok(()) => {
*discarded += 1;
skipped += 1;
}
Err(e) => {
log::warn!(
"decode: release_output_buffer_by_index({}, {render}): {e}",
@@ -952,6 +977,7 @@ fn present_ready(
}
}
}
stats.note_skipped(skipped); // HUD `skipped` counter (newest-wins drops); no-op while hidden
}
/// React to an output-format change by signalling the stream's HDR dataspace on the Surface (SDR
@@ -1143,6 +1169,7 @@ fn drain(
log::warn!("decode: release_output_buffer(discard): {e}");
}
discarded += 1;
stats.note_skipped(1); // HUD `skipped` counter; no-op while hidden
}
}
Ok(DequeuedOutputBufferInfoResult::OutputFormatChanged) => {
@@ -1203,18 +1230,20 @@ fn note_decoded(
in_flight,
clock_offset,
buf.info().presentation_time_us().max(0) as u64,
now_realtime_ns(), // sync loop: the dequeue IS the availability instant
);
}
/// The [`note_decoded`] body keyed by the echoed `presentationTimeUs` directly — the async loop has
/// the pts (from the output callback's `BufferInfo`) but no borrowed `OutputBuffer`, so it calls this.
/// the pts (from the output callback's `BufferInfo`) but no borrowed `OutputBuffer`, so it calls
/// this with the `decoded` stamp taken in the output callback itself (the availability instant).
fn note_decoded_pts(
stats: &crate::stats::VideoStats,
in_flight: &mut VecDeque<(u64, i128)>,
clock_offset: i64,
pts_us: u64,
decoded_ns: i128,
) {
let decoded_ns = now_realtime_ns();
// Pair the echoed pts back to its receipt stamp, evicting stale (older) entries as we go.
let mut received_ns = None;
while let Some(&(p, r)) = in_flight.front() {
+26 -7
View File
@@ -144,16 +144,20 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopVideo(
}
/// `NativeBridge.nativeVideoStats(handle): DoubleArray?` — drain ~1 s of decode stats for the HUD
/// (unified stats spec, `design/stats-unification.md`). Returns 18 doubles
/// (unified stats spec, `design/stats-unification.md`). Returns 22 doubles
/// `[fps, mbps, e2eP50Ms, e2eP95Ms, latValid, skewCorrected, width, height, refreshHz, framesLost,
/// bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc, hostNetP50Ms, decodeP50Ms, hostP50Ms,
/// netP50Ms]`
/// (the two flags are 1.0/0.0; indexes 015 match the previous 16-double layout — 013 the original
/// netP50Ms, lostWindow, skippedWindow, fecWindow, framesWindow]`
/// (the two flags are 1.0/0.0; indexes 017 match the previous 18-double layout — 013 the original
/// 14-double one with the latency pair re-based to the end-to-end capture→decoded headline, 14/15
/// the stage p50s tiling it: `host+network` = capture→received, `decode` = received→decoded; 16/17
/// are the Phase-2 split of the `host+network` term from the per-AU 0xCF host timings — `host` =
/// the host's capture→sent, `network` = the remainder — both 0.0 when no timing matched this
/// window, i.e. an old host), or `null` when no decode thread is running. Poll ~1 Hz from the UI; each call
/// window, i.e. an old host; 1821 are the spec's per-window line-4 counters — `lost` =
/// unrecoverable drops this window, `skipped` = client newest-wins/pacing drops, `fec` = shards
/// recovered, `frames` = AUs received, so the HUD can compute `lost/(frames+lost)` — index 9 stays
/// the cumulative session total for older readers), or `null` when no decode thread is running.
/// Poll ~1 Hz from the UI; each call
/// resets the measurement window. Not android-gated — pure `jni` + connector reads, so it links on
/// the host build too (Kotlin only ever calls it on device).
#[no_mangle]
@@ -171,10 +175,12 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats(
if h.video.lock().unwrap().is_none() {
return std::ptr::null_mut(); // not streaming → no stats
}
let snap = h.stats.drain();
let snap = h
.stats
.drain(h.client.frames_dropped(), h.client.fec_recovered_shards());
let mode = h.client.mode();
let color = h.client.color;
let buf: [f64; 18] = [
let buf: [f64; 22] = [
snap.fps,
snap.mbps,
snap.e2e_p50_ms,
@@ -200,6 +206,13 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats(
// when no timing matched this window (old host) — the HUD keeps the combined term.
snap.host_p50_ms,
snap.net_p50_ms,
// Spec line-4 counters, per-window: lost (unrecoverable drops), skipped (client
// newest-wins/pacing drops), FEC shards recovered, and the received-AU count so the
// HUD computes the loss percentage `lost/(frames+lost)` exactly.
snap.lost as f64,
snap.skipped as f64,
snap.fec as f64,
snap.frames as f64,
];
let arr = match env.new_double_array(buf.len() as jsize) {
Ok(a) => a,
@@ -228,7 +241,13 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSetVideoSta
if handle != 0 {
// SAFETY: live handle per the nativeConnect/nativeClose contract.
let h = unsafe { &*(handle as *const SessionHandle) };
h.stats.set_enabled(enabled != 0);
// The current cumulative counters seed the window baselines, so the first snapshot's
// `lost`/`FEC` cover only time the HUD was actually up.
h.stats.set_enabled(
enabled != 0,
h.client.frames_dropped(),
h.client.fec_recovered_shards(),
);
}
})
}
+58 -4
View File
@@ -3,7 +3,10 @@
//! headline `end-to-end` = capture→decoded (p50/p95) tiled by `host+network` = capture→received
//! and `decode` = received→decoded (stage p50s). When the host emits per-AU 0xCF host timings, the
//! `host+network` term further splits into `host` + `network` (Phase 2, `note_host_split`); an old
//! host emits none and the combined term stands. The decode thread is the sole writer
//! host emits none and the combined term stands. The spec's line-4 counters are per-window too:
//! `lost` / `FEC` are windowed here from the connector's cumulative counters (the caller passes the
//! current totals into `set_enabled`/`drain`), `skipped` counts the client's own newest-wins drops
//! (`note_skipped`). The decode thread is the sole writer
//! (`note_received` per access unit at receipt, `note_decoded` per decoder output buffer); the JNI
//! accessor `nativeVideoStats` drains a snapshot ~1 Hz and resets the window. Sampling is gated on
//! the HUD actually being visible (`set_enabled`, driven by `nativeSetVideoStatsEnabled`) so the
@@ -54,6 +57,14 @@ struct Inner {
net_us: Vec<u64>,
/// `decode` stage = received→decoded samples, in microseconds (client-local, single clock).
decode_us: Vec<u64>,
/// Client-side newest-wins/pacing drops this window (decoded frames released without
/// rendering, or parked AUs dropped on overflow) — the spec's `skipped` counter.
skipped: u64,
/// Baselines for windowing the session-cumulative connector counters: the unrecoverable-drop
/// and FEC-recovered totals as of the last drain (or the enable that opened the window), so
/// each snapshot reports only THIS window's `lost` / `FEC` (spec line 4).
last_dropped_total: u64,
last_fec_total: u64,
/// Whether the host answered the clock-skew handshake (latency is cross-machine valid).
skew_corrected: bool,
}
@@ -76,6 +87,16 @@ pub struct Snapshot {
pub net_p50_ms: f64,
pub lat_valid: bool,
pub skew_corrected: bool,
/// Access units received this window (the count behind `fps`) — lets the HUD compute the
/// spec's loss percentage `lost / (received + lost)` exactly.
pub frames: u64,
/// Unrecoverable network frame drops this window (spec `lost`, windowed from the
/// session-cumulative connector counter).
pub lost: u64,
/// Client-side newest-wins/pacing drops this window (spec `skipped`).
pub skipped: u64,
/// FEC shards recovered this window (spec `FEC`, windowed from the cumulative counter).
pub fec: u64,
}
/// Percentile over a sorted-in-place µs sample vec, in ms. 0.0 when empty.
@@ -101,6 +122,9 @@ impl VideoStats {
host_us: Vec::with_capacity(256),
net_us: Vec::with_capacity(256),
decode_us: Vec::with_capacity(256),
skipped: 0,
last_dropped_total: 0,
last_fec_total: 0,
skew_corrected: false,
}),
}
@@ -115,8 +139,10 @@ impl VideoStats {
}
/// Toggle sampling. Enabling resets the window, so the first HUD poll after a show never mixes
/// in counters (or a window start) from before the overlay was visible.
pub fn set_enabled(&self, on: bool) {
/// in counters (or a window start) from before the overlay was visible. `dropped_total` /
/// `fec_total` are the connector's session-cumulative counters at this instant — they seed the
/// windowing baselines so the first snapshot's `lost` / `FEC` cover only time the HUD was up.
pub fn set_enabled(&self, on: bool, dropped_total: u64, fec_total: u64) {
let was = self.enabled.swap(on, Ordering::Relaxed);
if on && !was {
let mut g = self
@@ -131,6 +157,9 @@ impl VideoStats {
g.host_us.clear();
g.net_us.clear();
g.decode_us.clear();
g.skipped = 0;
g.last_dropped_total = dropped_total;
g.last_fec_total = fec_total;
}
}
@@ -206,6 +235,22 @@ impl VideoStats {
g.net_us.push(net_us);
}
/// Record client-side frame skips (spec `skipped`): decoded output buffers released without
/// rendering under the newest-wins policy, or parked AUs dropped on queue overflow.
// Driven only by the android-only decode thread; unreferenced on the host build — expected.
#[cfg_attr(not(target_os = "android"), allow(dead_code))]
pub fn note_skipped(&self, n: u64) {
if n == 0 || !self.enabled.load(Ordering::Relaxed) {
return; // HUD hidden — skip the lock
}
// Poison-proof for the same reason as `note_received`.
let mut g = self
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
g.skipped += n;
}
/// Record one decoded output frame: its capture→decoded `end-to-end` sample and its
/// received→decoded `decode` stage sample (either may be absent — e.g. the receipt stamp for
/// this pts predates the HUD being shown).
@@ -229,7 +274,9 @@ impl VideoStats {
}
/// Compute the window's rates + latency percentiles, then reset for the next window.
pub fn drain(&self) -> Snapshot {
/// `dropped_total` / `fec_total` are the connector's session-cumulative counters now; the
/// snapshot's `lost` / `fec` are their deltas since the last drain (or the enabling show).
pub fn drain(&self, dropped_total: u64, fec_total: u64) -> Snapshot {
// Poison-proof for the same reason as `note_received` — a poisoned window still drains
// fine.
let mut g = self
@@ -255,6 +302,10 @@ impl VideoStats {
net_p50_ms: pctl_ms(&g.net_us, 0.50),
lat_valid: !g.e2e_us.is_empty(),
skew_corrected: g.skew_corrected,
frames: g.frames,
lost: dropped_total.saturating_sub(g.last_dropped_total),
skipped: g.skipped,
fec: fec_total.saturating_sub(g.last_fec_total),
};
g.window_start = Instant::now();
g.frames = 0;
@@ -264,6 +315,9 @@ impl VideoStats {
g.host_us.clear();
g.net_us.clear();
g.decode_us.clear();
g.skipped = 0;
g.last_dropped_total = dropped_total;
g.last_fec_total = fec_total;
snap
}
}
+20
View File
@@ -314,6 +314,11 @@ pub struct NativeClient {
/// a recovery keyframe under infinite GOP — the correct loss trigger, since unrecoverable loss
/// yields reference-missing frames the decoder silently conceals (a decode-error trigger misses them).
frames_dropped: Arc<AtomicU64>,
/// Cumulative count of FEC shards the reassembler recovered (parity repaired a lost data
/// packet), mirrored from the data-plane pump's `Session` like `frames_dropped`. Observability
/// for the client stats HUDs (the unified spec's per-window `FEC` counter — proof FEC is
/// earning its keep); readers window it by diffing successive reads.
fec_recovered: Arc<AtomicU64>,
/// Kernel ids of the client's latency-critical native threads: the internal data-plane pump
/// (UDP receive + FEC reassembly) plus any embedder plane threads registered via
/// [`NativeClient::register_hot_thread`]. The Android client feeds these to an ADPF hint session
@@ -490,6 +495,7 @@ impl NativeClient {
let mode_slot = Arc::new(std::sync::Mutex::new(mode));
let probe = Arc::new(Mutex::new(ProbeState::default()));
let frames_dropped = Arc::new(AtomicU64::new(0));
let fec_recovered = Arc::new(AtomicU64::new(0));
let hot_tids = Arc::new(Mutex::new(Vec::new()));
let host = host.to_string();
@@ -499,6 +505,7 @@ impl NativeClient {
let mode_slot_w = mode_slot.clone();
let probe_w = probe.clone();
let frames_dropped_w = frames_dropped.clone();
let fec_recovered_w = fec_recovered.clone();
let hot_tids_w = hot_tids.clone();
let ctrl_tx_pump = ctrl_tx.clone(); // the data-plane pump sends adaptive-FEC LossReports
let worker = std::thread::Builder::new()
@@ -550,6 +557,7 @@ impl NativeClient {
mode_slot: mode_slot_w,
probe: probe_w,
frames_dropped: frames_dropped_w,
fec_recovered: fec_recovered_w,
hot_tids: hot_tids_w,
}));
})
@@ -592,6 +600,7 @@ impl NativeClient {
quit,
worker: Some(worker),
frames_dropped,
fec_recovered,
hot_tids,
mode: mode_slot,
host_fingerprint: fingerprint,
@@ -734,6 +743,14 @@ impl NativeClient {
self.frames_dropped.load(Ordering::Relaxed)
}
/// Cumulative FEC shards the host→client reassembler recovered (a parity shard repaired a lost
/// data packet — loss that never became a dropped frame). Monotonic for the session; a stats
/// HUD windows it by diffing successive reads, pairing it with
/// [`frames_dropped`](Self::frames_dropped) (the losses FEC could NOT absorb).
pub fn fec_recovered_shards(&self) -> u64 {
self.fec_recovered.load(Ordering::Relaxed)
}
/// Whether the underlying QUIC session has ended — the worker's connection-close watcher set the
/// shutdown flag (`conn.closed()` fired: a host suspend / crash / network drop idle-timed the
/// connection out, or the host closed it), or a deliberate [`disconnect_quit`](Self::disconnect_quit)
@@ -987,6 +1004,7 @@ struct WorkerArgs {
mode_slot: Arc<std::sync::Mutex<Mode>>,
probe: Arc<Mutex<ProbeState>>,
frames_dropped: Arc<AtomicU64>,
fec_recovered: Arc<AtomicU64>,
hot_tids: Arc<Mutex<Vec<i32>>>,
}
@@ -1024,6 +1042,7 @@ async fn worker_main(args: WorkerArgs) {
mode_slot,
probe,
frames_dropped,
fec_recovered,
hot_tids,
} = args;
let setup = async {
@@ -1362,6 +1381,7 @@ async fn worker_main(args: WorkerArgs) {
// through a total-loss drought where no AU completes. Cheap: a few relaxed atomic loads.
let st = session.stats();
frames_dropped.store(st.frames_dropped, Ordering::Relaxed);
fec_recovered.store(st.fec_recovered_shards, Ordering::Relaxed);
let probe_active = {
let mut p = pump_probe.lock().unwrap();
if p.active && !p.done {