diff --git a/clients/android/app/src/main/AndroidManifest.xml b/clients/android/app/src/main/AndroidManifest.xml index d51e90b..932c3d6 100644 --- a/clients/android/app/src/main/AndroidManifest.xml +++ b/clients/android/app/src/main/AndroidManifest.xml @@ -43,6 +43,14 @@ android:supportsRtl="true" android:theme="@style/Theme.PunktfunkAndroid"> + + + Unit, context: an options = COMPOSITOR_OPTIONS.mapIndexed { i, lbl -> i to lbl }, selected = s.compositor, ) { c -> update(s.copy(compositor = c)) } + + ToggleRow( + title = "Low-latency mode", + subtitle = "Run the decoder at max clocks for the lowest latency. Turn off only if a " + + "device overheats or glitches during long sessions.", + checked = s.lowLatencyMode, + onCheckedChange = { on -> update(s.copy(lowLatencyMode = on)) }, + ) } } diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt index c10a82b..ea6f877 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt @@ -27,7 +27,7 @@ import kotlin.math.roundToInt * older layouts just omit those lines. */ @Composable -internal fun StatsOverlay(s: DoubleArray, modifier: Modifier = Modifier) { +internal fun StatsOverlay(s: DoubleArray, decoderLabel: String = "", modifier: Modifier = Modifier) { if (s.size < 10) return val w = s[6].toInt() val h = s[7].toInt() @@ -46,6 +46,14 @@ internal fun StatsOverlay(s: DoubleArray, modifier: Modifier = Modifier) { fontFamily = FontFamily.Monospace, fontSize = 12.sp, ) + if (decoderLabel.isNotEmpty()) { + Text( + decoderLabel, + color = Color(0xFFB0D0FF), + fontFamily = FontFamily.Monospace, + fontSize = 12.sp, + ) + } videoFeedLine(s)?.let { feed -> Text( feed, diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt index 9ce71d6..3354a63 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt @@ -1,8 +1,11 @@ package io.unom.punktfunk import android.Manifest +import android.content.Context import android.content.pm.ActivityInfo import android.content.pm.PackageManager +import android.net.wifi.WifiManager +import android.os.Build import android.view.SurfaceHolder import android.view.SurfaceView import android.view.WindowManager @@ -30,6 +33,7 @@ import androidx.core.view.WindowInsetsControllerCompat import io.unom.punktfunk.kit.Gamepad import io.unom.punktfunk.kit.GamepadFeedback import io.unom.punktfunk.kit.NativeBridge +import io.unom.punktfunk.kit.VideoDecoders import java.util.concurrent.atomic.AtomicBoolean import kotlinx.coroutines.delay @@ -55,15 +59,23 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { // comes from Settings. val initialSettings = remember { SettingsStore(context).load() } var stats by remember { mutableStateOf(null) } + var decoderLabel by remember { mutableStateOf("") } var showStats by remember { mutableStateOf(initialSettings.statsHudEnabled) } // Touch model is fixed per session (re-keys the gesture handler below if it ever changes). val touchMode = initialSettings.touchMode + // Master low-latency toggle, resolved once for the session and passed to the decoder at start. + val lowLatencyMode = initialSettings.lowLatencyMode + // TV form factor (leanback): the decoder actively switches the HDMI output mode to the stream + // refresh; a phone/tablet gets the softer seamless frame-rate hint instead. + val isTv = remember { context.packageManager.hasSystemFeature(PackageManager.FEATURE_LEANBACK) } LaunchedEffect(handle, showStats) { NativeBridge.nativeSetVideoStatsEnabled(handle, showStats) if (showStats) { while (true) { delay(1000) stats = NativeBridge.nativeVideoStats(handle) + // The decoder is fixed for the session; fetch its label once it's resolved. + if (decoderLabel.isEmpty()) decoderLabel = NativeBridge.nativeVideoDecoderLabel(handle) } } else { stats = null // drop the last snapshot so a re-show never flashes stale numbers @@ -76,8 +88,29 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { // main thread, so a plain flag is race-free; AtomicBoolean just makes the intent explicit. val closed = remember { AtomicBoolean(false) } + // A Wi-Fi low-latency lock held for the stream's duration: asks the Wi-Fi firmware to drop its + // power-save polling (a common source of tens-of-ms jitter). WIFI_MODE_FULL_LOW_LATENCY (API + // 29+) is the strongest; older releases fall back to FULL_HIGH_PERF. Needs no extra permission + // beyond ACCESS_WIFI_STATE (already declared). Non-reference-counted: one explicit acquire/release. + val wifiLock = remember(handle) { + val wm = context.applicationContext.getSystemService(Context.WIFI_SERVICE) as? WifiManager + val mode = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + WifiManager.WIFI_MODE_FULL_LOW_LATENCY + } else { + @Suppress("DEPRECATION") + WifiManager.WIFI_MODE_FULL_HIGH_PERF + } + wm?.createWifiLock(mode, "punktfunk:stream")?.apply { setReferenceCounted(false) } + } + DisposableEffect(handle) { window?.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + runCatching { wifiLock?.acquire() } + // HDMI Auto Low-Latency Mode: ask the display to drop its post-processing (game mode) — + // the biggest panel-side latency win on the TV boxes. No-op where ALLM isn't supported. API 30+. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + window?.setPreferMinimalPostProcessing(true) + } controller?.let { it.systemBarsBehavior = WindowInsetsControllerCompat.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE it.hide(WindowInsetsCompat.Type.systemBars()) @@ -105,6 +138,10 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { activity?.setConsoleHighRefreshRate(true) // back to the console UI's max refresh controller?.show(WindowInsetsCompat.Type.systemBars()) window?.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + window?.setPreferMinimalPostProcessing(false) + } + runCatching { if (wifiLock?.isHeld == true) wifiLock.release() } // Release the landscape lock so the rest of the app follows the device/system again. activity?.requestedOrientation = priorOrientation ?: ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED @@ -125,7 +162,19 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { SurfaceView(ctx).apply { holder.addCallback(object : SurfaceHolder.Callback { override fun surfaceCreated(holder: SurfaceHolder) { - NativeBridge.nativeStartVideo(handle, holder.surface) + // Rank MediaCodecList decoders for the negotiated MIME (framework-only + // API) and hand the chosen one to Rust, which creates it by name and + // applies the per-SoC vendor low-latency keys. + val mime = NativeBridge.nativeVideoMime(handle) + val choice = VideoDecoders.pickDecoder(mime) + NativeBridge.nativeStartVideo( + handle, + holder.surface, + choice?.name ?: "", + lowLatencyMode, + choice?.lowLatencyFeature ?: false, + isTv, + ) NativeBridge.nativeStartAudio(handle) if (micWanted) NativeBridge.nativeStartMic(handle) } @@ -150,7 +199,7 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { // Live stats HUD (FPS / throughput / capture→client latency), drawn over the video but // BEFORE the transparent gesture layer below, so it shows through and never eats touches. if (showStats) { - stats?.let { StatsOverlay(it, Modifier.align(Alignment.TopStart).padding(12.dp)) } + stats?.let { StatsOverlay(it, decoderLabel, Modifier.align(Alignment.TopStart).padding(12.dp)) } } // Touch input per the Settings model: trackpad/direct-pointer mouse (the shared gesture // vocabulary) or real multi-touch passthrough — see TouchInput.kt. diff --git a/clients/android/app/src/main/res/xml/game_mode_config.xml b/clients/android/app/src/main/res/xml/game_mode_config.xml new file mode 100644 index 0000000..fe0c093 --- /dev/null +++ b/clients/android/app/src/main/res/xml/game_mode_config.xml @@ -0,0 +1,14 @@ + + + diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index 2875ffb..c83f37e 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -104,14 +104,40 @@ object NativeBridge { external fun nativeWakeOnLan(macsCsv: String, lastIp: String): Boolean /** - * Start the HEVC decode thread rendering onto [surface] (a SurfaceView's surface). Decode runs - * entirely in Rust (NDK AMediaCodec → ANativeWindow) — no per-frame JNI. No-op if already started. + * The MediaCodec MIME the host resolved for this session (`"video/hevc"` / `"video/avc"` / + * `"video/av01"`), or `""` on a `0` handle. Kotlin ranks `MediaCodecList` decoders for this + * MIME (see [io.unom.punktfunk.kit.VideoDecoders]) before [nativeStartVideo]. Cheap; UI-safe. */ - external fun nativeStartVideo(handle: Long, surface: android.view.Surface) + external fun nativeVideoMime(handle: Long): String + + /** + * Start the decode thread rendering onto [surface] (a SurfaceView's surface). Decode runs + * entirely in Rust (NDK AMediaCodec → ANativeWindow) — no per-frame JNI. [decoderName] is the + * decoder Kotlin ranked from `MediaCodecList` (`""` = let the platform resolve the default for + * the MIME); [lowLatencyMode] is the user's master toggle (default on → aggressive per-SoC + * tuning; off → conservative); [lowLatencyFeature] is whether [decoderName] advertised + * `FEATURE_LowLatency` (HUD label only). [isTv] drives an active HDMI mode switch to the stream + * refresh on TV boxes (vs. the softer seamless hint on phones). No-op if already started. + */ + external fun nativeStartVideo( + handle: Long, + surface: android.view.Surface, + decoderName: String, + lowLatencyMode: Boolean, + lowLatencyFeature: Boolean, + isTv: Boolean, + ) /** Stop + join the decode thread without closing the session. No-op on `0`. */ external fun nativeStopVideo(handle: Long) + /** + * The resolved decoder identity for the HUD, e.g. `c2.qti.avc.decoder · low-latency`, or `""` + * before the decode thread has resolved one. One-shot (fixed for the session); poll once after + * the HUD appears. + */ + external fun nativeVideoDecoderLabel(handle: Long): String + /** * Drain ~1 s of live decode stats for the on-stream HUD, or `null` when no decode thread runs. * Returns 18 doubles (unified stats spec, `design/stats-unification.md`): diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/VideoDecoders.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/VideoDecoders.kt new file mode 100644 index 0000000..3fd5887 --- /dev/null +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/VideoDecoders.kt @@ -0,0 +1,85 @@ +package io.unom.punktfunk.kit + +import android.media.MediaCodecInfo.CodecCapabilities +import android.media.MediaCodecList +import android.os.Build + +/** The decoder Kotlin ranked for a MIME, handed to [NativeBridge.nativeStartVideo]. */ +data class DecoderChoice(val name: String, val lowLatencyFeature: Boolean) + +/** + * Rank the platform's `MediaCodecList` decoders for a video MIME and pick the best one for + * low-latency streaming, the way Moonlight-Android does. There is no NDK `MediaCodecList`, so this + * enumeration must live on the Kotlin (framework) side; Rust then creates the chosen decoder by + * name (`AMediaCodec_createCodecByName`) and derives the per-SoC vendor low-latency keys from it. + * + * Ranking (best first): hardware over software; a real SoC-vendor decoder (Qualcomm/Amlogic/…) over + * the generic AOSP software fallback; a decoder advertising `FEATURE_LowLatency` over one that + * doesn't. Known-bad software decoders (`omx.google.*`, `c2.android.*`, Qualcomm/Samsung SW HEVC) + * are dropped outright — matching Moonlight's blacklist. + */ +object VideoDecoders { + /** Decoder-name prefixes/names we never want, mirroring Moonlight's blacklist. */ + private val BLOCKED_PREFIXES = listOf("omx.google.", "c2.android.", "avcdecoder", "omx.ffmpeg.") + private val BLOCKED_EXACT = listOf("omx.qcom.video.decoder.hevcswvdec", "omx.sec.hevc.sw.dec") + + /** + * Real SoC-vendor decoder prefixes we prefer over the generic AOSP fallback, covering the common + * targets: Qualcomm Snapdragon and MediaTek (most phones + many TV boxes), Samsung Exynos (+ + * Google Tensor, whose decoder is `c2.exynos.*`), NVIDIA Tegra (Shield TV), Amlogic / Rockchip / + * Realtek (TV boxes & smart TVs), and HiSilicon Kirin (older Huawei). + */ + private val VENDOR_PREFIXES = listOf( + "omx.qcom", "c2.qti", + "omx.mtk", "c2.mtk", + "omx.exynos", "c2.exynos", + "omx.nvidia", "c2.nvidia", + "omx.amlogic", "c2.amlogic", + "omx.rk", "c2.rk", + "omx.realtek", "c2.realtek", + "omx.hisi", "c2.hisi", + ) + + /** + * Pick the best decoder for [mime] (`"video/hevc"` / `"video/avc"` / `"video/av01"`), or `null` + * to let the platform resolve its default. Enumerates once — call at stream start. + */ + fun pickDecoder(mime: String): DecoderChoice? { + if (mime.isEmpty()) return null + val infos = runCatching { MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos } + .getOrNull() ?: return null + + var bestName: String? = null + var bestLowLatency = false + var bestScore = Int.MIN_VALUE + for (info in infos) { + if (info.isEncoder) continue + val name = info.name + val lower = name.lowercase() + if (BLOCKED_PREFIXES.any { lower.startsWith(it) } || lower in BLOCKED_EXACT) continue + val caps = runCatching { info.getCapabilitiesForType(mime) }.getOrNull() ?: continue + + val hardware = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + info.isHardwareAccelerated + } else { + // Pre-Q heuristic: the software decoders are the ones we can name (already blocked + // above), so anything surviving the blacklist is treated as hardware. + true + } + val lowLatency = Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && + runCatching { caps.isFeatureSupported(CodecCapabilities.FEATURE_LowLatency) } + .getOrDefault(false) + val vendor = VENDOR_PREFIXES.any { lower.startsWith(it) } + + val score = (if (hardware) 100 else 0) + + (if (vendor) 40 else 0) + + (if (lowLatency) 20 else 0) + if (score > bestScore) { + bestScore = score + bestName = name + bestLowLatency = lowLatency + } + } + return bestName?.let { DecoderChoice(it, bestLowLatency) } + } +} diff --git a/clients/android/native/src/adpf.rs b/clients/android/native/src/adpf.rs index 9b561ba..6d7b9aa 100644 --- a/clients/android/native/src/adpf.rs +++ b/clients/android/native/src/adpf.rs @@ -28,6 +28,7 @@ type CreateSessionFn = unsafe extern "C" fn(*mut c_void, *const i32, usize, i64) type ReportFn = unsafe extern "C" fn(*mut c_void, i64) -> c_int; type UpdateTargetFn = unsafe extern "C" fn(*mut c_void, i64) -> c_int; type CloseFn = unsafe extern "C" fn(*mut c_void); +type SetPreferPowerEfficiencyFn = unsafe extern "C" fn(*mut c_void, bool) -> c_int; /// The entry points we use, resolved once from `libandroid.so`, plus the process-wide manager. struct Api { @@ -35,6 +36,9 @@ struct Api { report: ReportFn, update_target: UpdateTargetFn, close: CloseFn, + /// `APerformanceHint_setPreferPowerEfficiency` — NDK **API 35**, so `Option`al even when the + /// rest of ADPF resolved (a 33/34 device has the session API but not this one). + set_prefer_power_efficiency: Option, manager: *mut c_void, } @@ -70,11 +74,20 @@ fn resolve_api() -> Option { if manager.is_null() { return None; } + // Optional (API 35): resolve if present, else `None` — the session still works without it. + let set_prefer_power_efficiency = + libc::dlsym(lib, c"APerformanceHint_setPreferPowerEfficiency".as_ptr()); + let set_prefer_power_efficiency = (!set_prefer_power_efficiency.is_null()).then(|| { + std::mem::transmute::<*mut c_void, SetPreferPowerEfficiencyFn>( + set_prefer_power_efficiency, + ) + }); Some(Api { create_session: std::mem::transmute::<*mut c_void, CreateSessionFn>(create_session), report: std::mem::transmute::<*mut c_void, ReportFn>(report), update_target: std::mem::transmute::<*mut c_void, UpdateTargetFn>(update_target), close: std::mem::transmute::<*mut c_void, CloseFn>(close), + set_prefer_power_efficiency, manager, }) } @@ -103,6 +116,13 @@ impl HintSession { if session.is_null() { return None; } + // Tell the governor NOT to bias this session toward power efficiency (API 35+): our loop is + // latency-critical, so we want it kept on fast cores at high clocks over battery savings. + // Best-effort; absent below API 35. + if let Some(f) = api.set_prefer_power_efficiency { + // SAFETY: `session` is the live session just created; the fn takes it + a bool. + unsafe { f(session, false) }; + } Some(Self { api, session }) } diff --git a/clients/android/native/src/audio.rs b/clients/android/native/src/audio.rs index 0c6b3e3..4ea0a63 100644 --- a/clients/android/native/src/audio.rs +++ b/clients/android/native/src/audio.rs @@ -18,8 +18,8 @@ //! grown on XRuns (Google's anti-glitch technique). use ndk::audio::{ - AudioCallbackResult, AudioDirection, AudioFormat, AudioPerformanceMode, AudioSharingMode, - AudioStream, AudioStreamBuilder, + AudioCallbackResult, AudioContentType, AudioDirection, AudioFormat, AudioPerformanceMode, + AudioSharingMode, AudioStream, AudioStreamBuilder, AudioUsage, }; use punktfunk_core::client::NativeClient; use punktfunk_core::error::PunktfunkError; @@ -235,6 +235,11 @@ impl AudioPlayback { // captures + Opus-encodes in exactly this order. .channel_count(channels as i32) .format(AudioFormat::PCM_Float) + // Tag the stream as game audio (usage=Game / content=Movie): the audio HAL applies + // its low-latency game-audio routing/policy and it's grouped correctly with the + // game-mode profile. Advisory — ignored where the device has no such policy. + .usage(AudioUsage::Game) + .content_type(AudioContentType::Movie) .performance_mode(AudioPerformanceMode::LowLatency) .sharing_mode(sharing) .data_callback(Box::new(callback)) diff --git a/clients/android/native/src/decode.rs b/clients/android/native/src/decode.rs index 77b9f62..9e9e1b2 100644 --- a/clients/android/native/src/decode.rs +++ b/clients/android/native/src/decode.rs @@ -8,8 +8,8 @@ use ndk::data_space::DataSpace; use ndk::media::media_codec::{ - DequeuedInputBufferResult, DequeuedOutputBufferInfoResult, MediaCodec, MediaCodecDirection, - OutputBuffer, + AsyncNotifyCallback, DequeuedInputBufferResult, DequeuedOutputBufferInfoResult, MediaCodec, + MediaCodecDirection, OutputBuffer, }; use ndk::media::media_format::MediaFormat; use ndk::native_window::NativeWindow; @@ -19,9 +19,14 @@ use punktfunk_core::session::Frame; use std::collections::VecDeque; use std::ffi::c_void; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; +use std::sync::{mpsc, Arc, Mutex}; use std::time::{Duration, Instant}; +/// Cap on AUs parked in the async loop awaiting a free codec input slot. Matches the connector's +/// own frame-channel depth; on sustained overflow the oldest is dropped and a keyframe requested +/// (same recovery as a reassembler drop). In steady state this stays near-empty. +const FRAME_PARK_CAP: usize = 16; + /// Cap on the pts→received-timestamp map below: MediaCodec holds only a handful of frames in /// flight, so anything beyond this is stale (codec flushed / HUD toggled) and gets evicted. const IN_FLIGHT_CAP: usize = 64; @@ -31,29 +36,80 @@ const IN_FLIGHT_CAP: usize = 64; /// this deep is a lost datagram (or an old host that never sends any) and gets evicted. const PENDING_SPLIT_CAP: usize = 256; -/// The decode loop. Runs on the `pf-decode` thread until `shutdown` is set or the session closes. +/// Whether to run the event-driven async decode loop (default) or the synchronous poll loop kept as +/// a bring-up fallback. Flip to `false` to A/B the two on the HUD (`design/…`); the async loop +/// presents a decoded frame the instant it's ready instead of waiting out a poll interval. +const USE_ASYNC_DECODE: bool = true; + +/// Per-session decode configuration, resolved by the JNI layer (`nativeStartVideo`) and passed to +/// the decode loop. Bundled so the loop entry points don't sprout a wide argument list. +pub(crate) struct DecodeOptions { + /// The decoder Kotlin ranked from `MediaCodecList` (`VideoDecoders.pickDecoder`). `None`/empty ⇒ + /// let the platform resolve the default decoder for the MIME. + pub decoder_name: Option, + /// Whether Kotlin found the chosen decoder advertises `FEATURE_LowLatency` (queryable only via + /// the Java `CodecCapabilities` API) — surfaced on the HUD next to the decoder name. + pub ll_feature: bool, + /// The user's "Low-latency mode" master toggle (default on ⇒ full aggressive profile; off ⇒ + /// conservative, an escape hatch for a device that throttles under the clocks). + pub low_latency_mode: bool, + /// TV form factor (Kotlin's `UiModeManager`): actively drive the HDMI output into the stream's + /// refresh mode, vs. the softer seamless hint on a phone/tablet. + pub is_tv: bool, +} + +/// The decode entry point on the `pf-decode` thread: dispatches to the async or synchronous loop. +/// Both run until `shutdown` is set or the session closes. pub fn run( client: Arc, window: NativeWindow, shutdown: Arc, stats: Arc, + opts: DecodeOptions, ) { + if USE_ASYNC_DECODE { + run_async(client, window, shutdown, stats, opts); + } else { + run_sync(client, window, shutdown, stats, opts); + } +} + +/// The synchronous poll loop — the original decode path, kept as a bring-up fallback behind +/// [`USE_ASYNC_DECODE`]. Feeds and drains on this one thread; the only blocking wait is a short +/// output dequeue while input is backed up. +#[allow(dead_code)] +fn run_sync( + client: Arc, + window: NativeWindow, + shutdown: Arc, + stats: Arc, + opts: DecodeOptions, +) { + let DecodeOptions { + decoder_name, + ll_feature, + low_latency_mode, + is_tv, + } = opts; boost_thread_priority(); let mode = client.mode(); - // The MediaCodec MIME for the codec the host resolved (`Welcome.codec`): HEVC or H.264. AMediaCodec - // needs no out-of-band extradata — the in-band VPS/SPS/PPS on every IDR configure it either way. - let mime = match client.codec { - punktfunk_core::quic::CODEC_H264 => "video/avc", - _ => "video/hevc", - }; - let codec = match MediaCodec::from_decoder_type(mime) { + // The MediaCodec MIME for the codec the host resolved (`Welcome.codec`). AMediaCodec needs no + // out-of-band extradata — the in-band VPS/SPS/PPS on every IDR configure it either way. + let mime = codec_mime(client.codec); + let codec = match create_codec(mime, decoder_name.as_deref()) { Some(c) => c, None => { log::error!("decode: no {mime} decoder on this device"); return; } }; - log::info!("decode: codec mime = {mime}"); + // The decoder's *actual* resolved name (Kotlin's pick, or the platform default when it fell + // back) drives both the HUD label and which vendor low-latency keys apply below. + let codec_name = codec.name().unwrap_or_default(); + stats.set_decoder(&codec_name, ll_feature); + log::info!( + "decode: codec mime = {mime}, decoder = {codec_name} (low-latency feature: {ll_feature})" + ); let mut format = MediaFormat::new(); format.set_str("mime", mime); @@ -64,23 +120,9 @@ pub fn run( "max-input-size", (mode.width * mode.height).max(2_000_000) as i32, ); - // Ask for the low-latency decode path where the decoder supports it (no reordering buffer). - format.set_i32("low-latency", 1); - // Best-effort vendor twin of the standard key: older Qualcomm decoders only honor their own - // extension. Unknown keys are ignored by other vendors' codecs, so this is safe to set blind. - format.set_i32("vendor.qti-ext-dec-low-latency.enable", 1); - // Advisory low-latency hints (KEY_PRIORITY / KEY_OPERATING_RATE), ignored where unsupported: - // realtime priority + the target frame rate, so vendor decoders (e.g. Qualcomm) run at full - // clocks instead of a power-saving cadence that adds dequeue latency. - format.set_i32("priority", 0); // 0 = realtime - // Operating rate = the codec's clock hint. Setting it to the display rate merely asks the - // decoder to *sustain* that cadence — a Qualcomm decoder can meet 60/120 fps at a power-saving - // clock that adds a millisecond-plus of decode latency per frame. Setting it to the AOSP - // "unbounded" sentinel (Short.MAX) instead asks the decoder to run each frame at max clocks and - // finish ASAP, minimising per-frame decode latency — the right trade for a real-time stream - // (costs power/heat; the dial to lower if a device thermally throttles over a long session). - // Ignored where unsupported. - format.set_i32("operating-rate", i16::MAX as i32); // 32767 = "as fast as possible" + // Standard + per-SoC vendor low-latency keys and the clock hints, gated on the resolved decoder + // name and the master toggle (see `configure_low_latency`). + configure_low_latency(&mut format, &codec_name, low_latency_mode); // HDR static metadata (ST.2086 mastering + content light level): when an HDR session was // negotiated, set KEY_HDR_STATIC_INFO so the display tone-maps from the source's real grade. @@ -118,7 +160,7 @@ pub fn run( // above our API-28 floor, so we resolve it at runtime (see `try_set_frame_rate`) rather than link // it — a hard import would stop `libpunktfunk_android.so` loading at all on API 28/29. Absent // there ⇒ we simply skip the hint (non-fatal; the stream renders fine without it). - if mode.refresh_hz > 0 && !try_set_frame_rate(&window, mode.refresh_hz as f32) { + if mode.refresh_hz > 0 && !try_set_frame_rate(&window, mode.refresh_hz as f32, is_tv) { log::debug!( "decode: set_frame_rate({} Hz) unavailable/declined (non-fatal)", mode.refresh_hz @@ -277,6 +319,7 @@ pub fn run( // or where the platform declines → `None`, and the loop runs unhinted). hint_tried = true; let tids = client.hot_thread_ids(); + boost_hot_threads(&tids); hint = crate::adpf::HintSession::create(frame_period_ns, &tids); log::info!( "decode: ADPF hint session {} — {} hot thread(s), target {frame_period_ns} ns", @@ -326,6 +369,609 @@ fn now_realtime_ns() -> i128 { .unwrap_or(0) } +/// The MediaCodec MIME for the codec the host resolved (`Welcome.codec`). Shared by the decode +/// thread and `nativeVideoMime` (which tells Kotlin what to rank decoders for). AV1 uses the +/// AOSP `video/av01` type; anything not H.264/AV1 is treated as HEVC (every pre-negotiation host +/// emitted HEVC). +pub(crate) fn codec_mime(codec: u8) -> &'static str { + match codec { + punktfunk_core::quic::CODEC_H264 => "video/avc", + punktfunk_core::quic::CODEC_AV1 => "video/av01", + _ => "video/hevc", + } +} + +/// Create the decoder: prefer the specific codec Kotlin ranked from `MediaCodecList` +/// (`from_codec_name`), falling back to the platform's default decoder for the MIME +/// (`from_decoder_type`) if that name can't be created (codec busy / renamed across an OS update). +fn create_codec(mime: &str, preferred: Option<&str>) -> Option { + if let Some(name) = preferred.filter(|n| !n.is_empty()) { + if let Some(c) = MediaCodec::from_codec_name(name) { + return Some(c); + } + log::warn!( + "decode: from_codec_name({name}) failed — falling back to default {mime} decoder" + ); + } + MediaCodec::from_decoder_type(mime) +} + +/// Apply the low-latency MediaFormat keys for `codec_name`. The standard AOSP `low-latency` key is +/// always set (API 30+, harmless/ignored elsewhere). When `aggressive` (the "Low-latency mode" +/// master toggle) we additionally set MediaTek's `vdec-lowlatency` (unconditionally — ignored off +/// MediaTek), the per-SoC vendor extension keys (gated on the decoder-name prefix the way +/// Moonlight-Android does, since a key one vendor honours is meaningless on another), and one clock +/// hint. Off ⇒ the standard key only, a gentler profile for a device that throttles under max clocks. +/// +/// Vendor keys mirror Moonlight's `MediaCodecHelper` (verified against current source): Qualcomm +/// picture-order + low-latency, Exynos (also Google Tensor), Amlogic, HiSilicon, MediaTek. NVIDIA +/// Tegra / Rockchip / Realtek expose no such key (nor does Moonlight) — they're covered by the +/// standard key + clock hint + being ranked first in `VideoDecoders`. +fn configure_low_latency(format: &mut MediaFormat, codec_name: &str, aggressive: bool) { + // Standard key: request the no-reorder low-latency path where the platform decoder supports it. + format.set_i32("low-latency", 1); + if !aggressive { + return; + } + // MediaTek's low-latency key — very common (mid/budget phones + many Google TV / Fire TV boxes). + // Set unconditionally like the standard key: MediaTek decoders honour it, others ignore it, so it + // covers MediaTek whatever the exact decoder name (omx.mtk / c2.mtk / an OEM rename). Moonlight + // does the same, and also relies on it for Amazon's Amlogic fork. + format.set_i32("vdec-lowlatency", 1); + let name = codec_name.to_ascii_lowercase(); + let is = |prefix: &str| name.starts_with(prefix); + // Qualcomm Snapdragon (the most common phone SoC): picture-order forces decode-order output + // (kills the reorder buffer on decoders that predate the standard key); low-latency is the older + // vendor twin. + if is("omx.qcom") || is("c2.qti") { + format.set_i32("vendor.qti-ext-dec-picture-order.enable", 1); + format.set_i32("vendor.qti-ext-dec-low-latency.enable", 1); + } + // Samsung Exynos — also covers Google Tensor (Pixel 6+), whose hardware decoder is `c2.exynos.*`. + if is("omx.exynos") || is("c2.exynos") { + format.set_i32("vendor.rtc-ext-dec-low-latency.enable", 1); + } + // Amlogic — the Android TV boxes (onn 4K, Chromecast w/ Google TV, Homatics). + if is("omx.amlogic") || is("c2.amlogic") { + format.set_i32("vendor.low-latency.enable", 1); + } + // HiSilicon / Kirin (older Huawei; paired req/rdy keys). + if is("omx.hisi") || is("c2.hisi") { + format.set_i32( + "vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-req", + 1, + ); + format.set_i32( + "vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-rdy", + -1, + ); + } + // NVIDIA Tegra (Shield TV) and Rockchip/Realtek (budget TV boxes / smart TVs) expose no + // low-latency vendor key (Moonlight has none either) — their decoders are already low-latency + // oriented, so the standard `low-latency` key + the clock hint below + being ranked first + // (see `VideoDecoders`) is their treatment. + // + // Clock hint, mutually exclusive (matching Moonlight): the AOSP "unbounded" operating-rate + // sentinel (Short.MAX) tells the decoder to run each frame at max clocks and finish ASAP rather + // than pace to the frame rate — shaving per-frame decode latency at a power/heat cost. Only + // Qualcomm is known to handle the sentinel; every other vendor mis-paces on it, so they get the + // plain realtime `priority` hint instead. + if decoder_supports_max_operating_rate(&name) { + format.set_i32("operating-rate", i16::MAX as i32); // 32767 = "as fast as possible" + } else { + format.set_i32("priority", 0); // 0 = realtime + } +} + +/// Whether a decoder tolerates `operating-rate = Short.MAX` rather than regressing on it. Follows +/// Moonlight's allowlist: Qualcomm decoders honour the sentinel (the Adreno 620 generation is the +/// known exception Moonlight excludes by GPU model — undetectable from native code here, so it +/// rides the master toggle as its escape hatch). Other vendors fall back to the plain `priority` +/// hint above. +fn decoder_supports_max_operating_rate(name_lower: &str) -> bool { + name_lower.starts_with("omx.qcom") || name_lower.starts_with("c2.qti") +} + +/// One decoded output buffer ready to release: its codec buffer index + the pts the codec echoed +/// (from the output callback's `BufferInfo`), used to pair the `decode` HUD stat. +struct OutputReady { + index: usize, + pts_us: u64, +} + +/// Events the async decode loop reacts to. The codec's async-notify callbacks (which run on its +/// internal looper thread) push the codec ones; the feeder thread pushes `Au`. Each carries only +/// owned/`Copy` data so the callback closures satisfy the `Send` bound and never touch the codec. +enum DecodeEvent { + /// A received access unit from the feeder, ready to queue into the decoder. + Au(Frame), + /// An input buffer slot freed (index) — we can queue an AU into it. + InputAvailable(usize), + /// A decoded frame is ready (buffer index + echoed pts). + OutputAvailable { index: usize, pts_us: u64 }, + /// The output format changed — re-check the stream's colour signalling (HDR DataSpace). + FormatChanged, + /// The codec reported an error; `fatal` when neither recoverable nor transient. + Error { fatal: bool }, +} + +/// The event-driven async decode loop (default; see [`run`]/[`USE_ASYNC_DECODE`]). The codec drives +/// us: an async-notify callback fires the instant an input buffer frees or a frame finishes +/// decoding, so a decoded frame is presented immediately instead of waiting out a poll interval (the +/// latency the sync loop left on the table). The callbacks run on the codec's internal looper thread +/// and only *push events* — every `AMediaCodec` buffer op stays on this thread, which owns the codec, +/// sidestepping the self-reference that would arise from a callback calling back into the codec it's +/// stored in. A small `pf-decode-feed` thread blocks on the network so this loop never does. +fn run_async( + client: Arc, + window: NativeWindow, + shutdown: Arc, + stats: Arc, + opts: DecodeOptions, +) { + let DecodeOptions { + decoder_name, + ll_feature, + low_latency_mode, + is_tv, + } = opts; + boost_thread_priority(); + let mode = client.mode(); + let mime = codec_mime(client.codec); + let mut codec = match create_codec(mime, decoder_name.as_deref()) { + Some(c) => c, + None => { + log::error!("decode: no {mime} decoder on this device"); + return; + } + }; + let codec_name = codec.name().unwrap_or_default(); + stats.set_decoder(&codec_name, ll_feature); + log::info!( + "decode: codec mime = {mime}, decoder = {codec_name} (async, low-latency feature: {ll_feature})" + ); + + // The event channel: the callbacks + feeder push, this loop pulls. `Sender` is `Send`, so the + // callback closures (each capturing a clone) satisfy the async-notify `Send` bound. + let (ev_tx, ev_rx) = mpsc::channel::(); + // Install the callbacks BEFORE configure()/start() so we're in async mode from the first buffer. + // Each just forwards an index/flag — no codec access here (the codec owns these closures). + { + let out_tx = ev_tx.clone(); + let in_tx = ev_tx.clone(); + let fmt_tx = ev_tx.clone(); + let err_tx = ev_tx.clone(); + let cb = AsyncNotifyCallback { + on_input_available: Some(Box::new(move |idx| { + let _ = in_tx.send(DecodeEvent::InputAvailable(idx)); + })), + on_output_available: Some(Box::new(move |idx, info| { + let _ = out_tx.send(DecodeEvent::OutputAvailable { + index: idx, + pts_us: info.presentation_time_us().max(0) as u64, + }); + })), + on_format_changed: Some(Box::new(move |_fmt| { + let _ = fmt_tx.send(DecodeEvent::FormatChanged); + })), + on_error: Some(Box::new(move |e, code, _detail| { + let fatal = !code.is_recoverable() && !code.is_transient(); + log::warn!("decode: codec error {e:?} (fatal={fatal})"); + let _ = err_tx.send(DecodeEvent::Error { fatal }); + })), + }; + if let Err(e) = codec.set_async_notify_callback(Some(cb)) { + log::error!("decode: set_async_notify_callback failed: {e}"); + return; + } + } + + // Build the low-latency format (identical keys to the sync path). + let mut format = MediaFormat::new(); + format.set_str("mime", mime); + format.set_i32("width", mode.width as i32); + format.set_i32("height", mode.height as i32); + format.set_i32( + "max-input-size", + (mode.width * mode.height).max(2_000_000) as i32, + ); + configure_low_latency(&mut format, &codec_name, low_latency_mode); + if client.color.is_hdr() { + match client.next_hdr_meta(Duration::from_millis(250)) { + Ok(meta) => { + format.set_buffer("hdr-static-info", &android_hdr_static_info(&meta)); + log::info!("decode: HDR static metadata applied (KEY_HDR_STATIC_INFO)"); + } + Err(_) => { + log::info!("decode: HDR session but no mastering metadata yet — DataSpace only") + } + } + } + if let Err(e) = codec.configure(&format, Some(&window), MediaCodecDirection::Decoder) { + log::error!("decode: configure failed: {e}"); + return; + } + if let Err(e) = codec.start() { + log::error!("decode: start failed: {e}"); + return; + } + log::info!( + "decode: decoder started (async) at {}x{}", + mode.width, + mode.height + ); + if mode.refresh_hz > 0 && !try_set_frame_rate(&window, mode.refresh_hz as f32, is_tv) { + log::debug!( + "decode: set_frame_rate({} Hz) unavailable/declined (non-fatal)", + mode.refresh_hz + ); + } + + // Skew-corrected latency stats (spec: design/stats-unification.md). Receipt stamps (keyed by the + // pts we queue) live in a shared map: the feeder writes them at receipt, this loop pairs decoded + // output back to them. Behind a `Mutex` since two threads touch it — only ever locked while the + // HUD is visible. + let clock_offset = client.clock_offset_ns; + let in_flight = Arc::new(Mutex::new(VecDeque::<(u64, i128)>::new())); + + // Feeder thread: block on the network so this loop doesn't (an AU's arrival becomes an event that + // wakes us immediately, with no input-side poll latency). It also records the `received` HUD stat. + let feeder = { + let client = client.clone(); + let stats = stats.clone(); + let in_flight = in_flight.clone(); + let shutdown = shutdown.clone(); + let ev_tx = ev_tx.clone(); + std::thread::Builder::new() + .name("pf-decode-feed".into()) + .spawn(move || { + feeder_loop( + client, + stats, + in_flight, + clock_offset as i128, + shutdown, + ev_tx, + ); + }) + .ok() + }; + drop(ev_tx); // only the feeder + callbacks keep the channel alive now + + // ADPF: same as the sync path — register this thread now, create the session lazily on the first + // presented frame (by when the pump + audio + feeder threads have registered their tids too). + let frame_period_ns = if mode.refresh_hz > 0 { + 1_000_000_000i64 / mode.refresh_hz as i64 + } else { + 0 + }; + client.register_hot_thread(); + let mut hint: Option = None; + let mut hint_tried = false; + + let mut free_inputs: VecDeque = VecDeque::new(); + let mut pending_aus: VecDeque = VecDeque::new(); + let mut ready: Vec = Vec::new(); + let mut applied_ds: Option = None; + let mut fed: u64 = 0; + let mut rendered: u64 = 0; + let mut discarded: u64 = 0; + let mut last_dropped = client.frames_dropped(); + let mut last_kf_req: Option = None; + // Productive (dispatch+feed+present) time between displayed frames; reported to ADPF once one is + // presented. The blocking event wait is excluded (idle, not work) — same accounting as the sync loop. + let mut work_accum_ns: i64 = 0; + let mut fatal = false; + + while !shutdown.load(Ordering::Relaxed) && !fatal { + // Block for the next event (idle wait — excluded from the work tally). The short timeout + // drives loss-recovery housekeeping when the pipeline is momentarily quiet. + let ev0 = match ev_rx.recv_timeout(Duration::from_millis(5)) { + Ok(ev) => Some(ev), + Err(mpsc::RecvTimeoutError::Timeout) => None, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + let work_t0 = Instant::now(); + let mut fmt_dirty = false; + let mut au_dropped = false; + if let Some(ev) = ev0 { + au_dropped |= dispatch_event( + ev, + &mut pending_aus, + &mut free_inputs, + &mut ready, + &mut fmt_dirty, + &mut fatal, + ); + } + // Coalesce every other event already queued into this one work pass — correct newest-only + // presentation across a decode burst, and batched feeding. + while let Ok(ev) = ev_rx.try_recv() { + au_dropped |= dispatch_event( + ev, + &mut pending_aus, + &mut free_inputs, + &mut ready, + &mut fmt_dirty, + &mut fatal, + ); + } + if fmt_dirty { + apply_hdr_dataspace(&codec, &window, &mut applied_ds); + } + feed_ready(&codec, &mut pending_aus, &mut free_inputs, &mut fed); + let had_output = !ready.is_empty(); + present_ready( + &codec, + &mut ready, + &stats, + &in_flight, + clock_offset, + &mut rendered, + &mut discarded, + ); + + work_accum_ns += work_t0.elapsed().as_nanos() as i64; + if had_output { + if !hint_tried { + hint_tried = true; + let tids = client.hot_thread_ids(); + boost_hot_threads(&tids); + hint = crate::adpf::HintSession::create(frame_period_ns, &tids); + log::info!( + "decode: ADPF hint session {} — {} hot thread(s), target {frame_period_ns} ns", + if hint.is_some() { + "active" + } else { + "unavailable" + }, + tids.len(), + ); + } + if let Some(h) = &hint { + h.report_actual(work_accum_ns); + } + work_accum_ns = 0; + if rendered > 0 && rendered % 300 == 0 { + log::info!("decode: fed={fed} rendered={rendered} discarded={discarded}"); + } + } + // Loss recovery: request an IDR when the reassembler's unrecoverable-drop count climbs (or we + // dropped a parked AU on overflow), throttled so a multi-frame recovery gap doesn't flood the + // control stream. + let dropped = client.frames_dropped(); + if dropped > last_dropped || au_dropped { + last_dropped = dropped; + let now = Instant::now(); + if last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) { + last_kf_req = Some(now); + let _ = client.request_keyframe(); + } + } + } + + let _ = codec.stop(); + shutdown.store(true, Ordering::SeqCst); // ensure the feeder wakes and exits, then join it + if let Some(j) = feeder { + let _ = j.join(); + } + log::info!("decode: stopped (async, fed={fed} rendered={rendered} discarded={discarded})"); +} + +/// The `pf-decode-feed` thread: block on the connector for the next access unit so the async loop +/// never has to. Records the `received` HUD stat (receipt point) — including the Phase-2 host/network +/// split from any matching 0xCF host timings — then hands the AU to the loop via the event channel. +/// Exits when `shutdown` is set, the session closes, or the loop's receiver is gone. +fn feeder_loop( + client: Arc, + stats: Arc, + in_flight: Arc>>, + clock_offset: i128, + shutdown: Arc, + ev_tx: mpsc::Sender, +) { + // Received AUs awaiting their 0xCF host timing (Phase-2 split), as (pts_ns, capture→received µs). + let mut pending_split: VecDeque<(u64, u64)> = VecDeque::new(); + while !shutdown.load(Ordering::Relaxed) { + match client.next_frame(Duration::from_millis(5)) { + Ok(frame) => { + if stats.enabled() { + let received_ns = now_realtime_ns(); + let lat_ns = received_ns + clock_offset - frame.pts_ns as i128; + let lat_us = + (lat_ns > 0 && lat_ns < 10_000_000_000).then_some((lat_ns / 1000) as u64); + stats.note_received(frame.data.len(), lat_us, clock_offset != 0); + { + let mut g = in_flight + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + g.push_back((frame.pts_ns / 1000, received_ns)); + if g.len() > IN_FLIGHT_CAP { + g.pop_front(); // stale — codec never echoed it back + } + } + if let Some(hostnet_us) = lat_us { + pending_split.push_back((frame.pts_ns, hostnet_us)); + if pending_split.len() > PENDING_SPLIT_CAP { + pending_split.pop_front(); + } + } + while let Ok(t) = client.next_host_timing(Duration::ZERO) { + if let Some(i) = pending_split.iter().position(|&(p, _)| p == t.pts_ns) { + let (_, hostnet_us) = pending_split.remove(i).unwrap(); + stats.note_host_split( + t.host_us as u64, + hostnet_us.saturating_sub(t.host_us as u64), + ); + } + } + } + if ev_tx.send(DecodeEvent::Au(frame)).is_err() { + break; // the decode loop is gone + } + } + Err(PunktfunkError::NoFrame) => {} // timeout — re-check shutdown and poll again + Err(_) => break, // session closed + } + } +} + +/// Route one [`DecodeEvent`] into the loop's working sets. Returns `true` only when a parked AU was +/// dropped on overflow (the caller then requests a keyframe). +fn dispatch_event( + ev: DecodeEvent, + pending_aus: &mut VecDeque, + free_inputs: &mut VecDeque, + ready: &mut Vec, + fmt_dirty: &mut bool, + fatal: &mut bool, +) -> bool { + match ev { + DecodeEvent::Au(f) => { + pending_aus.push_back(f); + if pending_aus.len() > FRAME_PARK_CAP { + pending_aus.pop_front(); // sustained overflow — drop oldest, signal a keyframe request + return true; + } + } + DecodeEvent::InputAvailable(i) => free_inputs.push_back(i), + DecodeEvent::OutputAvailable { index, pts_us } => ready.push(OutputReady { index, pts_us }), + DecodeEvent::FormatChanged => *fmt_dirty = true, + DecodeEvent::Error { fatal: f } => { + if f { + *fatal = true; + } + } + } + false +} + +/// Queue as many parked AUs as there are free input buffer slots (async mode: the indices come from +/// `InputAvailable` callbacks, not a dequeue). Each AU is copied into its codec input buffer and +/// submitted; a too-large AU is truncated (logged) rather than dropped. +fn feed_ready( + codec: &MediaCodec, + pending_aus: &mut VecDeque, + free_inputs: &mut VecDeque, + fed: &mut u64, +) { + while !pending_aus.is_empty() && !free_inputs.is_empty() { + let idx = free_inputs.pop_front().unwrap(); + let frame = pending_aus.pop_front().unwrap(); + let pts_us = frame.pts_ns / 1000; + let Some(dst) = codec.input_buffer(idx) else { + log::warn!("decode: input_buffer({idx}) returned None — dropping AU"); + continue; + }; + let au = &frame.data; + let n = au.len().min(dst.len()); + if n < au.len() { + log::warn!( + "decode: AU {} > input buffer {}, truncated", + au.len(), + dst.len() + ); + } + // SAFETY: `au` (wire AU) and `dst` (codec input buffer) are distinct allocations, both valid + // for `n` bytes; `MaybeUninit` is layout-identical to `u8`, so this initializes dst[..n]. + unsafe { + std::ptr::copy_nonoverlapping(au.as_ptr(), dst.as_mut_ptr().cast::(), n); + } + if let Err(e) = codec.queue_input_buffer_by_index(idx, 0, n, pts_us, 0) { + log::warn!("decode: queue_input_buffer_by_index: {e}"); + } else { + *fed += 1; + } + } +} + +/// Present only the NEWEST ready output (render = true) and release the rest without rendering — a +/// burst of stale frames on glass is worse than skipping to the freshest (the sync loop's newest-ready +/// policy, callback-driven). Every dequeued buffer, rendered or not, is the HUD's `decoded` +/// measurement point (it finished decoding either way); samples are recorded in pts order so the +/// receipt-map eviction stays monotonic. `ready` is drained. +fn present_ready( + codec: &MediaCodec, + ready: &mut Vec, + stats: &crate::stats::VideoStats, + in_flight: &Mutex>, + clock_offset: i64, + rendered: &mut u64, + discarded: &mut u64, +) { + if ready.is_empty() { + return; + } + if stats.enabled() { + let mut g = in_flight + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for o in ready.iter() { + note_decoded_pts(stats, &mut g, clock_offset, o.pts_us); + } + } + let last = ready.len() - 1; + for (i, o) in ready.drain(..).enumerate() { + let render = i == last; + match codec.release_output_buffer_by_index(o.index, render) { + Ok(()) if render => *rendered += 1, + Ok(()) => *discarded += 1, + Err(e) => { + log::warn!( + "decode: release_output_buffer_by_index({}, {render}): {e}", + o.index + ) + } + } + } +} + +/// React to an output-format change by signalling the stream's HDR dataspace on the Surface (SDR +/// streams leave the default alone). The AMediaCodec analogue of the sync loop's `OutputFormatChanged` +/// handling; safe to call repeatedly (`applied_ds` dedups). +fn apply_hdr_dataspace( + codec: &MediaCodec, + window: &NativeWindow, + applied_ds: &mut Option, +) { + if let Some(ds) = hdr_dataspace(codec) { + if *applied_ds != Some(ds) { + match window.set_buffers_data_space(ds) { + Ok(()) => { + *applied_ds = Some(ds); + log::info!("decode: HDR stream → Surface dataspace {ds}"); + } + Err(e) => { + log::warn!("decode: set_buffers_data_space({ds}) failed (non-fatal): {e}") + } + } + } + } +} + +/// Raise the pipeline's OTHER hot threads — the core's data-plane pump (UDP receive + FEC +/// reassembly) and the audio decode thread — toward the display band, matching this decode thread's +/// own boost. `setpriority(PRIO_PROCESS, tid)` targets any task in the process, so we do it from +/// here once their tids are known (the same set ADPF hints), without a per-platform priority hook +/// in the shared core. Slightly below the decode thread's -10 so the display path still wins. +/// Best-effort; skips this thread (already boosted) and is non-fatal if the platform refuses. +fn boost_hot_threads(tids: &[i32]) { + // SAFETY: `gettid` is an always-safe syscall on the calling thread. + let self_tid = unsafe { libc::gettid() }; + for &tid in tids { + if tid == self_tid { + continue; + } + // SAFETY: `setpriority` with PRIO_PROCESS + a live tid in our own process is an always-safe + // syscall; a refusal is reported via the return value, not UB. + unsafe { + if libc::setpriority(libc::PRIO_PROCESS, tid as libc::id_t, -8) != 0 { + log::debug!("decode: setpriority(-8) on hot tid {tid} failed (non-fatal)"); + } + } + } +} + /// Best-effort: raise the decode thread toward Android's URGENT_DISPLAY band so background work /// can't preempt it under load (which shows up as late/dropped frames). Non-fatal if the platform /// refuses (foreground apps may set their own threads; the exact floor is policy-dependent). @@ -343,23 +989,48 @@ fn boost_thread_priority() { } } -/// `ANativeWindow_setFrameRate` (NDK **API 30**) resolved from `libandroid.so` at runtime, so the lib -/// still loads on our API-28 floor — a hard import of a >floor symbol makes `dlopen`/`System.load` -/// fail on every API-28/29 device, even where this path is never hit. Mirrors the dlsym approach in -/// [`crate::adpf`]. Returns `true` when the platform accepted the hint; `false` on API < 30 (symbol -/// absent) or when the platform declined. `compatibility` is fixed to the DEFAULT (0) policy. -fn try_set_frame_rate(window: &NativeWindow, frame_rate: f32) -> bool { +/// Set the surface's frame-rate hint to the stream's refresh so SurfaceFlinger picks a matching +/// display mode and aligns vsync (no 60-in-120 judder). Both NDK entry points sit above our API-28 +/// floor, so both are dlsym-resolved at runtime (a hard import of a >floor symbol makes +/// `dlopen`/`System.load` fail on every API-28/29 device, even where this path is never hit — +/// mirrors [`crate::adpf`]): +/// - On a **TV** (`is_tv`): `ANativeWindow_setFrameRateWithChangeStrategy` (**API 31**) with +/// `changeFrameRateStrategy = ALWAYS`, which actively drives the HDMI output into the matching +/// mode (e.g. 60↔120) instead of leaving the panel at its default and judder-matching. The +/// forced switch may blank the panel briefly — acceptable once at stream start, not wanted on a +/// phone. Falls through to the 2-arg hint on API 30. +/// - Otherwise: `ANativeWindow_setFrameRate` (**API 30**) with `compatibility = DEFAULT` — the +/// softer, seamless-preferred hint for phones/tablets and the universal fallback. +/// +/// Returns `true` when the platform accepted a hint; `false` on API < 30 (symbols absent) or a +/// decline. +fn try_set_frame_rate(window: &NativeWindow, frame_rate: f32, is_tv: bool) -> bool { // int32_t ANativeWindow_setFrameRate(ANativeWindow*, float frameRate, int8_t compatibility) type SetFrameRateFn = unsafe extern "C" fn(*mut c_void, f32, i8) -> i32; + // int32_t ANativeWindow_setFrameRateWithChangeStrategy( + // ANativeWindow*, float frameRate, int8_t compatibility, int8_t changeFrameRateStrategy) + type SetFrameRateStrategyFn = unsafe extern "C" fn(*mut c_void, f32, i8, i8) -> i32; // SAFETY: `dlopen` of the always-mapped `libandroid.so` (only bumps its refcount; never closed — - // process-lifetime handle). `dlsym` returns null when the symbol is absent (device API < 30), - // checked before transmuting the non-null pointer to its fn-pointer type. `window.ptr()` is the - // live `ANativeWindow` this `NativeWindow` owns for the call's duration. + // process-lifetime handle). Each `dlsym` returns null when the symbol is absent (device below the + // symbol's API level), checked before transmuting the non-null pointer to its fn-pointer type. + // `window.ptr()` is the live `ANativeWindow` this `NativeWindow` owns for the call's duration. unsafe { let lib = libc::dlopen(c"libandroid.so".as_ptr(), libc::RTLD_NOW); if lib.is_null() { return false; } + // TV: prefer the API-31 change-strategy form to force the mode switch (strategy 1 = ALWAYS, + // compatibility 0 = DEFAULT). Absent on API 30 ⇒ fall through to the 2-arg hint below. + if is_tv { + let sym = libc::dlsym( + lib, + c"ANativeWindow_setFrameRateWithChangeStrategy".as_ptr(), + ); + if !sym.is_null() { + let set = std::mem::transmute::<*mut c_void, SetFrameRateStrategyFn>(sym); + return set(window.ptr().as_ptr().cast(), frame_rate, 0, 1) == 0; + } + } let sym = libc::dlsym(lib, c"ANativeWindow_setFrameRate".as_ptr()); if sym.is_null() { return false; // device API < 30 — no per-surface frame-rate hint @@ -499,7 +1170,22 @@ fn note_decoded( clock_offset: i64, buf: &OutputBuffer<'_>, ) { - let pts_us = buf.info().presentation_time_us().max(0) as u64; + note_decoded_pts( + stats, + in_flight, + clock_offset, + buf.info().presentation_time_us().max(0) as u64, + ); +} + +/// The [`note_decoded`] body keyed by the echoed `presentationTimeUs` directly — the async loop has +/// the pts (from the output callback's `BufferInfo`) but no borrowed `OutputBuffer`, so it calls this. +fn note_decoded_pts( + stats: &crate::stats::VideoStats, + in_flight: &mut VecDeque<(u64, i128)>, + clock_offset: i64, + pts_us: u64, +) { let decoded_ns = now_realtime_ns(); // Pair the echoed pts back to its receipt stamp, evicting stale (older) entries as we go. let mut received_ns = None; diff --git a/clients/android/native/src/session/planes.rs b/clients/android/native/src/session/planes.rs index 960db7d..a5b32b5 100644 --- a/clients/android/native/src/session/planes.rs +++ b/clients/android/native/src/session/planes.rs @@ -2,20 +2,31 @@ //! ~1 Hz decode-stats drain for the HUD. use jni::objects::JObject; -use jni::sys::{jboolean, jdoubleArray, jlong, jsize}; +// Used only by the android-gated `nativeStartVideo`; on the host build that fn is cfg'd out. +#[cfg(target_os = "android")] +use jni::objects::JString; +use jni::sys::{jboolean, jdoubleArray, jlong, jsize, jstring}; use jni::JNIEnv; use super::{jni_guard, SessionHandle}; -/// `NativeBridge.nativeStartVideo(handle, surface)` — wrap the SurfaceView's `Surface` as an -/// `ANativeWindow` and start the HEVC decode thread rendering onto it. No-op if already started. +/// `NativeBridge.nativeStartVideo(handle, surface, decoderName, lowLatencyMode, lowLatencyFeature)` +/// — wrap the SurfaceView's `Surface` as an `ANativeWindow` and start the decode thread rendering +/// onto it. `decoderName` is the codec Kotlin ranked from `MediaCodecList` (`""` = let the platform +/// resolve the default for the MIME); `lowLatencyMode` is the user's master toggle; +/// `lowLatencyFeature` is whether that decoder advertised `FEATURE_LowLatency` (HUD label only). +/// No-op if already started. #[cfg(target_os = "android")] #[no_mangle] pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartVideo( - env: JNIEnv, + mut env: JNIEnv, _this: JObject, handle: jlong, surface: JObject, + decoder_name: JString, + low_latency_mode: jboolean, + ll_feature: jboolean, + is_tv: jboolean, ) { use super::VideoThread; use std::sync::atomic::AtomicBool; @@ -24,6 +35,12 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartVideo( if handle == 0 { return; } + // The decoder name Kotlin picked (empty string / read failure ⇒ None ⇒ default resolver). + let decoder = env + .get_string(&decoder_name) + .ok() + .map(String::from) + .filter(|s| !s.is_empty()); // SAFETY: live handle per the nativeConnect/nativeClose contract. let h = unsafe { &*(handle as *const SessionHandle) }; let mut guard = h.video.lock().unwrap(); @@ -48,13 +65,67 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartVideo( let client = h.client.clone(); let sd = shutdown.clone(); let st = h.stats.clone(); // session-lifetime stats (gate survives surface recreate) + let opts = crate::decode::DecodeOptions { + decoder_name: decoder, + ll_feature: ll_feature != 0, + low_latency_mode: low_latency_mode != 0, + is_tv: is_tv != 0, + }; let join = std::thread::Builder::new() .name("pf-decode".into()) - .spawn(move || crate::decode::run(client, window, sd, st)) + .spawn(move || crate::decode::run(client, window, sd, st, opts)) .ok(); *guard = Some(VideoThread { shutdown, join }); } +/// `NativeBridge.nativeVideoMime(handle): String` — the MediaCodec MIME for the codec the host +/// resolved (`"video/hevc"` / `"video/avc"` / `"video/av01"`), so Kotlin can rank `MediaCodecList` +/// decoders for it before calling [`Java_io_unom_punktfunk_kit_NativeBridge_nativeStartVideo`]. +/// Empty string on a `0` handle. Cheap; safe on the UI thread. +#[cfg(target_os = "android")] +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoMime<'local>( + env: JNIEnv<'local>, + _this: JObject<'local>, + handle: jlong, +) -> jstring { + jni_guard(std::ptr::null_mut(), || { + if handle == 0 { + return std::ptr::null_mut(); + } + // SAFETY: live handle per the nativeConnect/nativeClose contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + match env.new_string(crate::decode::codec_mime(h.client.codec)) { + Ok(s) => s.into_raw(), + Err(_) => std::ptr::null_mut(), + } + }) +} + +/// `NativeBridge.nativeVideoDecoderLabel(handle): String` — the resolved decoder identity for the +/// HUD, e.g. `c2.qti.avc.decoder · low-latency`, or `""` before the decode thread has resolved one. +/// One-shot (the decoder is fixed for the session); poll once after the HUD appears. Not +/// android-gated — pure `jni` + a lock, so it links on the host build too (Kotlin only calls it on +/// device). +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoDecoderLabel<'local>( + env: JNIEnv<'local>, + _this: JObject<'local>, + handle: jlong, +) -> jstring { + jni_guard(std::ptr::null_mut(), || { + if handle == 0 { + return std::ptr::null_mut(); + } + // SAFETY: live handle per the nativeConnect/nativeClose contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + match env.new_string(h.stats.decoder_label()) { + Ok(s) => s.into_raw(), + Err(_) => std::ptr::null_mut(), + } + }) +} + /// `NativeBridge.nativeStopVideo(handle)` — stop + join the decode thread (without closing the /// session). No-op on `0`. #[no_mangle] diff --git a/clients/android/native/src/stats.rs b/clients/android/native/src/stats.rs index 45d28bb..ad2425b 100644 --- a/clients/android/native/src/stats.rs +++ b/clients/android/native/src/stats.rs @@ -22,9 +22,21 @@ pub struct VideoStats { /// they (and the caller's latency computation — see `enabled`) early-out on this flag alone. /// Off until Kotlin shows the HUD. enabled: AtomicBool, + /// The resolved decoder identity for the HUD: the codec's actual `AMediaCodec` name (e.g. + /// `c2.qti.avc.decoder`) and whether it advertised `FEATURE_LowLatency`. Set once when the + /// decode thread creates the codec (`set_decoder`), read one-shot by `nativeVideoDecoderLabel`. + /// Separate from `inner` (never touched per-frame) so naming it costs nothing on the hot path. + decoder: Mutex>, inner: Mutex, } +/// The chosen decoder's identity, surfaced on the stats HUD so before/after latency comparisons +/// name the codec that produced them. +struct DecoderInfo { + name: String, + low_latency: bool, +} + struct Inner { window_start: Instant, frames: u64, @@ -79,6 +91,7 @@ impl VideoStats { pub fn new() -> VideoStats { VideoStats { enabled: AtomicBool::new(false), + decoder: Mutex::new(None), inner: Mutex::new(Inner { window_start: Instant::now(), frames: 0, @@ -121,6 +134,36 @@ impl VideoStats { } } + /// Record the resolved decoder identity for the HUD — the codec's real `AMediaCodec` name and + /// whether it reported `FEATURE_LowLatency`. Called once from the decode thread right after the + /// codec is created (before `configure`), overwriting any prior value on a surface recreate. + // Set only by the android-only decode thread; unreferenced on the host build — expected. + #[cfg_attr(not(target_os = "android"), allow(dead_code))] + pub fn set_decoder(&self, name: &str, low_latency: bool) { + let mut g = self + .decoder + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *g = Some(DecoderInfo { + name: name.to_owned(), + low_latency, + }); + } + + /// The decoder label for the HUD, e.g. `c2.qti.avc.decoder · low-latency`, or `""` before the + /// decode thread has resolved one. Cheap (a lock + a string build); safe on the UI thread. + pub fn decoder_label(&self) -> String { + let g = self + .decoder + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match &*g { + Some(d) if d.low_latency => format!("{} · low-latency", d.name), + Some(d) => d.name.clone(), + None => String::new(), + } + } + /// Record one received access unit: its wire size and (if in range) its capture→received /// `host+network` stage sample. Receipt is the fps/goodput counting point per the spec. // Driven only by the android-only decode thread; unreferenced on the host build — expected. diff --git a/crates/punktfunk-core/src/transport/qos.rs b/crates/punktfunk-core/src/transport/qos.rs index 2ba095e..cde2420 100644 --- a/crates/punktfunk-core/src/transport/qos.rs +++ b/crates/punktfunk-core/src/transport/qos.rs @@ -66,12 +66,18 @@ impl MediaClass { } } -/// Whether DSCP/QoS marking is enabled (`PUNKTFUNK_DSCP=1`). Off by default. +/// Whether DSCP/QoS marking is enabled. Default **on for Android**, **off elsewhere**: on Wi-Fi +/// (where most Android clients live) access points commonly map DSCP to WMM access categories, so +/// tagging the video/audio sockets can win real airtime priority against other traffic on the link; +/// on the wired paths the other clients use it's rarely honoured and some paths bleach or reject +/// marked packets, so it stays opt-in there. `PUNKTFUNK_DSCP` overrides either way — `1`/`true`/`on` +/// forces it on, `0`/`false`/`off` forces it off (e.g. to rule QoS out while debugging a flaky AP). pub(crate) fn dscp_enabled() -> bool { - matches!( - std::env::var("PUNKTFUNK_DSCP").as_deref(), - Ok("1") | Ok("true") | Ok("on") - ) + match std::env::var("PUNKTFUNK_DSCP").as_deref() { + Ok("1") | Ok("true") | Ok("on") => true, + Ok("0") | Ok("false") | Ok("off") => false, + _ => cfg!(target_os = "android"), + } } /// Best-effort: tag `socket`'s outgoing packets for prioritized delivery of its media class. A no-op