From 87b6fa881345b1d0007297b0094e710c53111566 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Thu, 30 Jul 2026 19:47:27 +0200 Subject: [PATCH] feat(android): the panel is pinned to the stream's refresh, and touch skips the vsync batch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three quick latency wins for phones, ahead of the presenter rebuild: - setStreamDisplayMode: the window-level preferredDisplayModeId is pinned to the stream's refresh (exact rate, else the smallest integer multiple, else the highest available) for the session. The surface-level frame-rate hint alone is advisory and some OEM refresh governors (Nothing OS's LTPO logic among them) ignore it for third-party apps — leaving a 120 Hz session presenting on a 60/90 Hz panel. nativeVideoSize gained a trailing refreshHz element for this (old readers index only 0/1). TV keeps the native HDMI mode switch instead. - The surface hint itself now passes compatibility = FIXED_SOURCE on every form factor: the stream is fixed-rate video the client cannot re-pace; DEFAULT invited governors to not switch. - requestUnbufferedDispatch(SOURCE_CLASS_POINTER) on the hosting view while streaming: touch/pointer events were vsync-batched — up to a frame of input latency the stream shouldn't pay. - The HUD polls the panel's live refresh each second and flags '⚠ panel N Hz' when it sits below the stream rate, so an unpinned panel is visible instead of reading as inexplicable judder. Stale nativeStartVideo kdoc (low-latency 'off, the default') corrected — it defaults ON under low_latency_mode_v2. Co-Authored-By: Claude Fable 5 --- .../kotlin/io/unom/punktfunk/MainActivity.kt | 46 ++++++++++++++++++- .../kotlin/io/unom/punktfunk/StatsOverlay.kt | 12 ++++- .../kotlin/io/unom/punktfunk/StreamScreen.kt | 29 +++++++++++- .../io/unom/punktfunk/kit/NativeBridge.kt | 14 +++--- clients/android/native/src/decode/setup.rs | 20 +++++--- clients/android/native/src/session/planes.rs | 16 +++++-- 6 files changed, 115 insertions(+), 22 deletions(-) diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt index 91ecd366..83834d29 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt @@ -444,8 +444,8 @@ class MainActivity : ComponentActivity() { /** * Opt the CONSOLE UI into the panel's highest refresh mode. Some OEMs (Nothing OS among them) pin * third-party apps to 60Hz unless they explicitly ask for more, which halves the smoothness of the - * UI's scrolling/animation on a 120/144Hz panel. [StreamScreen] turns this OFF while streaming so - * its own `ANativeWindow_setFrameRate` (matched to the video) governs the panel instead. + * UI's scrolling/animation on a 120/144Hz panel. [StreamScreen] replaces this with + * [setStreamDisplayMode] while streaming (matched to the video, not to the panel maximum). */ fun setConsoleHighRefreshRate(high: Boolean) { if (highRefreshModeId == 0) return @@ -454,6 +454,48 @@ class MainActivity : ComponentActivity() { } } + /** + * Pin the panel to a display mode matching the STREAM's refresh for the session's duration — + * exact rate first, else the smallest integer multiple (120 for a 60 stream: judder-free 2:1 + * pulldown), else the highest available. Same-resolution modes only. + * + * The window-level mode pin is the belt to the decoder's `ANativeWindow_setFrameRate` braces: + * the surface hint alone is advisory, and several OEM refresh governors (Nothing OS's LTPO + * logic among them) ignore it entirely for third-party apps — leaving a 120 Hz session + * presenting on a 60/90 Hz panel, which reads as judder + a refresh of extra latency. The + * preferredDisplayModeId is the one signal they all honor. [hz] ≤ 0 falls back to releasing + * the pin (the pre-pin behaviour). + */ + fun setStreamDisplayMode(hz: Int) { + if (hz <= 0) { + setConsoleHighRefreshRate(false) + return + } + @Suppress("DEPRECATION") + val disp = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) display else windowManager.defaultDisplay + val current = disp?.mode ?: return + val sameRes = disp.supportedModes.filter { + it.physicalWidth == current.physicalWidth && it.physicalHeight == current.physicalHeight + } + fun multiple(rate: Float): Int { + val k = (rate / hz).toInt() + return if (k >= 2 && kotlin.math.abs(rate - hz * k) < 1f) k else 0 + } + val target = sameRes.minWithOrNull( + compareBy( + { + when { + kotlin.math.abs(it.refreshRate - hz) < 1f -> 0 // exact + multiple(it.refreshRate) > 0 -> 1 // integer multiple — prefer smallest + else -> 2 // no relation — prefer highest so at least nothing is halved + } + }, + { if (multiple(it.refreshRate) > 0) it.refreshRate else -it.refreshRate }, + ), + ) ?: return + window.attributes = window.attributes.apply { preferredDisplayModeId = target.modeId } + } + override fun dispatchKeyEvent(event: KeyEvent): Boolean { val handle = streamHandle if (handle != 0L) { diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt index 37887bdd..ba23c882 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt @@ -46,12 +46,20 @@ internal fun StatsOverlay( * common case: no profile) the line is exactly what it always was. */ profileName: String? = null, + /** + * The panel's live refresh rate (0 = unknown). Shown as a warning on the first line whenever + * it sits below the stream rate — the "an OEM governor ignored the mode pin" tell, which + * otherwise reads as inexplicable judder and an extra refresh of latency. + */ + panelHz: Float = 0f, modifier: Modifier = Modifier, ) { if (verbosity == StatsVerbosity.OFF || s.size < 10) return val w = s[6].toInt() val h = s[7].toInt() val hz = s[8].toInt() + val panelBelowStream = panelHz > 0f && hz > 0 && panelHz + 1f < hz.toFloat() + val panelTag = if (panelBelowStream) " ⚠ panel ${panelHz.roundToInt()} Hz" else "" val latValid = s[4] != 0.0 val skew = s[5] != 0.0 val lost = s[9].toLong() @@ -65,12 +73,12 @@ internal fun StatsOverlay( val profileTag = profileName?.let { " · $it" }.orEmpty() // Compact: everything the glance-value needs on one line, nothing else. if (verbosity == StatsVerbosity.COMPACT) { - statLine(compactLine(s, latValid) + profileTag, Color.White) + statLine(compactLine(s, latValid) + profileTag + panelTag, Color.White) return@Column } statLine( - "$w×$h@$hz ${s[0].roundToInt()} fps ${"%.1f".format(s[1])} Mb/s$profileTag", + "$w×$h@$hz ${s[0].roundToInt()} fps ${"%.1f".format(s[1])} Mb/s$profileTag$panelTag", Color.White, ) if (detailed && decoderLabel.isNotEmpty()) { diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt index f4f861d3..537d9b25 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt @@ -78,6 +78,10 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { val micEnabled = initialSettings.micEnabled val context = LocalContext.current val activity = context as? MainActivity + // The View hosting this composition — the one that receives the stream's touch/pointer events + // (the gesture Box below is a Compose node inside it), so it is where unbuffered dispatch is + // requested. + val composeView = androidx.compose.ui.platform.LocalView.current val window = activity?.window val controller = remember(window) { window?.let { WindowCompat.getInsetsController(it, it.decorView) } @@ -100,6 +104,9 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { var stats by remember { mutableStateOf(null) } var decoderLabel by remember { mutableStateOf("") } var codecLabel by remember { mutableStateOf("") } + // The panel's LIVE refresh rate, re-read each poll — the HUD flags a session whose panel sits + // below the stream rate (an OEM governor that ignored both the mode pin and the surface hint). + var panelHz by remember { mutableStateOf(0f) } var statsVerbosity by remember { mutableStateOf(initialSettings.statsVerbosity) } val statsOn = statsVerbosity != StatsVerbosity.OFF // Touch model is fixed per session (re-keys the gesture handler below if it ever changes). @@ -121,6 +128,7 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { while (true) { delay(1000) stats = NativeBridge.nativeVideoStats(handle) + panelHz = runCatching { context.display }.getOrNull()?.refreshRate ?: 0f // The decoder is fixed for the session; fetch its label once it's resolved. if (decoderLabel.isEmpty()) decoderLabel = NativeBridge.nativeVideoDecoderLabel(handle) } @@ -305,7 +313,22 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { } else { null } - activity?.setConsoleHighRefreshRate(false) // let the decoder's setFrameRate pick the panel rate + // Pin the panel to the stream's refresh (exact / multiple) for the session. The decoder's + // own ANativeWindow_setFrameRate hint still aligns vsync, but it is advisory — some OEM + // refresh governors ignore it outright and would leave a 120 Hz session on a 60/90 Hz + // panel. TV boxes skip the pin: the native side actively drives the HDMI mode there. + val streamHz = NativeBridge.nativeVideoSize(handle)?.getOrNull(2) ?: 0 + if (isTv) { + activity?.setConsoleHighRefreshRate(false) // the decoder's HDMI mode switch governs + } else { + activity?.setStreamDisplayMode(streamHz) + } + // Touch/pointer events are vsync-batched by default — up to a frame of input latency the + // stream shouldn't pay. Unbuffered dispatch delivers them the moment the kernel does. + // Undone by passing 0 on the way out (API 30+). + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + composeView.requestUnbufferedDispatch(android.view.InputDevice.SOURCE_CLASS_POINTER) + } // Host→client feedback (rumble + DualSense lightbar/LEDs), routed to each controller by pad // index via the router; poll threads stopped + joined before the router is released and the // session closed. "Rumble on this phone" (opt-in) additionally mirrors controller 1's @@ -435,6 +458,9 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { // Back in the menus: the SC2 (if present) resumes driving the console UI. activity?.startSc2MenuNav() activity?.setConsoleHighRefreshRate(true) // back to the console UI's max refresh + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + composeView.requestUnbufferedDispatch(0) // back to ordinary batched dispatch + } controller?.hide(WindowInsetsCompat.Type.ime()) // drop any keyboard left showing window?.setSoftInputMode(priorSoftInput) controller?.show(WindowInsetsCompat.Type.systemBars()) @@ -555,6 +581,7 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { stats?.let { StatsOverlay( it, statsVerbosity, decoderLabel, codecLabel, session.profileName, + panelHz, Modifier.align(Alignment.TopStart).padding(12.dp), ) } diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index 8cd56ccd..677a41ec 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -184,10 +184,12 @@ object NativeBridge { external fun nativeVideoMime(handle: Long): String /** - * The negotiated video mode as `[width, height]`, or `null` on a `0` handle. Resolved at the - * handshake, so it is known before the first frame — the stream view sizes itself to THIS - * aspect rather than stretching the picture to the panel's. Fixed for the session; read once. - * Cheap; UI-safe. + * The negotiated video mode as `[width, height, refreshHz]`, or `null` on a `0` handle. + * Resolved at the handshake, so it is known before the first frame — the stream view sizes + * itself to THIS aspect rather than stretching the picture to the panel's, and pins the + * panel's display mode to the stream refresh. The trailing `refreshHz` was appended later + * (an older native lib returns only `[width, height]` — index defensively). Fixed for the + * session; read once. Cheap; UI-safe. */ external fun nativeVideoSize(handle: Long): IntArray? @@ -204,8 +206,8 @@ object NativeBridge { * entirely in Rust (NDK AMediaCodec → ANativeWindow) — no per-frame JNI. [decoderName] is the * decoder Kotlin ranked from `MediaCodecList` (`""` = let the platform resolve the default for * the MIME — what the pre-overhaul client always did); [lowLatencyMode] is the user's - * "Low-latency mode (experimental)" toggle (off, the default, runs the original decode - * pipeline; on, the aggressive per-SoC tuning + async loop); [lowLatencyFeature] is whether + * "Low-latency mode" master toggle (ON by default: async loop + per-SoC tuning; off runs the + * original synchronous pipeline as the per-device escape hatch); [lowLatencyFeature] is whether * [decoderName] advertised `FEATURE_LowLatency` (HUD label only). [isTv] drives an active HDMI * mode switch to the stream refresh on TV boxes when the toggle is on (vs. the softer seamless * hint otherwise). No-op if already started. diff --git a/clients/android/native/src/decode/setup.rs b/clients/android/native/src/decode/setup.rs index ac98b1f4..769dcb00 100644 --- a/clients/android/native/src/decode/setup.rs +++ b/clients/android/native/src/decode/setup.rs @@ -180,8 +180,14 @@ pub(super) fn boost_thread_priority() { /// mode (e.g. 60↔120) instead of leaving the panel at its default and judder-matching. The /// forced switch may blank the panel briefly — acceptable once at stream start, not wanted on a /// phone. Falls through to the 2-arg hint on API 30. -/// - Otherwise: `ANativeWindow_setFrameRate` (**API 30**) with `compatibility = DEFAULT` — the -/// softer, seamless-preferred hint for phones/tablets and the universal fallback. +/// - Otherwise: `ANativeWindow_setFrameRate` (**API 30**) — the seamless-preferred hint for +/// phones/tablets and the universal fallback. +/// +/// Both paths pass `compatibility = FIXED_SOURCE` (1): the stream is fixed-rate video content the +/// client cannot re-pace, which is exactly what that value declares — `DEFAULT` (0) told the +/// platform the app could adapt to whatever rate it picked, an invitation some OEM refresh +/// governors accepted by simply not switching. (The window-level `preferredDisplayModeId` pin in +/// `MainActivity.setStreamDisplayMode` is the phone-side belt to this braces.) /// /// Returns `true` when the platform accepted a hint; `false` on API < 30 (symbols absent) or a /// decline. @@ -200,8 +206,10 @@ pub(super) fn try_set_frame_rate(window: &NativeWindow, frame_rate: f32, is_tv: if lib.is_null() { return false; } - // TV: prefer the API-31 change-strategy form to force the mode switch (strategy 1 = ALWAYS, - // compatibility 0 = DEFAULT). Absent on API 30 ⇒ fall through to the 2-arg hint below. + // ANATIVEWINDOW_FRAME_RATE_COMPATIBILITY_FIXED_SOURCE — fixed-rate video content. + const FIXED_SOURCE: i8 = 1; + // TV: prefer the API-31 change-strategy form to force the mode switch (strategy 1 = + // ALWAYS). Absent on API 30 ⇒ fall through to the 2-arg hint below. if is_tv { let sym = libc::dlsym( lib, @@ -209,7 +217,7 @@ pub(super) fn try_set_frame_rate(window: &NativeWindow, frame_rate: f32, is_tv: ); if !sym.is_null() { let set = std::mem::transmute::<*mut c_void, SetFrameRateStrategyFn>(sym); - return set(window.ptr().as_ptr().cast(), frame_rate, 0, 1) == 0; + return set(window.ptr().as_ptr().cast(), frame_rate, FIXED_SOURCE, 1) == 0; } } let sym = libc::dlsym(lib, c"ANativeWindow_setFrameRate".as_ptr()); @@ -217,7 +225,7 @@ pub(super) fn try_set_frame_rate(window: &NativeWindow, frame_rate: f32, is_tv: return false; // device API < 30 — no per-surface frame-rate hint } let set_frame_rate = std::mem::transmute::<*mut c_void, SetFrameRateFn>(sym); - set_frame_rate(window.ptr().as_ptr().cast(), frame_rate, 0) == 0 + set_frame_rate(window.ptr().as_ptr().cast(), frame_rate, FIXED_SOURCE) == 0 } } diff --git a/clients/android/native/src/session/planes.rs b/clients/android/native/src/session/planes.rs index 5c3b059f..84a71ce6 100644 --- a/clients/android/native/src/session/planes.rs +++ b/clients/android/native/src/session/planes.rs @@ -264,10 +264,12 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats( } /// `NativeBridge.nativeVideoSize(handle): IntArray?` — the negotiated video mode as -/// `[width, height]`. Resolved at the handshake (Welcome), so it is known before a single frame -/// arrives: the UI sizes the video surface to the STREAM's aspect rather than stretching it to the -/// panel's. `null` on a `0` handle. Not android-gated — pure `jni` + a connector read, so it links -/// on the host build too. Cheap; safe on the UI thread. +/// `[width, height, refreshHz]`. Resolved at the handshake (Welcome), so it is known before a +/// single frame arrives: the UI sizes the video surface to the STREAM's aspect rather than +/// stretching it to the panel's, and pins the panel's display mode to the stream refresh. The +/// trailing `refreshHz` was appended later — old readers index only 0/1 and never see it. `null` +/// on a `0` handle. Not android-gated — pure `jni` + a connector read, so it links on the host +/// build too. Cheap; safe on the UI thread. #[no_mangle] pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoSize( env: JNIEnv, @@ -281,7 +283,11 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoSize( // SAFETY: live handle per the nativeConnect/nativeClose contract. let h = unsafe { &*(handle as *const SessionHandle) }; let mode = h.client.mode(); - let buf: [i32; 2] = [mode.width as i32, mode.height as i32]; + let buf: [i32; 3] = [ + mode.width as i32, + mode.height as i32, + mode.refresh_hz as i32, + ]; let arr = match env.new_int_array(buf.len() as jsize) { Ok(a) => a, Err(_) => return std::ptr::null_mut(),