From a94b1d3cccc93a4a8f2142626fbe5ac7f65b67bc Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Thu, 30 Jul 2026 00:44:39 +0200 Subject: [PATCH] fix(client/android): the stream keeps its aspect instead of stretching to the panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MediaCodec scales whatever it decodes to fill the Surface it renders into, and the Surface filled the screen — so a stream whose resolution didn't match the panel's aspect came out stretched. Nothing downstream of the Surface can correct that; the Surface itself has to carry the aspect. Size the video to the negotiated mode's ratio, centred, with the remainder black. The mode is known from the handshake before the first frame arrives, via a new `nativeVideoSize` (the same `client.mode()` the HUD already reports as `w×h@hz`); an older native lib returning nothing falls back to filling, exactly as before. Input follows the picture. Direct-pointer touch, multi-touch passthrough and the pen lane all map positions against the size of the node they sit on, so the gesture layer moves onto the same rect as the video and all three stay correct by construction instead of each needing an offset threaded through it. The physical-mouse path can't work that way — its events arrive from the activity in WINDOW coordinates — so it now measures against the SurfaceView's rect on screen, subtracting the letterbox origin and clamping into the picture: a pointer out on a bar has no host position of its own, and the edge is the honest answer for it. One deliberate consequence: trackpad swipes that START inside a letterbox bar no longer register. Trackpad input is relative and could have kept the whole panel, but one rule — input lands on the picture — beats a mode- dependent input surface, and the pen lane rides inside trackpad mode too. Co-Authored-By: Claude Opus 5 (1M context) --- .../kotlin/io/unom/punktfunk/MouseInput.kt | 18 ++++++-- .../kotlin/io/unom/punktfunk/StreamScreen.kt | 46 +++++++++++++++++-- .../io/unom/punktfunk/kit/NativeBridge.kt | 8 ++++ clients/android/native/src/session/planes.rs | 32 ++++++++++++- 4 files changed, 94 insertions(+), 10 deletions(-) diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/MouseInput.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/MouseInput.kt index 26fd5a4a..ad41893f 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/MouseInput.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/MouseInput.kt @@ -32,7 +32,13 @@ class MouseForwarder( private val handle: Long, private val invertScroll: Boolean, private val captureWanted: Boolean, - private val surfaceSize: () -> Pair, + /** + * The picture's rect in WINDOW coordinates — where the letterboxed video actually sits, which is + * the frame absolute positions must be measured against. Events arrive from the activity's + * dispatch overrides in window coordinates, so a stream narrower than the panel needs the origin + * subtracted as well as the size divided; `null` while the surface isn't laid out yet. + */ + private val videoRect: () -> android.graphics.Rect?, ) { /** Capture plumbing, owned by StreamScreen (the focusable capture view). */ var onRequestCapture: (() -> Unit)? = null @@ -152,12 +158,16 @@ class MouseForwarder( } private fun sendAbs(ev: MotionEvent) { - val (w, h) = surfaceSize() + val r = videoRect() ?: return + val w = r.width() + val h = r.height() if (w <= 0 || h <= 0) return + // Clamped into the picture: a pointer out on a letterbox bar has no host position of its + // own, and the edge is the honest answer for it. NativeBridge.nativeSendPointerAbs( handle, - ev.x.roundToInt().coerceIn(0, w - 1), - ev.y.roundToInt().coerceIn(0, h - 1), + (ev.x - r.left).roundToInt().coerceIn(0, w - 1), + (ev.y - r.top).roundToInt().coerceIn(0, h - 1), w, h, ) diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt index 30d89171..fc900ee8 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt @@ -26,6 +26,7 @@ import android.widget.Toast import androidx.activity.compose.BackHandler import androidx.compose.foundation.background import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.aspectRatio import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size @@ -197,6 +198,11 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { // below so the capture callbacks can reach the view once it exists. var keyCapture by remember { mutableStateOf(null) } + // The video SurfaceView, hoisted for the same reason: the pointer paths built below map WINDOW + // coordinates onto the picture, and with a letterboxed stream that rect is the video's, not the + // panel's. Set when the view is created. + var videoView by remember { mutableStateOf(null) } + DisposableEffect(handle) { window?.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) wifiLocks.forEach { lock -> @@ -256,7 +262,15 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { handle, invertScroll = initialSettings.invertScroll, captureWanted = initialSettings.mouseMode == MouseMode.CAPTURE, - surfaceSize = { (decor?.width ?: 0) to (decor?.height ?: 0) }, + // The picture's rect in window coordinates (see MouseForwarder.videoRect) — read live, + // so it is right from the frame the SurfaceView is first laid out. + videoRect = { + videoView?.takeIf { it.width > 0 && it.height > 0 }?.let { v -> + val loc = IntArray(2) + v.getLocationInWindow(loc) + android.graphics.Rect(loc[0], loc[1], loc[0] + v.width, loc[1] + v.height) + } + }, ) mouse.onRequestCapture = { // The grab needs the (focusable) capture view: focus it, then ask. Posted so a @@ -275,7 +289,7 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { val remote = if (isTv) { RemotePointer( handle, - surfaceWidth = { decor?.width ?: 1920 }, + surfaceWidth = { videoView?.width?.takeIf { it > 0 } ?: decor?.width ?: 1920 }, onActiveChanged = { on -> remotePointerOn = on }, onKeyboardToggle = { keyCapture?.let { it.setImeVisible(!it.imeShown) } }, ) @@ -423,11 +437,33 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { activity?.mouseForwarder?.engageFromStart() } - Box(modifier = Modifier.fillMaxSize()) { + // Fit the picture to the stream's own aspect, letterboxing the rest in black. MediaCodec scales + // whatever it decodes to fill the Surface it renders into, so a 16:9 stream on a 20:9 panel came + // out stretched — the surface has to carry the aspect, because nothing downstream of it can. + // The mode is the negotiated one (known from the handshake, before the first frame); 0/absent — + // an older native lib — falls back to filling, i.e. exactly the previous behaviour. + val videoAspect = remember(handle) { + val size = NativeBridge.nativeVideoSize(handle) + val w = size?.getOrNull(0) ?: 0 + val h = size?.getOrNull(1) ?: 0 + if (w > 0 && h > 0) w.toFloat() / h.toFloat() else 0f + } + Box(modifier = Modifier.fillMaxSize().background(Color.Black)) { + // One rect for the picture AND for the input that lands on it. Every absolute mapping — + // direct-pointer touch, multi-touch passthrough, the pen lane — measures against the size of + // the node it sits on, so putting the gesture layer on this same rect keeps all three correct + // by construction rather than by threading an offset through each of them. The cost is that + // trackpad swipes starting inside a letterbox bar don't register; the picture is the surface. + val videoFit = if (videoAspect > 0f) { + Modifier.align(Alignment.Center).aspectRatio(videoAspect) + } else { + Modifier.fillMaxSize() + } AndroidView( - modifier = Modifier.fillMaxSize(), + modifier = videoFit, factory = { ctx -> SurfaceView(ctx).apply { + videoView = this holder.addCallback(object : SurfaceHolder.Callback { override fun surfaceCreated(holder: SurfaceHolder) { // Low-latency mode: rank MediaCodecList decoders for the negotiated @@ -517,7 +553,7 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { LaunchedEffect(stylus) { stylus.heartbeatLoop() } } Box( - Modifier.fillMaxSize().pointerInput(handle, touchMode) { + videoFit.pointerInput(handle, touchMode) { when (touchMode) { TouchMode.TOUCH -> streamTouchPassthrough(handle, stylus) else -> streamTouchInput( diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index 5bc3bdc1..a61dbbe1 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -183,6 +183,14 @@ object NativeBridge { */ external fun nativeVideoMime(handle: Long): String + /** + * The negotiated video mode as `[width, height]`, or `null` on a `0` handle. Resolved at the + * handshake, so it is known before the first frame — the stream view sizes itself to THIS + * aspect rather than stretching the picture to the panel's. Fixed for the session; read once. + * Cheap; UI-safe. + */ + external fun nativeVideoSize(handle: Long): IntArray? + /** * A short human label for the codec the host resolved (`"H.264"` / `"HEVC"` / `"AV1"` / * `"PyroWave"`), for the stats HUD's video-feed line, or `""` on a `0` handle. Distinct from diff --git a/clients/android/native/src/session/planes.rs b/clients/android/native/src/session/planes.rs index 6e34efa6..5c3b059f 100644 --- a/clients/android/native/src/session/planes.rs +++ b/clients/android/native/src/session/planes.rs @@ -5,7 +5,7 @@ use jni::objects::JObject; // Used only by the android-gated `nativeStartVideo`; on the host build that fn is cfg'd out. #[cfg(target_os = "android")] use jni::objects::JString; -use jni::sys::{jboolean, jdoubleArray, jlong, jsize, jstring}; +use jni::sys::{jboolean, jdoubleArray, jintArray, jlong, jsize, jstring}; use jni::JNIEnv; use super::{jni_guard, SessionHandle}; @@ -263,6 +263,36 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats( }) } +/// `NativeBridge.nativeVideoSize(handle): IntArray?` — the negotiated video mode as +/// `[width, height]`. Resolved at the handshake (Welcome), so it is known before a single frame +/// arrives: the UI sizes the video surface to the STREAM's aspect rather than stretching it to the +/// panel's. `null` on a `0` handle. Not android-gated — pure `jni` + a connector read, so it links +/// on the host build too. Cheap; safe on the UI thread. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoSize( + env: JNIEnv, + _this: JObject, + handle: jlong, +) -> jintArray { + jni_guard(std::ptr::null_mut(), || { + if handle == 0 { + return std::ptr::null_mut(); + } + // SAFETY: live handle per the nativeConnect/nativeClose contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + let mode = h.client.mode(); + let buf: [i32; 2] = [mode.width as i32, mode.height as i32]; + let arr = match env.new_int_array(buf.len() as jsize) { + Ok(a) => a, + Err(_) => return std::ptr::null_mut(), + }; + if env.set_int_array_region(&arr, 0, &buf).is_err() { + return std::ptr::null_mut(); + } + arr.into_raw() + }) +} + /// `NativeBridge.nativeSetVideoStatsEnabled(handle, enabled)` — gate per-frame stats sampling on the /// HUD actually being visible: while disabled the decode thread skips the clock read + lock per AU. /// Enabling resets the measurement window so a later show never reports stale data. Sticky for the