diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/MouseInput.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/MouseInput.kt index 26fd5a4a..ad41893f 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/MouseInput.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/MouseInput.kt @@ -32,7 +32,13 @@ class MouseForwarder( private val handle: Long, private val invertScroll: Boolean, private val captureWanted: Boolean, - private val surfaceSize: () -> Pair, + /** + * The picture's rect in WINDOW coordinates — where the letterboxed video actually sits, which is + * the frame absolute positions must be measured against. Events arrive from the activity's + * dispatch overrides in window coordinates, so a stream narrower than the panel needs the origin + * subtracted as well as the size divided; `null` while the surface isn't laid out yet. + */ + private val videoRect: () -> android.graphics.Rect?, ) { /** Capture plumbing, owned by StreamScreen (the focusable capture view). */ var onRequestCapture: (() -> Unit)? = null @@ -152,12 +158,16 @@ class MouseForwarder( } private fun sendAbs(ev: MotionEvent) { - val (w, h) = surfaceSize() + val r = videoRect() ?: return + val w = r.width() + val h = r.height() if (w <= 0 || h <= 0) return + // Clamped into the picture: a pointer out on a letterbox bar has no host position of its + // own, and the edge is the honest answer for it. NativeBridge.nativeSendPointerAbs( handle, - ev.x.roundToInt().coerceIn(0, w - 1), - ev.y.roundToInt().coerceIn(0, h - 1), + (ev.x - r.left).roundToInt().coerceIn(0, w - 1), + (ev.y - r.top).roundToInt().coerceIn(0, h - 1), w, h, ) diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt index 30d89171..fc900ee8 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt @@ -26,6 +26,7 @@ import android.widget.Toast import androidx.activity.compose.BackHandler import androidx.compose.foundation.background import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.aspectRatio import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size @@ -197,6 +198,11 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { // below so the capture callbacks can reach the view once it exists. var keyCapture by remember { mutableStateOf(null) } + // The video SurfaceView, hoisted for the same reason: the pointer paths built below map WINDOW + // coordinates onto the picture, and with a letterboxed stream that rect is the video's, not the + // panel's. Set when the view is created. + var videoView by remember { mutableStateOf(null) } + DisposableEffect(handle) { window?.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) wifiLocks.forEach { lock -> @@ -256,7 +262,15 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { handle, invertScroll = initialSettings.invertScroll, captureWanted = initialSettings.mouseMode == MouseMode.CAPTURE, - surfaceSize = { (decor?.width ?: 0) to (decor?.height ?: 0) }, + // The picture's rect in window coordinates (see MouseForwarder.videoRect) — read live, + // so it is right from the frame the SurfaceView is first laid out. + videoRect = { + videoView?.takeIf { it.width > 0 && it.height > 0 }?.let { v -> + val loc = IntArray(2) + v.getLocationInWindow(loc) + android.graphics.Rect(loc[0], loc[1], loc[0] + v.width, loc[1] + v.height) + } + }, ) mouse.onRequestCapture = { // The grab needs the (focusable) capture view: focus it, then ask. Posted so a @@ -275,7 +289,7 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { val remote = if (isTv) { RemotePointer( handle, - surfaceWidth = { decor?.width ?: 1920 }, + surfaceWidth = { videoView?.width?.takeIf { it > 0 } ?: decor?.width ?: 1920 }, onActiveChanged = { on -> remotePointerOn = on }, onKeyboardToggle = { keyCapture?.let { it.setImeVisible(!it.imeShown) } }, ) @@ -423,11 +437,33 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { activity?.mouseForwarder?.engageFromStart() } - Box(modifier = Modifier.fillMaxSize()) { + // Fit the picture to the stream's own aspect, letterboxing the rest in black. MediaCodec scales + // whatever it decodes to fill the Surface it renders into, so a 16:9 stream on a 20:9 panel came + // out stretched — the surface has to carry the aspect, because nothing downstream of it can. + // The mode is the negotiated one (known from the handshake, before the first frame); 0/absent — + // an older native lib — falls back to filling, i.e. exactly the previous behaviour. + val videoAspect = remember(handle) { + val size = NativeBridge.nativeVideoSize(handle) + val w = size?.getOrNull(0) ?: 0 + val h = size?.getOrNull(1) ?: 0 + if (w > 0 && h > 0) w.toFloat() / h.toFloat() else 0f + } + Box(modifier = Modifier.fillMaxSize().background(Color.Black)) { + // One rect for the picture AND for the input that lands on it. Every absolute mapping — + // direct-pointer touch, multi-touch passthrough, the pen lane — measures against the size of + // the node it sits on, so putting the gesture layer on this same rect keeps all three correct + // by construction rather than by threading an offset through each of them. The cost is that + // trackpad swipes starting inside a letterbox bar don't register; the picture is the surface. + val videoFit = if (videoAspect > 0f) { + Modifier.align(Alignment.Center).aspectRatio(videoAspect) + } else { + Modifier.fillMaxSize() + } AndroidView( - modifier = Modifier.fillMaxSize(), + modifier = videoFit, factory = { ctx -> SurfaceView(ctx).apply { + videoView = this holder.addCallback(object : SurfaceHolder.Callback { override fun surfaceCreated(holder: SurfaceHolder) { // Low-latency mode: rank MediaCodecList decoders for the negotiated @@ -517,7 +553,7 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) { LaunchedEffect(stylus) { stylus.heartbeatLoop() } } Box( - Modifier.fillMaxSize().pointerInput(handle, touchMode) { + videoFit.pointerInput(handle, touchMode) { when (touchMode) { TouchMode.TOUCH -> streamTouchPassthrough(handle, stylus) else -> streamTouchInput( diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index 5bc3bdc1..a61dbbe1 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -183,6 +183,14 @@ object NativeBridge { */ external fun nativeVideoMime(handle: Long): String + /** + * The negotiated video mode as `[width, height]`, or `null` on a `0` handle. Resolved at the + * handshake, so it is known before the first frame — the stream view sizes itself to THIS + * aspect rather than stretching the picture to the panel's. Fixed for the session; read once. + * Cheap; UI-safe. + */ + external fun nativeVideoSize(handle: Long): IntArray? + /** * A short human label for the codec the host resolved (`"H.264"` / `"HEVC"` / `"AV1"` / * `"PyroWave"`), for the stats HUD's video-feed line, or `""` on a `0` handle. Distinct from diff --git a/clients/android/native/src/session/planes.rs b/clients/android/native/src/session/planes.rs index 6e34efa6..5c3b059f 100644 --- a/clients/android/native/src/session/planes.rs +++ b/clients/android/native/src/session/planes.rs @@ -5,7 +5,7 @@ use jni::objects::JObject; // Used only by the android-gated `nativeStartVideo`; on the host build that fn is cfg'd out. #[cfg(target_os = "android")] use jni::objects::JString; -use jni::sys::{jboolean, jdoubleArray, jlong, jsize, jstring}; +use jni::sys::{jboolean, jdoubleArray, jintArray, jlong, jsize, jstring}; use jni::JNIEnv; use super::{jni_guard, SessionHandle}; @@ -263,6 +263,36 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats( }) } +/// `NativeBridge.nativeVideoSize(handle): IntArray?` — the negotiated video mode as +/// `[width, height]`. Resolved at the handshake (Welcome), so it is known before a single frame +/// arrives: the UI sizes the video surface to the STREAM's aspect rather than stretching it to the +/// panel's. `null` on a `0` handle. Not android-gated — pure `jni` + a connector read, so it links +/// on the host build too. Cheap; safe on the UI thread. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoSize( + env: JNIEnv, + _this: JObject, + handle: jlong, +) -> jintArray { + jni_guard(std::ptr::null_mut(), || { + if handle == 0 { + return std::ptr::null_mut(); + } + // SAFETY: live handle per the nativeConnect/nativeClose contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + let mode = h.client.mode(); + let buf: [i32; 2] = [mode.width as i32, mode.height as i32]; + let arr = match env.new_int_array(buf.len() as jsize) { + Ok(a) => a, + Err(_) => return std::ptr::null_mut(), + }; + if env.set_int_array_region(&arr, 0, &buf).is_err() { + return std::ptr::null_mut(); + } + arr.into_raw() + }) +} + /// `NativeBridge.nativeSetVideoStatsEnabled(handle, enabled)` — gate per-frame stats sampling on the /// HUD actually being visible: while disabled the decode thread skips the clock read + lock per AU. /// Enabling resets the measurement window so a later show never reports stale data. Sticky for the