diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt index 7843e98..267f989 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt @@ -1,6 +1,7 @@ package io.unom.punktfunk import android.os.Bundle +import android.view.KeyEvent import android.view.SurfaceHolder import android.view.SurfaceView import android.view.WindowManager @@ -8,7 +9,10 @@ import androidx.activity.ComponentActivity import androidx.activity.compose.BackHandler import androidx.activity.compose.setContent import androidx.activity.enableEdgeToEdge +import androidx.compose.foundation.gestures.awaitEachGesture +import androidx.compose.foundation.gestures.awaitFirstDown import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxSize @@ -30,16 +34,27 @@ import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.input.pointer.pointerInput +import androidx.compose.ui.input.pointer.positionChange import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.text.input.KeyboardType import androidx.compose.ui.unit.dp import androidx.compose.ui.viewinterop.AndroidView +import io.unom.punktfunk.kit.Keymap import io.unom.punktfunk.kit.NativeBridge +import kotlin.math.abs import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext class MainActivity : ComponentActivity() { + /** + * The active stream session handle (0 = not streaming). Set by [StreamScreen] while it's shown. + * `dispatchKeyEvent` is the earliest, most reliable key hook — above Compose's focus system — + * so hardware keys are forwarded to the host regardless of which view holds focus. + */ + var streamHandle: Long = 0L + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) enableEdgeToEdge() @@ -49,6 +64,33 @@ class MainActivity : ComponentActivity() { } } } + + override fun dispatchKeyEvent(event: KeyEvent): Boolean { + val handle = streamHandle + if (handle != 0L) { + when (event.keyCode) { + // Leave these to the system even while streaming. + KeyEvent.KEYCODE_BACK, // → BackHandler leaves the stream + KeyEvent.KEYCODE_VOLUME_UP, + KeyEvent.KEYCODE_VOLUME_DOWN, + KeyEvent.KEYCODE_VOLUME_MUTE, + KeyEvent.KEYCODE_POWER -> {} + else -> { + val down = when (event.action) { + KeyEvent.ACTION_DOWN -> true + KeyEvent.ACTION_UP -> false + else -> return super.dispatchKeyEvent(event) + } + val vk = Keymap.toVk(event.keyCode) + if (vk != 0) { + NativeBridge.nativeSendKey(handle, vk, down, 0) + return true // consumed — don't let the system also act on it + } + } + } + } + return super.dispatchKeyEvent(event) + } } /** Scaffold mode requested from the host (WxH@Hz). TODO: derive from the display. */ @@ -131,11 +173,14 @@ private fun ConnectScreen(onConnected: (Long) -> Unit) { @Composable private fun StreamScreen(handle: Long, onDisconnect: () -> Unit) { val context = LocalContext.current - val window = (context as? ComponentActivity)?.window + val activity = context as? MainActivity + val window = activity?.window DisposableEffect(handle) { window?.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + activity?.streamHandle = handle // route hardware keys to this session onDispose { + activity?.streamHandle = 0L window?.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) // Leaving the stream: stop the audio + decode threads and tear down the session. NativeBridge.nativeStopAudio(handle) @@ -146,24 +191,62 @@ private fun StreamScreen(handle: Long, onDisconnect: () -> Unit) { BackHandler { onDisconnect() } - AndroidView( - modifier = Modifier.fillMaxSize(), - factory = { ctx -> - SurfaceView(ctx).apply { - holder.addCallback(object : SurfaceHolder.Callback { - override fun surfaceCreated(holder: SurfaceHolder) { - NativeBridge.nativeStartVideo(handle, holder.surface) - NativeBridge.nativeStartAudio(handle) - } + Box(modifier = Modifier.fillMaxSize()) { + AndroidView( + modifier = Modifier.fillMaxSize(), + factory = { ctx -> + SurfaceView(ctx).apply { + holder.addCallback(object : SurfaceHolder.Callback { + override fun surfaceCreated(holder: SurfaceHolder) { + NativeBridge.nativeStartVideo(handle, holder.surface) + NativeBridge.nativeStartAudio(handle) + } - override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {} + override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {} - override fun surfaceDestroyed(holder: SurfaceHolder) { - NativeBridge.nativeStopAudio(handle) - NativeBridge.nativeStopVideo(handle) + override fun surfaceDestroyed(holder: SurfaceHolder) { + NativeBridge.nativeStopAudio(handle) + NativeBridge.nativeStopVideo(handle) + } + }) + } + }, + ) + // Touch virtual-trackpad overlay: 1-finger drag → relative mouse move; tap → left click; + // 2-finger drag → scroll. (Physical-mouse pointer capture comes in a later increment.) + Box( + Modifier.fillMaxSize().pointerInput(handle) { + awaitEachGesture { + val first = awaitFirstDown(requireUnconsumed = false) + var moved = false + var maxFingers = 1 + while (true) { + val ev = awaitPointerEvent() + val fingers = ev.changes.count { it.pressed } + if (fingers == 0) break + if (fingers > maxFingers) maxFingers = fingers + val primary = ev.changes.firstOrNull { it.id == first.id } ?: ev.changes.first() + val d = primary.positionChange() + if (abs(d.x) > 0.5f || abs(d.y) > 0.5f) { + moved = true + if (fingers >= 2) { + // screen +y down → wire +up, so negate y. Coarse divisor; tune live. + val sy = (-d.y / 4f).toInt() + val sx = (d.x / 4f).toInt() + if (sy != 0) NativeBridge.nativeSendScroll(handle, 0, sy * 120) + if (sx != 0) NativeBridge.nativeSendScroll(handle, 1, sx * 120) + } else { + NativeBridge.nativeSendPointerMove(handle, d.x.toInt(), d.y.toInt()) + } + } + ev.changes.forEach { it.consume() } } - }) - } - }, - ) + if (!moved && maxFingers == 1) { + NativeBridge.nativeSendPointerButton(handle, 1, true) + NativeBridge.nativeSendPointerButton(handle, 1, false) + } + } + }, + ) + } } diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/Keymap.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/Keymap.kt new file mode 100644 index 0000000..f1b8a55 --- /dev/null +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/Keymap.kt @@ -0,0 +1,77 @@ +package io.unom.punktfunk.kit + +import android.view.KeyEvent + +/** + * Android `KEYCODE_*` → Windows Virtual-Key code (the punktfunk wire contract; the host maps VK → + * evdev via `inject::vk_to_evdev`). The Android analogue of the Linux client's evdev→VK table + * (`punktfunk-client-linux/src/keymap.rs`) and the Apple client's `hidToVK`. Positional/US-layout — + * we forward the physical key, not the typed character. Unmapped keys → 0 (the Rust side drops them). + * Extend this alongside `punktfunk-host/src/inject.rs::vk_to_evdev` (emit only VKs the host knows). + */ +object Keymap { + fun toVk(keyCode: Int): Int = when (keyCode) { + in KeyEvent.KEYCODE_A..KeyEvent.KEYCODE_Z -> 0x41 + (keyCode - KeyEvent.KEYCODE_A) // A–Z + in KeyEvent.KEYCODE_0..KeyEvent.KEYCODE_9 -> 0x30 + (keyCode - KeyEvent.KEYCODE_0) // 0–9 row + in KeyEvent.KEYCODE_F1..KeyEvent.KEYCODE_F12 -> 0x70 + (keyCode - KeyEvent.KEYCODE_F1) // F1–F12 + in KeyEvent.KEYCODE_NUMPAD_0..KeyEvent.KEYCODE_NUMPAD_9 -> + 0x60 + (keyCode - KeyEvent.KEYCODE_NUMPAD_0) // numpad 0–9 + + // Whitespace / editing + KeyEvent.KEYCODE_DEL -> 0x08 // Backspace (Android KEYCODE_DEL == backspace) + KeyEvent.KEYCODE_TAB -> 0x09 + KeyEvent.KEYCODE_ENTER, KeyEvent.KEYCODE_NUMPAD_ENTER -> 0x0D + KeyEvent.KEYCODE_ESCAPE -> 0x1B + KeyEvent.KEYCODE_SPACE -> 0x20 + KeyEvent.KEYCODE_CAPS_LOCK -> 0x14 + KeyEvent.KEYCODE_BREAK -> 0x13 // Pause + KeyEvent.KEYCODE_SYSRQ -> 0x2C // PrintScreen + KeyEvent.KEYCODE_INSERT -> 0x2D + KeyEvent.KEYCODE_FORWARD_DEL -> 0x2E // Delete (forward) + KeyEvent.KEYCODE_NUM_LOCK -> 0x90 + KeyEvent.KEYCODE_SCROLL_LOCK -> 0x91 + + // Navigation + KeyEvent.KEYCODE_PAGE_UP -> 0x21 + KeyEvent.KEYCODE_PAGE_DOWN -> 0x22 + KeyEvent.KEYCODE_MOVE_END -> 0x23 + KeyEvent.KEYCODE_MOVE_HOME -> 0x24 + KeyEvent.KEYCODE_DPAD_LEFT -> 0x25 + KeyEvent.KEYCODE_DPAD_UP -> 0x26 + KeyEvent.KEYCODE_DPAD_RIGHT -> 0x27 + KeyEvent.KEYCODE_DPAD_DOWN -> 0x28 + + // Modifiers (L/R-specific VKs; the host folds the generic ones onto the left variant) + KeyEvent.KEYCODE_SHIFT_LEFT -> 0xA0 + KeyEvent.KEYCODE_SHIFT_RIGHT -> 0xA1 + KeyEvent.KEYCODE_CTRL_LEFT -> 0xA2 + KeyEvent.KEYCODE_CTRL_RIGHT -> 0xA3 + KeyEvent.KEYCODE_ALT_LEFT -> 0xA4 + KeyEvent.KEYCODE_ALT_RIGHT -> 0xA5 // AltGr + KeyEvent.KEYCODE_META_LEFT -> 0x5B // Super/Win + KeyEvent.KEYCODE_META_RIGHT -> 0x5C + KeyEvent.KEYCODE_MENU -> 0x5D // Application + + // Numpad operators + KeyEvent.KEYCODE_NUMPAD_MULTIPLY -> 0x6A + KeyEvent.KEYCODE_NUMPAD_ADD -> 0x6B + KeyEvent.KEYCODE_NUMPAD_SUBTRACT -> 0x6D + KeyEvent.KEYCODE_NUMPAD_DOT -> 0x6E + KeyEvent.KEYCODE_NUMPAD_DIVIDE -> 0x6F + + // OEM punctuation (US-layout positional) + KeyEvent.KEYCODE_SEMICOLON -> 0xBA + KeyEvent.KEYCODE_EQUALS -> 0xBB + KeyEvent.KEYCODE_COMMA -> 0xBC + KeyEvent.KEYCODE_MINUS -> 0xBD + KeyEvent.KEYCODE_PERIOD -> 0xBE + KeyEvent.KEYCODE_SLASH -> 0xBF + KeyEvent.KEYCODE_GRAVE -> 0xC0 + KeyEvent.KEYCODE_LEFT_BRACKET -> 0xDB + KeyEvent.KEYCODE_BACKSLASH -> 0xDC + KeyEvent.KEYCODE_RIGHT_BRACKET -> 0xDD + KeyEvent.KEYCODE_APOSTROPHE -> 0xDE + + else -> 0 // unmapped → Rust drops it + } +} diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index 2138f2b..4c59a3a 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -46,4 +46,18 @@ object NativeBridge { /** Stop + join the audio thread and close AAudio, without closing the session. No-op on `0`. */ external fun nativeStopAudio(handle: Long) + + // ---- Input: Kotlin captures, Rust forwards to the host (send_input) ---- + + /** Relative mouse move; dx/dy are device-pixel deltas (screen +y down). */ + external fun nativeSendPointerMove(handle: Long, dx: Int, dy: Int) + + /** One mouse-button transition. button: 1=left 2=middle 3=right 4=X1 5=X2. */ + external fun nativeSendPointerButton(handle: Long, button: Int, down: Boolean) + + /** One scroll step. axis: 0=vertical 1=horizontal. delta: signed, 120-scaled, +=up/right. */ + external fun nativeSendScroll(handle: Long, axis: Int, delta: Int) + + /** One key transition. vk: Windows VK (0 = dropped by Rust). mods: VK modifier mask (0 for now). */ + external fun nativeSendKey(handle: Long, vk: Int, down: Boolean, mods: Int) } diff --git a/crates/punktfunk-android/src/session.rs b/crates/punktfunk-android/src/session.rs index f8f09f0..da84786 100644 --- a/crates/punktfunk-android/src/session.rs +++ b/crates/punktfunk-android/src/session.rs @@ -12,10 +12,11 @@ //! from `crates/punktfunk-client-linux`. use jni::objects::{JObject, JString}; -use jni::sys::{jint, jlong}; +use jni::sys::{jboolean, jint, jlong}; use jni::JNIEnv; use punktfunk_core::client::NativeClient; use punktfunk_core::config::{CompositorPref, GamepadPref, Mode}; +use punktfunk_core::input::{InputEvent, InputKind}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; @@ -228,3 +229,119 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopAudio( h.stop_audio(); } } + +// ---- Input plane: Kotlin capture → NativeClient::send_input ---------------------------------- +// All four are `&self` on the `Sync` connector (send_input is a non-blocking datagram push), safe +// from the Kotlin UI thread. NOT android-gated — send_input exists on the host build too, so these +// compile everywhere (parity with nativeConnect/nativeClose). The wire codes are the GameStream +// conventions: buttons 1=left/2=middle/3=right/4=X1/5=X2; scroll axis 0=vertical/1=horizontal, +// signed 120-unit delta, +=up/right; keys are Windows VK (mapped from KEYCODE_* on the Kotlin side). + +/// `NativeBridge.nativeSendPointerMove(handle, dx, dy)` — relative mouse motion (screen +y down). +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendPointerMove( + _env: JNIEnv, + _this: JObject, + handle: jlong, + dx: jint, + dy: jint, +) { + if handle == 0 { + return; + } + // SAFETY: live handle per the nativeConnect/nativeClose contract; send_input is &self. + let h = unsafe { &*(handle as *const SessionHandle) }; + let _ = h.client.send_input(&InputEvent { + kind: InputKind::MouseMove, + _pad: [0; 3], + code: 0, + x: dx, + y: dy, + flags: 0, + }); +} + +/// `NativeBridge.nativeSendPointerButton(handle, button, down)` — one button transition. +/// `button`: GameStream id (1=left, 2=middle, 3=right, 4=X1, 5=X2). `down`: 1=press, 0=release. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendPointerButton( + _env: JNIEnv, + _this: JObject, + handle: jlong, + button: jint, + down: jboolean, +) { + if handle == 0 { + return; + } + // SAFETY: live handle per the contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + let _ = h.client.send_input(&InputEvent { + kind: if down != 0 { + InputKind::MouseButtonDown + } else { + InputKind::MouseButtonUp + }, + _pad: [0; 3], + code: button as u32, + x: 0, + y: 0, + flags: 0, + }); +} + +/// `NativeBridge.nativeSendScroll(handle, axis, delta)` — one scroll step. `axis`: 0=vertical, +/// 1=horizontal. `delta`: signed, WHEEL_DELTA(120)-scaled, +=up/right. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendScroll( + _env: JNIEnv, + _this: JObject, + handle: jlong, + axis: jint, + delta: jint, +) { + if handle == 0 { + return; + } + // SAFETY: live handle per the contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + let _ = h.client.send_input(&InputEvent { + kind: InputKind::MouseScroll, + _pad: [0; 3], + code: axis as u32, + x: delta, + y: 0, + flags: 0, + }); +} + +/// `NativeBridge.nativeSendKey(handle, vk, down, mods)` — one key transition. `vk`: Windows +/// Virtual-Key code (0 = unmapped → dropped). `down`: 1=press, 0=release. `mods`: VK modifier +/// bitmask (0 for now — the host folds modifiers from the L/R modifier key events themselves). +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendKey( + _env: JNIEnv, + _this: JObject, + handle: jlong, + vk: jint, + down: jboolean, + mods: jint, +) { + if handle == 0 || vk == 0 { + return; + } + // SAFETY: live handle per the contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + let _ = h.client.send_input(&InputEvent { + kind: if down != 0 { + InputKind::KeyDown + } else { + InputKind::KeyUp + }, + _pad: [0; 3], + code: vk as u32, + x: 0, + y: 0, + flags: mods as u32, + }); +}