feat(android): input forwarding — keyboard + touch trackpad → send_input

M4 Android stage 1 (input). Kotlin captures input and forwards it over JNI to
NativeClient::send_input (the connector is linked as a Rust crate).

- crates/punktfunk-android: 4 JNI send fns (pointer move / button / scroll / key)
  building InputEvent with the GameStream wire codes — ungated, &self on the Sync
  connector (safe from the UI thread).
- clients/android: Keymap.kt (Android KEYCODE_* -> Windows VK, the host's wire
  contract, mirroring the Linux/Apple tables); Activity-level dispatchKeyEvent forwards
  hardware keys to the active session (above the Compose focus system, so it's reliable);
  a Compose touch-trackpad overlay -- 1-finger drag -> relative move, tap -> left click,
  2-finger drag -> scroll.

Verified live (emulator -> gamescope host on the LAN box, synthetic `adb input`): host
received 31 input datagrams (input=31) and libei injected KeyDown/KeyUp, MouseButtonDown/Up
and MouseMove all emitted=true. Physical-mouse pointer capture + gamepad are next.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt

@@ -1,6 +1,7 @@
 package io.unom.punktfunk
 
 import android.os.Bundle
+import android.view.KeyEvent
 import android.view.SurfaceHolder
 import android.view.SurfaceView
 import android.view.WindowManager
@@ -8,7 +9,10 @@ import androidx.activity.ComponentActivity
 import androidx.activity.compose.BackHandler
 import androidx.activity.compose.setContent
 import androidx.activity.enableEdgeToEdge
+import androidx.compose.foundation.gestures.awaitEachGesture
+import androidx.compose.foundation.gestures.awaitFirstDown
 import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.Box
 import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.Spacer
 import androidx.compose.foundation.layout.fillMaxSize
@@ -30,16 +34,27 @@ import androidx.compose.runtime.rememberCoroutineScope
 import androidx.compose.runtime.setValue
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
+import androidx.compose.ui.input.pointer.pointerInput
+import androidx.compose.ui.input.pointer.positionChange
 import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.text.input.KeyboardType
 import androidx.compose.ui.unit.dp
 import androidx.compose.ui.viewinterop.AndroidView
+import io.unom.punktfunk.kit.Keymap
 import io.unom.punktfunk.kit.NativeBridge
+import kotlin.math.abs
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
 
 class MainActivity : ComponentActivity() {
+    /**
+     * The active stream session handle (0 = not streaming). Set by [StreamScreen] while it's shown.
+     * `dispatchKeyEvent` is the earliest, most reliable key hook — above Compose's focus system —
+     * so hardware keys are forwarded to the host regardless of which view holds focus.
+     */
+    var streamHandle: Long = 0L
+
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
         enableEdgeToEdge()
@@ -49,6 +64,33 @@ class MainActivity : ComponentActivity() {
             }
         }
     }
+
+    override fun dispatchKeyEvent(event: KeyEvent): Boolean {
+        val handle = streamHandle
+        if (handle != 0L) {
+            when (event.keyCode) {
+                // Leave these to the system even while streaming.
+                KeyEvent.KEYCODE_BACK, // → BackHandler leaves the stream
+                KeyEvent.KEYCODE_VOLUME_UP,
+                KeyEvent.KEYCODE_VOLUME_DOWN,
+                KeyEvent.KEYCODE_VOLUME_MUTE,
+                KeyEvent.KEYCODE_POWER -> {}
+                else -> {
+                    val down = when (event.action) {
+                        KeyEvent.ACTION_DOWN -> true
+                        KeyEvent.ACTION_UP -> false
+                        else -> return super.dispatchKeyEvent(event)
+                    }
+                    val vk = Keymap.toVk(event.keyCode)
+                    if (vk != 0) {
+                        NativeBridge.nativeSendKey(handle, vk, down, 0)
+                        return true // consumed — don't let the system also act on it
+                    }
+                }
+            }
+        }
+        return super.dispatchKeyEvent(event)
+    }
 }
 
 /** Scaffold mode requested from the host (WxH@Hz). TODO: derive from the display. */
@@ -131,11 +173,14 @@ private fun ConnectScreen(onConnected: (Long) -> Unit) {
 @Composable
 private fun StreamScreen(handle: Long, onDisconnect: () -> Unit) {
     val context = LocalContext.current
-    val window = (context as? ComponentActivity)?.window
+    val activity = context as? MainActivity
+    val window = activity?.window
 
     DisposableEffect(handle) {
         window?.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
+        activity?.streamHandle = handle // route hardware keys to this session
         onDispose {
+            activity?.streamHandle = 0L
             window?.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
             // Leaving the stream: stop the audio + decode threads and tear down the session.
             NativeBridge.nativeStopAudio(handle)
@@ -146,24 +191,62 @@ private fun StreamScreen(handle: Long, onDisconnect: () -> Unit) {
 
     BackHandler { onDisconnect() }
 
-    AndroidView(
+    Box(modifier = Modifier.fillMaxSize()) {
-        modifier = Modifier.fillMaxSize(),
+        AndroidView(
-        factory = { ctx ->
+            modifier = Modifier.fillMaxSize(),
-            SurfaceView(ctx).apply {
+            factory = { ctx ->
-                holder.addCallback(object : SurfaceHolder.Callback {
+                SurfaceView(ctx).apply {
-                    override fun surfaceCreated(holder: SurfaceHolder) {
+                    holder.addCallback(object : SurfaceHolder.Callback {
-                        NativeBridge.nativeStartVideo(handle, holder.surface)
+                        override fun surfaceCreated(holder: SurfaceHolder) {
-                        NativeBridge.nativeStartAudio(handle)
+                            NativeBridge.nativeStartVideo(handle, holder.surface)
-                    }
+                            NativeBridge.nativeStartAudio(handle)
+                        }
 
-                    override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {}
+                        override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {}
 
-                    override fun surfaceDestroyed(holder: SurfaceHolder) {
+                        override fun surfaceDestroyed(holder: SurfaceHolder) {
-                        NativeBridge.nativeStopAudio(handle)
+                            NativeBridge.nativeStopAudio(handle)
-                        NativeBridge.nativeStopVideo(handle)
+                            NativeBridge.nativeStopVideo(handle)
+                        }
+                    })
+                }
+            },
+        )
+        // Touch virtual-trackpad overlay: 1-finger drag → relative mouse move; tap → left click;
+        // 2-finger drag → scroll. (Physical-mouse pointer capture comes in a later increment.)
+        Box(
+            Modifier.fillMaxSize().pointerInput(handle) {
+                awaitEachGesture {
+                    val first = awaitFirstDown(requireUnconsumed = false)
+                    var moved = false
+                    var maxFingers = 1
+                    while (true) {
+                        val ev = awaitPointerEvent()
+                        val fingers = ev.changes.count { it.pressed }
+                        if (fingers == 0) break
+                        if (fingers > maxFingers) maxFingers = fingers
+                        val primary = ev.changes.firstOrNull { it.id == first.id } ?: ev.changes.first()
+                        val d = primary.positionChange()
+                        if (abs(d.x) > 0.5f || abs(d.y) > 0.5f) {
+                            moved = true
+                            if (fingers >= 2) {
+                                // screen +y down → wire +up, so negate y. Coarse divisor; tune live.
+                                val sy = (-d.y / 4f).toInt()
+                                val sx = (d.x / 4f).toInt()
+                                if (sy != 0) NativeBridge.nativeSendScroll(handle, 0, sy * 120)
+                                if (sx != 0) NativeBridge.nativeSendScroll(handle, 1, sx * 120)
+                            } else {
+                                NativeBridge.nativeSendPointerMove(handle, d.x.toInt(), d.y.toInt())
+                            }
+                        }
+                        ev.changes.forEach { it.consume() }
                     }
-                })
+                    if (!moved && maxFingers == 1) {
-            }
+                        NativeBridge.nativeSendPointerButton(handle, 1, true)
-        },
+                        NativeBridge.nativeSendPointerButton(handle, 1, false)
-    )
+                    }
+                }
+            },
+        )
+    }
 }

clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/Keymap.kt

@@ -0,0 +1,77 @@
+package io.unom.punktfunk.kit
+
+import android.view.KeyEvent
+
+/**
+ * Android `KEYCODE_*` → Windows Virtual-Key code (the punktfunk wire contract; the host maps VK →
+ * evdev via `inject::vk_to_evdev`). The Android analogue of the Linux client's evdev→VK table
+ * (`punktfunk-client-linux/src/keymap.rs`) and the Apple client's `hidToVK`. Positional/US-layout —
+ * we forward the physical key, not the typed character. Unmapped keys → 0 (the Rust side drops them).
+ * Extend this alongside `punktfunk-host/src/inject.rs::vk_to_evdev` (emit only VKs the host knows).
+ */
+object Keymap {
+    fun toVk(keyCode: Int): Int = when (keyCode) {
+        in KeyEvent.KEYCODE_A..KeyEvent.KEYCODE_Z -> 0x41 + (keyCode - KeyEvent.KEYCODE_A) // A–Z
+        in KeyEvent.KEYCODE_0..KeyEvent.KEYCODE_9 -> 0x30 + (keyCode - KeyEvent.KEYCODE_0) // 0–9 row
+        in KeyEvent.KEYCODE_F1..KeyEvent.KEYCODE_F12 -> 0x70 + (keyCode - KeyEvent.KEYCODE_F1) // F1–F12
+        in KeyEvent.KEYCODE_NUMPAD_0..KeyEvent.KEYCODE_NUMPAD_9 ->
+            0x60 + (keyCode - KeyEvent.KEYCODE_NUMPAD_0) // numpad 0–9
+
+        // Whitespace / editing
+        KeyEvent.KEYCODE_DEL -> 0x08 // Backspace (Android KEYCODE_DEL == backspace)
+        KeyEvent.KEYCODE_TAB -> 0x09
+        KeyEvent.KEYCODE_ENTER, KeyEvent.KEYCODE_NUMPAD_ENTER -> 0x0D
+        KeyEvent.KEYCODE_ESCAPE -> 0x1B
+        KeyEvent.KEYCODE_SPACE -> 0x20
+        KeyEvent.KEYCODE_CAPS_LOCK -> 0x14
+        KeyEvent.KEYCODE_BREAK -> 0x13 // Pause
+        KeyEvent.KEYCODE_SYSRQ -> 0x2C // PrintScreen
+        KeyEvent.KEYCODE_INSERT -> 0x2D
+        KeyEvent.KEYCODE_FORWARD_DEL -> 0x2E // Delete (forward)
+        KeyEvent.KEYCODE_NUM_LOCK -> 0x90
+        KeyEvent.KEYCODE_SCROLL_LOCK -> 0x91
+
+        // Navigation
+        KeyEvent.KEYCODE_PAGE_UP -> 0x21
+        KeyEvent.KEYCODE_PAGE_DOWN -> 0x22
+        KeyEvent.KEYCODE_MOVE_END -> 0x23
+        KeyEvent.KEYCODE_MOVE_HOME -> 0x24
+        KeyEvent.KEYCODE_DPAD_LEFT -> 0x25
+        KeyEvent.KEYCODE_DPAD_UP -> 0x26
+        KeyEvent.KEYCODE_DPAD_RIGHT -> 0x27
+        KeyEvent.KEYCODE_DPAD_DOWN -> 0x28
+
+        // Modifiers (L/R-specific VKs; the host folds the generic ones onto the left variant)
+        KeyEvent.KEYCODE_SHIFT_LEFT -> 0xA0
+        KeyEvent.KEYCODE_SHIFT_RIGHT -> 0xA1
+        KeyEvent.KEYCODE_CTRL_LEFT -> 0xA2
+        KeyEvent.KEYCODE_CTRL_RIGHT -> 0xA3
+        KeyEvent.KEYCODE_ALT_LEFT -> 0xA4
+        KeyEvent.KEYCODE_ALT_RIGHT -> 0xA5 // AltGr
+        KeyEvent.KEYCODE_META_LEFT -> 0x5B // Super/Win
+        KeyEvent.KEYCODE_META_RIGHT -> 0x5C
+        KeyEvent.KEYCODE_MENU -> 0x5D // Application
+
+        // Numpad operators
+        KeyEvent.KEYCODE_NUMPAD_MULTIPLY -> 0x6A
+        KeyEvent.KEYCODE_NUMPAD_ADD -> 0x6B
+        KeyEvent.KEYCODE_NUMPAD_SUBTRACT -> 0x6D
+        KeyEvent.KEYCODE_NUMPAD_DOT -> 0x6E
+        KeyEvent.KEYCODE_NUMPAD_DIVIDE -> 0x6F
+
+        // OEM punctuation (US-layout positional)
+        KeyEvent.KEYCODE_SEMICOLON -> 0xBA
+        KeyEvent.KEYCODE_EQUALS -> 0xBB
+        KeyEvent.KEYCODE_COMMA -> 0xBC
+        KeyEvent.KEYCODE_MINUS -> 0xBD
+        KeyEvent.KEYCODE_PERIOD -> 0xBE
+        KeyEvent.KEYCODE_SLASH -> 0xBF
+        KeyEvent.KEYCODE_GRAVE -> 0xC0
+        KeyEvent.KEYCODE_LEFT_BRACKET -> 0xDB
+        KeyEvent.KEYCODE_BACKSLASH -> 0xDC
+        KeyEvent.KEYCODE_RIGHT_BRACKET -> 0xDD
+        KeyEvent.KEYCODE_APOSTROPHE -> 0xDE
+
+        else -> 0 // unmapped → Rust drops it
+    }
+}

clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt

@@ -46,4 +46,18 @@ object NativeBridge {
 
     /** Stop + join the audio thread and close AAudio, without closing the session. No-op on `0`. */
     external fun nativeStopAudio(handle: Long)
+
+    // ---- Input: Kotlin captures, Rust forwards to the host (send_input) ----
+
+    /** Relative mouse move; dx/dy are device-pixel deltas (screen +y down). */
+    external fun nativeSendPointerMove(handle: Long, dx: Int, dy: Int)
+
+    /** One mouse-button transition. button: 1=left 2=middle 3=right 4=X1 5=X2. */
+    external fun nativeSendPointerButton(handle: Long, button: Int, down: Boolean)
+
+    /** One scroll step. axis: 0=vertical 1=horizontal. delta: signed, 120-scaled, +=up/right. */
+    external fun nativeSendScroll(handle: Long, axis: Int, delta: Int)
+
+    /** One key transition. vk: Windows VK (0 = dropped by Rust). mods: VK modifier mask (0 for now). */
+    external fun nativeSendKey(handle: Long, vk: Int, down: Boolean, mods: Int)
 }

crates/punktfunk-android/src/session.rs

@@ -12,10 +12,11 @@
 //! from `crates/punktfunk-client-linux`.
 
 use jni::objects::{JObject, JString};
-use jni::sys::{jint, jlong};
+use jni::sys::{jboolean, jint, jlong};
 use jni::JNIEnv;
 use punktfunk_core::client::NativeClient;
 use punktfunk_core::config::{CompositorPref, GamepadPref, Mode};
+use punktfunk_core::input::{InputEvent, InputKind};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread::JoinHandle;
@@ -228,3 +229,119 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopAudio(
         h.stop_audio();
     }
 }
+
+// ---- Input plane: Kotlin capture → NativeClient::send_input ----------------------------------
+// All four are `&self` on the `Sync` connector (send_input is a non-blocking datagram push), safe
+// from the Kotlin UI thread. NOT android-gated — send_input exists on the host build too, so these
+// compile everywhere (parity with nativeConnect/nativeClose). The wire codes are the GameStream
+// conventions: buttons 1=left/2=middle/3=right/4=X1/5=X2; scroll axis 0=vertical/1=horizontal,
+// signed 120-unit delta, +=up/right; keys are Windows VK (mapped from KEYCODE_* on the Kotlin side).
+
+/// `NativeBridge.nativeSendPointerMove(handle, dx, dy)` — relative mouse motion (screen +y down).
+#[no_mangle]
+pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendPointerMove(
+    _env: JNIEnv,
+    _this: JObject,
+    handle: jlong,
+    dx: jint,
+    dy: jint,
+) {
+    if handle == 0 {
+        return;
+    }
+    // SAFETY: live handle per the nativeConnect/nativeClose contract; send_input is &self.
+    let h = unsafe { &*(handle as *const SessionHandle) };
+    let _ = h.client.send_input(&InputEvent {
+        kind: InputKind::MouseMove,
+        _pad: [0; 3],
+        code: 0,
+        x: dx,
+        y: dy,
+        flags: 0,
+    });
+}
+
+/// `NativeBridge.nativeSendPointerButton(handle, button, down)` — one button transition.
+/// `button`: GameStream id (1=left, 2=middle, 3=right, 4=X1, 5=X2). `down`: 1=press, 0=release.
+#[no_mangle]
+pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendPointerButton(
+    _env: JNIEnv,
+    _this: JObject,
+    handle: jlong,
+    button: jint,
+    down: jboolean,
+) {
+    if handle == 0 {
+        return;
+    }
+    // SAFETY: live handle per the contract.
+    let h = unsafe { &*(handle as *const SessionHandle) };
+    let _ = h.client.send_input(&InputEvent {
+        kind: if down != 0 {
+            InputKind::MouseButtonDown
+        } else {
+            InputKind::MouseButtonUp
+        },
+        _pad: [0; 3],
+        code: button as u32,
+        x: 0,
+        y: 0,
+        flags: 0,
+    });
+}
+
+/// `NativeBridge.nativeSendScroll(handle, axis, delta)` — one scroll step. `axis`: 0=vertical,
+/// 1=horizontal. `delta`: signed, WHEEL_DELTA(120)-scaled, +=up/right.
+#[no_mangle]
+pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendScroll(
+    _env: JNIEnv,
+    _this: JObject,
+    handle: jlong,
+    axis: jint,
+    delta: jint,
+) {
+    if handle == 0 {
+        return;
+    }
+    // SAFETY: live handle per the contract.
+    let h = unsafe { &*(handle as *const SessionHandle) };
+    let _ = h.client.send_input(&InputEvent {
+        kind: InputKind::MouseScroll,
+        _pad: [0; 3],
+        code: axis as u32,
+        x: delta,
+        y: 0,
+        flags: 0,
+    });
+}
+
+/// `NativeBridge.nativeSendKey(handle, vk, down, mods)` — one key transition. `vk`: Windows
+/// Virtual-Key code (0 = unmapped → dropped). `down`: 1=press, 0=release. `mods`: VK modifier
+/// bitmask (0 for now — the host folds modifiers from the L/R modifier key events themselves).
+#[no_mangle]
+pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendKey(
+    _env: JNIEnv,
+    _this: JObject,
+    handle: jlong,
+    vk: jint,
+    down: jboolean,
+    mods: jint,
+) {
+    if handle == 0 || vk == 0 {
+        return;
+    }
+    // SAFETY: live handle per the contract.
+    let h = unsafe { &*(handle as *const SessionHandle) };
+    let _ = h.client.send_input(&InputEvent {
+        kind: if down != 0 {
+            InputKind::KeyDown
+        } else {
+            InputKind::KeyUp
+        },
+        _pad: [0; 3],
+        code: vk as u32,
+        x: 0,
+        y: 0,
+        flags: mods as u32,
+    });
+}