unom/punktfunk
1
0
Fork 0
Code Issues Pull Requests Actions 17 Packages Projects Releases 18 Wiki Activity

feat(android): input forwarding — keyboard + touch trackpad → send_input

Browse Source 
M4 Android stage 1 (input). Kotlin captures input and forwards it over JNI to
NativeClient::send_input (the connector is linked as a Rust crate).

- crates/punktfunk-android: 4 JNI send fns (pointer move / button / scroll / key)
  building InputEvent with the GameStream wire codes — ungated, &self on the Sync
  connector (safe from the UI thread).
- clients/android: Keymap.kt (Android KEYCODE_* -> Windows VK, the host's wire
  contract, mirroring the Linux/Apple tables); Activity-level dispatchKeyEvent forwards
  hardware keys to the active session (above the Compose focus system, so it's reliable);
  a Compose touch-trackpad overlay -- 1-finger drag -> relative move, tap -> left click,
  2-finger drag -> scroll.

Verified live (emulator -> gamescope host on the LAN box, synthetic `adb input`): host
received 31 input datagrams (input=31) and libei injected KeyDown/KeyUp, MouseButtonDown/Up
and MouseMove all emitted=true. Physical-mouse pointer capture + gamepad are next.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Enrico Bühler
2026-06-15 09:49:26 +02:00
parent 4debc4b821
commit bb47c811ee
4 changed files with 310 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
clients/android/app/src/main/kotlin/io/unom/punktfunk/MainActivity.kt
+101 -18
View File
@@ -1,6 +1,7 @@
package io.unom.punktfunk
import android.os.Bundle
import android.view.KeyEvent
import android.view.SurfaceHolder
import android.view.SurfaceView
import android.view.WindowManager
@@ -8,7 +9,10 @@ import androidx.activity.ComponentActivity
import androidx.activity.compose.BackHandler
import androidx.activity.compose.setContent
import androidx.activity.enableEdgeToEdge
import androidx.compose.foundation.gestures.awaitEachGesture
import androidx.compose.foundation.gestures.awaitFirstDown
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
@@ -30,16 +34,27 @@ import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.input.pointer.pointerInput
import androidx.compose.ui.input.pointer.positionChange
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.input.KeyboardType
import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import io.unom.punktfunk.kit.Keymap
import io.unom.punktfunk.kit.NativeBridge
import kotlin.math.abs
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
class MainActivity : ComponentActivity() {
/**
* The active stream session handle (0 = not streaming). Set by [StreamScreen] while it's shown.
* `dispatchKeyEvent` is the earliest, most reliable key hook — above Compose's focus system —
* so hardware keys are forwarded to the host regardless of which view holds focus.
*/
var streamHandle: Long = 0L
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
enableEdgeToEdge()
@@ -49,6 +64,33 @@ class MainActivity : ComponentActivity() {
}
}
}
override fun dispatchKeyEvent(event: KeyEvent): Boolean {
val handle = streamHandle
if (handle != 0L) {
when (event.keyCode) {
// Leave these to the system even while streaming.
KeyEvent.KEYCODE_BACK, // → BackHandler leaves the stream
KeyEvent.KEYCODE_VOLUME_UP,
KeyEvent.KEYCODE_VOLUME_DOWN,
KeyEvent.KEYCODE_VOLUME_MUTE,
KeyEvent.KEYCODE_POWER -> {}
else -> {
val down = when (event.action) {
KeyEvent.ACTION_DOWN -> true
KeyEvent.ACTION_UP -> false
else -> return super.dispatchKeyEvent(event)
}
val vk = Keymap.toVk(event.keyCode)
if (vk != 0) {
NativeBridge.nativeSendKey(handle, vk, down, 0)
return true // consumed — don't let the system also act on it
}
}
}
}
return super.dispatchKeyEvent(event)
}
}
/** Scaffold mode requested from the host (WxH@Hz). TODO: derive from the display. */
@@ -131,11 +173,14 @@ private fun ConnectScreen(onConnected: (Long) -> Unit) {
@Composable
private fun StreamScreen(handle: Long, onDisconnect: () -> Unit) {
val context = LocalContext.current
val window = (context as? ComponentActivity)?.window
val activity = context as? MainActivity
val window = activity?.window
DisposableEffect(handle) {
window?.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
activity?.streamHandle = handle // route hardware keys to this session
onDispose {
activity?.streamHandle = 0L
window?.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
// Leaving the stream: stop the audio + decode threads and tear down the session.
NativeBridge.nativeStopAudio(handle)
@@ -146,24 +191,62 @@ private fun StreamScreen(handle: Long, onDisconnect: () -> Unit) {
BackHandler { onDisconnect() }
AndroidView(
modifier = Modifier.fillMaxSize(),
factory = { ctx ->
SurfaceView(ctx).apply {
holder.addCallback(object : SurfaceHolder.Callback {
override fun surfaceCreated(holder: SurfaceHolder) {
NativeBridge.nativeStartVideo(handle, holder.surface)
NativeBridge.nativeStartAudio(handle)
}
Box(modifier = Modifier.fillMaxSize()) {
AndroidView(
modifier = Modifier.fillMaxSize(),
factory = { ctx ->
SurfaceView(ctx).apply {
holder.addCallback(object : SurfaceHolder.Callback {
override fun surfaceCreated(holder: SurfaceHolder) {
NativeBridge.nativeStartVideo(handle, holder.surface)
NativeBridge.nativeStartAudio(handle)
}
override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {}
override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {}
override fun surfaceDestroyed(holder: SurfaceHolder) {
NativeBridge.nativeStopAudio(handle)
NativeBridge.nativeStopVideo(handle)
override fun surfaceDestroyed(holder: SurfaceHolder) {
NativeBridge.nativeStopAudio(handle)
NativeBridge.nativeStopVideo(handle)
}
})
}
},
)
// Touch virtual-trackpad overlay: 1-finger drag → relative mouse move; tap → left click;
// 2-finger drag → scroll. (Physical-mouse pointer capture comes in a later increment.)
Box(
Modifier.fillMaxSize().pointerInput(handle) {
awaitEachGesture {
val first = awaitFirstDown(requireUnconsumed = false)
var moved = false
var maxFingers = 1
while (true) {
val ev = awaitPointerEvent()
val fingers = ev.changes.count { it.pressed }
if (fingers == 0) break
if (fingers > maxFingers) maxFingers = fingers
val primary = ev.changes.firstOrNull { it.id == first.id } ?: ev.changes.first()
val d = primary.positionChange()
if (abs(d.x) > 0.5f || abs(d.y) > 0.5f) {
moved = true
if (fingers >= 2) {
// screen +y down → wire +up, so negate y. Coarse divisor; tune live.
val sy = (-d.y / 4f).toInt()
val sx = (d.x / 4f).toInt()
if (sy != 0) NativeBridge.nativeSendScroll(handle, 0, sy * 120)
if (sx != 0) NativeBridge.nativeSendScroll(handle, 1, sx * 120)
} else {
NativeBridge.nativeSendPointerMove(handle, d.x.toInt(), d.y.toInt())
}
}
ev.changes.forEach { it.consume() }
}
})
}
},
)
if (!moved && maxFingers == 1) {
NativeBridge.nativeSendPointerButton(handle, 1, true)
NativeBridge.nativeSendPointerButton(handle, 1, false)
}
}
},
)
}
}