feat(android): input forwarding — keyboard + touch trackpad → send_input
ci / rust (push) Failing after 0s
ci / web (push) Failing after 0s
ci / docs-site (push) Failing after 0s
ci / bench (push) Failing after 1s
android / android (push) Failing after 0s
deb / build-publish (push) Failing after 0s
decky / build-publish (push) Failing after 1s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Failing after 0s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 6s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 6s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 7s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 5s
docker / deploy-docs (push) Has been skipped
flatpak / build-publish (push) Failing after 4s
rpm / build-publish (bazzite, punktfunk-fedora-rpm) (push) Failing after 40s
apple / swift (push) Successful in 53s
rpm / build-publish (fedora-44, punktfunk-fedora44-rpm) (push) Failing after 1m21s
ci / rust (push) Failing after 0s
ci / web (push) Failing after 0s
ci / docs-site (push) Failing after 0s
ci / bench (push) Failing after 1s
android / android (push) Failing after 0s
deb / build-publish (push) Failing after 0s
decky / build-publish (push) Failing after 1s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Failing after 0s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 6s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 6s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 7s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 5s
docker / deploy-docs (push) Has been skipped
flatpak / build-publish (push) Failing after 4s
rpm / build-publish (bazzite, punktfunk-fedora-rpm) (push) Failing after 40s
apple / swift (push) Successful in 53s
rpm / build-publish (fedora-44, punktfunk-fedora44-rpm) (push) Failing after 1m21s
M4 Android stage 1 (input). Kotlin captures input and forwards it over JNI to NativeClient::send_input (the connector is linked as a Rust crate). - crates/punktfunk-android: 4 JNI send fns (pointer move / button / scroll / key) building InputEvent with the GameStream wire codes — ungated, &self on the Sync connector (safe from the UI thread). - clients/android: Keymap.kt (Android KEYCODE_* -> Windows VK, the host's wire contract, mirroring the Linux/Apple tables); Activity-level dispatchKeyEvent forwards hardware keys to the active session (above the Compose focus system, so it's reliable); a Compose touch-trackpad overlay -- 1-finger drag -> relative move, tap -> left click, 2-finger drag -> scroll. Verified live (emulator -> gamescope host on the LAN box, synthetic `adb input`): host received 31 input datagrams (input=31) and libei injected KeyDown/KeyUp, MouseButtonDown/Up and MouseMove all emitted=true. Physical-mouse pointer capture + gamepad are next. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package io.unom.punktfunk
|
||||
|
||||
import android.os.Bundle
|
||||
import android.view.KeyEvent
|
||||
import android.view.SurfaceHolder
|
||||
import android.view.SurfaceView
|
||||
import android.view.WindowManager
|
||||
@@ -8,7 +9,10 @@ import androidx.activity.ComponentActivity
|
||||
import androidx.activity.compose.BackHandler
|
||||
import androidx.activity.compose.setContent
|
||||
import androidx.activity.enableEdgeToEdge
|
||||
import androidx.compose.foundation.gestures.awaitEachGesture
|
||||
import androidx.compose.foundation.gestures.awaitFirstDown
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
@@ -30,16 +34,27 @@ import androidx.compose.runtime.rememberCoroutineScope
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.input.pointer.pointerInput
|
||||
import androidx.compose.ui.input.pointer.positionChange
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.text.input.KeyboardType
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.viewinterop.AndroidView
|
||||
import io.unom.punktfunk.kit.Keymap
|
||||
import io.unom.punktfunk.kit.NativeBridge
|
||||
import kotlin.math.abs
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
|
||||
class MainActivity : ComponentActivity() {
|
||||
/**
|
||||
* The active stream session handle (0 = not streaming). Set by [StreamScreen] while it's shown.
|
||||
* `dispatchKeyEvent` is the earliest, most reliable key hook — above Compose's focus system —
|
||||
* so hardware keys are forwarded to the host regardless of which view holds focus.
|
||||
*/
|
||||
var streamHandle: Long = 0L
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
enableEdgeToEdge()
|
||||
@@ -49,6 +64,33 @@ class MainActivity : ComponentActivity() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun dispatchKeyEvent(event: KeyEvent): Boolean {
|
||||
val handle = streamHandle
|
||||
if (handle != 0L) {
|
||||
when (event.keyCode) {
|
||||
// Leave these to the system even while streaming.
|
||||
KeyEvent.KEYCODE_BACK, // → BackHandler leaves the stream
|
||||
KeyEvent.KEYCODE_VOLUME_UP,
|
||||
KeyEvent.KEYCODE_VOLUME_DOWN,
|
||||
KeyEvent.KEYCODE_VOLUME_MUTE,
|
||||
KeyEvent.KEYCODE_POWER -> {}
|
||||
else -> {
|
||||
val down = when (event.action) {
|
||||
KeyEvent.ACTION_DOWN -> true
|
||||
KeyEvent.ACTION_UP -> false
|
||||
else -> return super.dispatchKeyEvent(event)
|
||||
}
|
||||
val vk = Keymap.toVk(event.keyCode)
|
||||
if (vk != 0) {
|
||||
NativeBridge.nativeSendKey(handle, vk, down, 0)
|
||||
return true // consumed — don't let the system also act on it
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return super.dispatchKeyEvent(event)
|
||||
}
|
||||
}
|
||||
|
||||
/** Scaffold mode requested from the host (WxH@Hz). TODO: derive from the display. */
|
||||
@@ -131,11 +173,14 @@ private fun ConnectScreen(onConnected: (Long) -> Unit) {
|
||||
@Composable
|
||||
private fun StreamScreen(handle: Long, onDisconnect: () -> Unit) {
|
||||
val context = LocalContext.current
|
||||
val window = (context as? ComponentActivity)?.window
|
||||
val activity = context as? MainActivity
|
||||
val window = activity?.window
|
||||
|
||||
DisposableEffect(handle) {
|
||||
window?.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
|
||||
activity?.streamHandle = handle // route hardware keys to this session
|
||||
onDispose {
|
||||
activity?.streamHandle = 0L
|
||||
window?.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
|
||||
// Leaving the stream: stop the audio + decode threads and tear down the session.
|
||||
NativeBridge.nativeStopAudio(handle)
|
||||
@@ -146,24 +191,62 @@ private fun StreamScreen(handle: Long, onDisconnect: () -> Unit) {
|
||||
|
||||
BackHandler { onDisconnect() }
|
||||
|
||||
AndroidView(
|
||||
modifier = Modifier.fillMaxSize(),
|
||||
factory = { ctx ->
|
||||
SurfaceView(ctx).apply {
|
||||
holder.addCallback(object : SurfaceHolder.Callback {
|
||||
override fun surfaceCreated(holder: SurfaceHolder) {
|
||||
NativeBridge.nativeStartVideo(handle, holder.surface)
|
||||
NativeBridge.nativeStartAudio(handle)
|
||||
}
|
||||
Box(modifier = Modifier.fillMaxSize()) {
|
||||
AndroidView(
|
||||
modifier = Modifier.fillMaxSize(),
|
||||
factory = { ctx ->
|
||||
SurfaceView(ctx).apply {
|
||||
holder.addCallback(object : SurfaceHolder.Callback {
|
||||
override fun surfaceCreated(holder: SurfaceHolder) {
|
||||
NativeBridge.nativeStartVideo(handle, holder.surface)
|
||||
NativeBridge.nativeStartAudio(handle)
|
||||
}
|
||||
|
||||
override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {}
|
||||
override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {}
|
||||
|
||||
override fun surfaceDestroyed(holder: SurfaceHolder) {
|
||||
NativeBridge.nativeStopAudio(handle)
|
||||
NativeBridge.nativeStopVideo(handle)
|
||||
override fun surfaceDestroyed(holder: SurfaceHolder) {
|
||||
NativeBridge.nativeStopAudio(handle)
|
||||
NativeBridge.nativeStopVideo(handle)
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
)
|
||||
// Touch virtual-trackpad overlay: 1-finger drag → relative mouse move; tap → left click;
|
||||
// 2-finger drag → scroll. (Physical-mouse pointer capture comes in a later increment.)
|
||||
Box(
|
||||
Modifier.fillMaxSize().pointerInput(handle) {
|
||||
awaitEachGesture {
|
||||
val first = awaitFirstDown(requireUnconsumed = false)
|
||||
var moved = false
|
||||
var maxFingers = 1
|
||||
while (true) {
|
||||
val ev = awaitPointerEvent()
|
||||
val fingers = ev.changes.count { it.pressed }
|
||||
if (fingers == 0) break
|
||||
if (fingers > maxFingers) maxFingers = fingers
|
||||
val primary = ev.changes.firstOrNull { it.id == first.id } ?: ev.changes.first()
|
||||
val d = primary.positionChange()
|
||||
if (abs(d.x) > 0.5f || abs(d.y) > 0.5f) {
|
||||
moved = true
|
||||
if (fingers >= 2) {
|
||||
// screen +y down → wire +up, so negate y. Coarse divisor; tune live.
|
||||
val sy = (-d.y / 4f).toInt()
|
||||
val sx = (d.x / 4f).toInt()
|
||||
if (sy != 0) NativeBridge.nativeSendScroll(handle, 0, sy * 120)
|
||||
if (sx != 0) NativeBridge.nativeSendScroll(handle, 1, sx * 120)
|
||||
} else {
|
||||
NativeBridge.nativeSendPointerMove(handle, d.x.toInt(), d.y.toInt())
|
||||
}
|
||||
}
|
||||
ev.changes.forEach { it.consume() }
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
)
|
||||
if (!moved && maxFingers == 1) {
|
||||
NativeBridge.nativeSendPointerButton(handle, 1, true)
|
||||
NativeBridge.nativeSendPointerButton(handle, 1, false)
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user