From bd4e15b68d2730e20883840bb266030329648660 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Thu, 2 Jul 2026 11:04:43 +0200 Subject: [PATCH] refactor(android): split session JNI into modules, HUD-gated stats, AAudio open retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - native: the 756-line session.rs becomes session/{mod,connect,input,planes}.rs around a SessionHandle (connect lifecycle + trust, input plane shims, plane start/stop + stats drain). - Decode-stats sampling is HUD-gated (nativeSetVideoStatsEnabled): with the overlay hidden the decode thread skips the per-AU clock read + lock; enabling resets the measurement window. - audio: the AAudio open path is a per-sharing-mode try_open closure — the realtime callback state (ring, prime, free-list) is rebuilt per attempt, so a failed exclusive-mode try can't leak state into the shared-mode retry. - Kotlin: ConnectScreen/StreamScreen slimmed by extracting ConnectDialogs, StatsOverlay and TouchInput. Co-Authored-By: Claude Fable 5 --- clients/android/README.md | 11 +- .../io/unom/punktfunk/ConnectDialogs.kt | 355 ++++++++ .../kotlin/io/unom/punktfunk/ConnectScreen.kt | 388 ++------- .../kotlin/io/unom/punktfunk/StatsOverlay.kt | 98 +++ .../kotlin/io/unom/punktfunk/StreamScreen.kt | 283 +------ .../kotlin/io/unom/punktfunk/TouchInput.kt | 184 +++++ .../io/unom/punktfunk/kit/NativeBridge.kt | 8 + clients/android/native/Cargo.toml | 4 +- clients/android/native/src/audio.rs | 225 +++--- clients/android/native/src/decode.rs | 158 +++- clients/android/native/src/lib.rs | 8 +- clients/android/native/src/mic.rs | 150 +++- clients/android/native/src/session.rs | 762 ------------------ clients/android/native/src/session/connect.rs | 244 ++++++ clients/android/native/src/session/input.rs | 159 ++++ clients/android/native/src/session/mod.rs | 124 +++ clients/android/native/src/session/planes.rs | 236 ++++++ clients/android/native/src/stats.rs | 57 +- 18 files changed, 1922 insertions(+), 1532 deletions(-) create mode 100644 clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectDialogs.kt create mode 100644 clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt create mode 100644 clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt delete mode 100644 clients/android/native/src/session.rs create mode 100644 clients/android/native/src/session/connect.rs create mode 100644 clients/android/native/src/session/input.rs create mode 100644 clients/android/native/src/session/mod.rs create mode 100644 clients/android/native/src/session/planes.rs diff --git a/clients/android/README.md b/clients/android/README.md index d1495d7..ce8b4d2 100644 --- a/clients/android/README.md +++ b/clients/android/README.md @@ -9,7 +9,7 @@ couch (D-pad / gamepad focus navigation). - **Hardware decode** — NDK `AMediaCodec` HEVC → `SurfaceView`, including **HDR10** (Main10 / BT.2020 PQ), with low-latency tuning and a live stats HUD. -- **Audio both ways** — Opus + Oboe playback with a jitter ring, plus mic uplink to the host. +- **Audio both ways** — Opus + AAudio playback with a jitter ring, plus mic uplink to the host. - **Controller support** — buttons + axes with rumble and HID feedback (lightbar / adaptive triggers); D-pad / gamepad focus navigation for TV and phone. - **Find hosts automatically** — native mDNS discovery; first connect does a one-time **SPAKE2 PIN @@ -33,18 +33,19 @@ machine, trust logic) instead of re-porting it into Kotlin. | Side | Owns | |------|------| -| **Rust** (`native/` → `libpunktfunk_android.so`) | the JNI seam, `NativeClient` (QUIC control + UDP data plane), AnnexB → `AMediaCodec` decode (incl. HDR10), Opus + Oboe audio + mic, controller feedback, latency math, trust/pairing, `mdns-sd` discovery | +| **Rust** (`native/` → `libpunktfunk_android.so`) | the JNI seam, `NativeClient` (QUIC control + UDP data plane), AnnexB → `AMediaCodec` decode (incl. HDR10), Opus + AAudio audio + mic, controller feedback, latency math, trust/pairing, `mdns-sd` discovery | | **Kotlin** (`app/`, `kit/`) | Compose UI, `SurfaceView` lifecycle, input capture, the Wi-Fi `MulticastLock` + permission UX, Keystore identity | The single seam is `io.unom.punktfunk.kit.NativeBridge` ⇄ `Java_io_unom_punktfunk_kit_NativeBridge_*`. ``` native/ Rust cdylib (workspace member) — links punktfunk-core directly - src/lib.rs JNI seam (connect/pair, input, plane getters, versions) - src/session.rs session lifecycle + plane pumps + src/lib.rs crate doc · JNI_OnLoad · version probes + src/session/ session lifecycle: connect/pair + trust, plane start/stop, input shims src/decode.rs AnnexB → AMediaCodec HEVC hardware decode → SurfaceView (incl. HDR10) - src/audio.rs · src/mic.rs Opus + Oboe playback / mic uplink + src/audio.rs · src/mic.rs Opus + AAudio playback / mic uplink src/feedback.rs · src/stats.rs rumble + HID feedback; live video stats + src/discovery.rs native mdns-sd browse of the host's _punktfunk._udp advert app/ :app — Compose UI: Connect / Settings / Stream (phone + TV) kit/ :kit — NativeBridge · native mDNS discovery · Gamepad · Keymap · Keystore identity ``` diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectDialogs.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectDialogs.kt new file mode 100644 index 0000000..f6f4be1 --- /dev/null +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectDialogs.kt @@ -0,0 +1,355 @@ +package io.unom.punktfunk + +import android.os.Build +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.text.KeyboardOptions +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.Button +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.ModalBottomSheet +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.material3.rememberModalBottomSheetState +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.text.input.KeyboardType +import androidx.compose.ui.unit.dp +import io.unom.punktfunk.kit.NativeBridge +import io.unom.punktfunk.kit.security.ClientIdentity +import io.unom.punktfunk.kit.security.KnownHost +import io.unom.punktfunk.models.PendingTrust +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +/** + * The "Add host" bottom sheet: optional name + address + port, then connect at [modeLabel]. Field + * state stays hoisted in ConnectScreen so a dismissed sheet keeps its half-typed values. + */ +@OptIn(ExperimentalMaterial3Api::class) +@Composable +internal fun AddHostSheet( + hostName: String, + onHostNameChange: (String) -> Unit, + host: String, + onHostChange: (String) -> Unit, + port: String, + onPortChange: (String) -> Unit, + connecting: Boolean, + modeLabel: String, + onDismiss: () -> Unit, + onConnect: (host: String, port: Int, name: String) -> Unit, +) { + val scope = rememberCoroutineScope() + val sheetState = rememberModalBottomSheetState() + ModalBottomSheet( + onDismissRequest = onDismiss, + sheetState = sheetState, + ) { + Column( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 24.dp) + .padding(bottom = 32.dp), + ) { + Text("Add a host", style = MaterialTheme.typography.titleLarge) + Spacer(Modifier.height(4.dp)) + Text( + "Enter its address. You'll pair with the host's PIN on first connect.", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + Spacer(Modifier.height(20.dp)) + OutlinedTextField( + value = hostName, + onValueChange = onHostNameChange, + label = { Text("Name (optional)") }, + placeholder = { Text("e.g. Living Room") }, + singleLine = true, + modifier = Modifier.fillMaxWidth(), + ) + Spacer(Modifier.height(16.dp)) + OutlinedTextField( + value = host, + onValueChange = onHostChange, + label = { Text("Host") }, + singleLine = true, + modifier = Modifier.fillMaxWidth(), + ) + Spacer(Modifier.height(16.dp)) + OutlinedTextField( + value = port, + onValueChange = { v -> onPortChange(v.filter { it.isDigit() }.take(5)) }, + label = { Text("Port") }, + singleLine = true, + keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number), + modifier = Modifier.fillMaxWidth(), + ) + Spacer(Modifier.height(20.dp)) + Button( + enabled = !connecting && host.isNotBlank() && port.isNotBlank(), + onClick = { + val h = host.trim() + val p = port.toIntOrNull() ?: 9777 + val n = hostName + scope.launch { sheetState.hide() }.invokeOnCompletion { + onDismiss() + onConnect(h, p, n) + } + }, + modifier = Modifier.fillMaxWidth(), + ) { Text("Connect ($modeLabel)") } + } + } +} + +/** First connection to a host that advertised pair=optional: offer TOFU, but pitch PIN pairing. */ +@Composable +internal fun TrustNewHostDialog( + pt: PendingTrust, + onTrust: () -> Unit, + onPairInstead: () -> Unit, + onDismiss: () -> Unit, +) { + AlertDialog( + onDismissRequest = onDismiss, + title = { Text("Trust this host?") }, + text = { + Column { + Text("First connection to ${pt.host}:${pt.port}.") + pt.advertisedFp?.let { Text("Fingerprint ${it.take(16)}…") } + Text( + "This host allows trust-on-first-use, but that can't tell an impostor " + + "from the real host. Pairing with a PIN is stronger — it proves both sides.", + ) + } + }, + confirmButton = { + TextButton(onClick = onTrust) { Text("Trust (TOFU)") } + }, + dismissButton = { + Row { + TextButton(onClick = onPairInstead) { Text("Pair with PIN…") } + TextButton(onClick = onDismiss) { Text("Cancel") } + } + }, + ) +} + +/** The pinned fingerprint no longer matches — force re-pairing (never a silent re-trust). */ +@Composable +internal fun FingerprintChangedDialog( + pt: PendingTrust, + onRepair: () -> Unit, + onDismiss: () -> Unit, +) { + AlertDialog( + onDismissRequest = onDismiss, + title = { Text("Host identity changed") }, + text = { + Text( + "The pinned fingerprint for ${pt.host} no longer matches what it now " + + "advertises. This can mean a host reinstall — or an impostor. Re-pair " + + "with the host's PIN to continue.", + ) + }, + confirmButton = { + TextButton(onClick = onRepair) { Text("Re-pair") } + }, + dismissButton = { + TextButton(onClick = onDismiss) { Text("Cancel") } + }, + ) +} + +/** + * A fresh pair=required (or manual/unknown-policy) host: offer the two ways in. "Request access" is + * the no-PIN path — connect and wait for the operator to click Approve in the host's console; + * "Use a PIN…" switches to the SPAKE2 ceremony. + */ +@Composable +internal fun RequestAccessDialog( + pt: PendingTrust, + onRequestAccess: () -> Unit, + onUsePin: () -> Unit, + onDismiss: () -> Unit, +) { + AlertDialog( + onDismissRequest = onDismiss, + title = { Text("Pairing required") }, + text = { + Column { + Text("${pt.host}:${pt.port} requires pairing before it will stream.") + Text( + "Request access and approve this device in the host's console (or web " + + "UI) — no PIN needed. Or pair with the 4-digit PIN the host displays.", + ) + } + }, + confirmButton = { + TextButton(onClick = onRequestAccess) { Text("Request access") } + }, + dismissButton = { + Row { + TextButton(onClick = onUsePin) { Text("Use a PIN…") } + TextButton(onClick = onDismiss) { Text("Cancel") } + } + }, + ) +} + +/** + * The SPAKE2 PIN ceremony dialog. Runs [NativeBridge.nativePair] off the UI thread itself (the + * pin/name/error state is dialog-local); on success hands the host's verified fingerprint to + * [onPaired], which saves + connects. Dismissal is blocked while a pair attempt is in flight. + */ +@Composable +internal fun PairPinDialog( + pt: PendingTrust, + identity: ClientIdentity?, + onPaired: (fpHex: String) -> Unit, + onDismiss: () -> Unit, +) { + val scope = rememberCoroutineScope() + var pin by remember(pt) { mutableStateOf("") } + var name by remember(pt) { mutableStateOf(Build.MODEL ?: "Android") } + var pairing by remember(pt) { mutableStateOf(false) } + var err by remember(pt) { mutableStateOf(null) } + AlertDialog( + onDismissRequest = { if (!pairing) onDismiss() }, + title = { Text("Pair with PIN") }, + text = { + Column { + Text("Enter the 4-digit PIN shown on the host.") + OutlinedTextField( + value = pin, + onValueChange = { v -> pin = v.filter { it.isDigit() }.take(4) }, + label = { Text("PIN") }, + singleLine = true, + keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number), + ) + OutlinedTextField( + value = name, + onValueChange = { name = it }, + label = { Text("This device") }, + singleLine = true, + ) + err?.let { Text(it, color = MaterialTheme.colorScheme.error) } + } + }, + confirmButton = { + TextButton( + enabled = !pairing && pin.length == 4 && identity != null, + onClick = { + val id = identity + if (id != null) { + pairing = true + err = null + scope.launch { + val fp = withContext(Dispatchers.IO) { + NativeBridge.nativePair( + pt.host, pt.port, id.certPem, id.privateKeyPem, pin, name, + ) + } + pairing = false + if (fp.isNotEmpty()) { + onPaired(fp) // verified host fp — caller saves + connects + } else { + err = "Pairing failed — wrong PIN, or the host isn't armed." + } + } + } + }, + ) { Text(if (pairing) "Pairing…" else "Pair") } + }, + dismissButton = { + TextButton(enabled = !pairing, onClick = onDismiss) { Text("Cancel") } + }, + ) +} + +/** + * The no-PIN "request access" wait: the connect is parked on the host until the operator approves + * this device. Cancel returns the UI immediately — the caller trips the per-attempt flag so a late + * approval is torn down silently (see ConnectScreen.requestAccess) and resumes discovery. + */ +@Composable +internal fun AwaitingApprovalDialog(hostLabel: String, onCancel: () -> Unit) { + AlertDialog( + onDismissRequest = onCancel, + title = { Text("Waiting for approval") }, + text = { + val deviceName = Build.MODEL ?: "this device" + Column(verticalArrangement = Arrangement.spacedBy(12.dp)) { + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(12.dp), + ) { + CircularProgressIndicator(modifier = Modifier.size(20.dp), strokeWidth = 2.dp) + Text("Approve this device on $hostLabel.") + } + Text( + "Open the host's console (or web UI) and approve “$deviceName”. It connects " + + "automatically once you approve — no PIN needed.", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + }, + confirmButton = {}, + dismissButton = { + TextButton(onClick = onCancel) { Text("Cancel") } + }, + ) +} + +/** + * Rename a saved host's label (discovered hosts are named by mDNS; this is how you give one a + * friendly name like "Living Room" after pairing). Keyed by the host so reopening resets the field. + */ +@Composable +internal fun RenameHostDialog( + target: KnownHost, + onRename: (String) -> Unit, + onDismiss: () -> Unit, +) { + var newName by remember(target) { mutableStateOf(target.name) } + AlertDialog( + onDismissRequest = onDismiss, + title = { Text("Rename host") }, + text = { + OutlinedTextField( + value = newName, + onValueChange = { newName = it }, + label = { Text("Name") }, + placeholder = { Text(target.address) }, + singleLine = true, + ) + }, + confirmButton = { + TextButton( + enabled = newName.isNotBlank(), + onClick = { onRename(newName.trim()) }, + ) { Text("Save") } + }, + dismissButton = { + TextButton(onClick = onDismiss) { Text("Cancel") } + }, + ) +} diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectScreen.kt index ad110dd..2621c04 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectScreen.kt @@ -6,11 +6,6 @@ import android.content.pm.PackageManager import android.os.Build import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.contract.ActivityResultContracts -import androidx.compose.animation.AnimatedVisibility -import androidx.compose.animation.fadeIn -import androidx.compose.animation.fadeOut -import androidx.compose.animation.scaleIn -import androidx.compose.animation.scaleOut import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column @@ -27,24 +22,14 @@ import androidx.compose.foundation.lazy.grid.GridCells import androidx.compose.foundation.lazy.grid.GridItemSpan import androidx.compose.foundation.lazy.grid.LazyVerticalGrid import androidx.compose.foundation.lazy.grid.items -import androidx.compose.foundation.rememberScrollState -import androidx.compose.foundation.text.KeyboardOptions -import androidx.compose.foundation.verticalScroll import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.Add -import androidx.compose.material3.AlertDialog -import androidx.compose.material3.Button import androidx.compose.material3.CircularProgressIndicator -import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.ExtendedFloatingActionButton import androidx.compose.material3.Icon import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.ModalBottomSheet -import androidx.compose.material3.OutlinedTextField import androidx.compose.material3.Surface import androidx.compose.material3.Text -import androidx.compose.material3.TextButton -import androidx.compose.material3.rememberModalBottomSheetState import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect @@ -56,7 +41,6 @@ import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.text.input.KeyboardType import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp import androidx.core.content.ContextCompat @@ -99,7 +83,6 @@ private class RequestAccessState(val target: PendingTrust) { val cancelled = AtomicBoolean(false) } -@OptIn(ExperimentalMaterial3Api::class) @Composable fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) { val scope = rememberCoroutineScope() @@ -162,6 +145,26 @@ fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) { // it survives a DHCP address change; else by address:port). Mirrors the Apple client. val discoveredUnsaved = discovered.filter { dh -> savedHosts.none { it.matches(dh) } } + // The one place the full nativeConnect is issued (shared by the normal connect and the + // request-access path), including the HDR/gamepad derivation both need. + suspend fun connectNative(id: ClientIdentity, targetHost: String, targetPort: Int, pinHex: String, timeoutMs: Int): Long { + // Advertise HDR only when the user enabled it AND this device's display can present it + // (else the host sends a proper SDR stream rather than PQ the panel would mis-tone-map). + val hdrEnabled = settings.hdrEnabled && displaySupportsHdr(context) + // "Automatic" resolves to a concrete pad type from the connected controller's VID/PID + // (Android exposes no controller-type enum) — parity with the Linux/Apple clients. An + // explicit choice is passed through unchanged. + val gamepadPref = Gamepad.resolvePref(settings.gamepad) + return withContext(Dispatchers.IO) { + NativeBridge.nativeConnect( + targetHost, targetPort, w, h, hz, + id.certPem, id.privateKeyPem, pinHex, + settings.bitrateKbps, settings.compositor, gamepadPref, + hdrEnabled, settings.audioChannels, settings.preferredCodec(), timeoutMs, + ) + } + } + // Issue the actual connect with identity + (optional) pin. On a TOFU connect (pinHex null), // pin the fingerprint the host presented (as an unpaired known host) so the next connect goes // straight through and it appears in the saved-hosts list. @@ -175,22 +178,7 @@ fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) { status = "Connecting to $targetHost:$targetPort…" discovery.stop() // free the Wi-Fi radio before the stream session scope.launch { - // Advertise HDR only when the user enabled it AND this device's display can present it - // (else the host sends a proper SDR stream rather than PQ the panel would mis-tone-map). - val hdrEnabled = settings.hdrEnabled && displaySupportsHdr(context) - // "Automatic" resolves to a concrete pad type from the connected controller's VID/PID - // (Android exposes no controller-type enum) — parity with the Linux/Apple clients. An - // explicit choice is passed through unchanged. - val gamepadPref = Gamepad.resolvePref(settings.gamepad) - val handle = withContext(Dispatchers.IO) { - NativeBridge.nativeConnect( - targetHost, targetPort, w, h, hz, - id.certPem, id.privateKeyPem, pinHex ?: "", - settings.bitrateKbps, settings.compositor, gamepadPref, - hdrEnabled, settings.audioChannels, settings.preferredCodec(), - CONNECT_TIMEOUT_MS, - ) - } + val handle = connectNative(id, targetHost, targetPort, pinHex ?: "", CONNECT_TIMEOUT_MS) connecting = false if (handle != 0L) { if (pinHex == null) { // TOFU: pin what we observed (unpaired) @@ -225,19 +213,10 @@ fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) { status = null discovery.stop() // free the Wi-Fi radio before the (parked) stream session scope.launch { - val hdrEnabled = settings.hdrEnabled && displaySupportsHdr(context) - val gamepadPref = Gamepad.resolvePref(settings.gamepad) // Pin the advertised fingerprint for a discovered host (defence against an impostor while // we wait); a manually-typed host has none, so trust-on-first-use. val pinHex = target.advertisedFp ?: "" - val handle = withContext(Dispatchers.IO) { - NativeBridge.nativeConnect( - target.host, target.port, w, h, hz, - id.certPem, id.privateKeyPem, pinHex, - settings.bitrateKbps, settings.compositor, gamepadPref, - hdrEnabled, settings.audioChannels, REQUEST_ACCESS_TIMEOUT_MS, - ) - } + val handle = connectNative(id, target.host, target.port, pinHex, REQUEST_ACCESS_TIMEOUT_MS) // Cancelled while we were parked: tear the (possibly just-approved) session down and // don't touch UI a fresh action may now own. if (req.cancelled.get()) { @@ -296,7 +275,6 @@ fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) { } } - val sheetState = rememberModalBottomSheetState() var showManualSheet by remember { mutableStateOf(false) } Box(Modifier.fillMaxSize()) { @@ -428,291 +406,87 @@ fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) { } } - AnimatedVisibility( - visible = true, // Static for now, could be based on scroll if needed - enter = scaleIn() + fadeIn(), - exit = scaleOut() + fadeOut(), + ExtendedFloatingActionButton( + onClick = { showManualSheet = true }, + icon = { Icon(Icons.Filled.Add, contentDescription = null) }, + text = { Text("Add host") }, + expanded = !connecting, modifier = Modifier .align(Alignment.BottomEnd) - .padding(20.dp) - ) { - ExtendedFloatingActionButton( - onClick = { showManualSheet = true }, - icon = { Icon(Icons.Filled.Add, contentDescription = null) }, - text = { Text("Add host") }, - expanded = !connecting, - ) - } + .padding(20.dp), + ) } if (showManualSheet) { - ModalBottomSheet( - onDismissRequest = { showManualSheet = false }, - sheetState = sheetState, - ) { - Column( - modifier = Modifier - .fillMaxWidth() - .padding(horizontal = 24.dp) - .padding(bottom = 32.dp), - ) { - Text("Add a host", style = MaterialTheme.typography.titleLarge) - Spacer(Modifier.height(4.dp)) - Text( - "Enter its address. You'll pair with the host's PIN on first connect.", - style = MaterialTheme.typography.bodyMedium, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - Spacer(Modifier.height(20.dp)) - OutlinedTextField( - value = hostName, - onValueChange = { hostName = it }, - label = { Text("Name (optional)") }, - placeholder = { Text("e.g. Living Room") }, - singleLine = true, - modifier = Modifier.fillMaxWidth(), - ) - Spacer(Modifier.height(16.dp)) - OutlinedTextField( - value = host, - onValueChange = { host = it }, - label = { Text("Host") }, - singleLine = true, - modifier = Modifier.fillMaxWidth(), - ) - Spacer(Modifier.height(16.dp)) - OutlinedTextField( - value = port, - onValueChange = { v -> port = v.filter { it.isDigit() }.take(5) }, - label = { Text("Port") }, - singleLine = true, - keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number), - modifier = Modifier.fillMaxWidth(), - ) - Spacer(Modifier.height(20.dp)) - Button( - enabled = !connecting && host.isNotBlank() && port.isNotBlank(), - onClick = { - val h = host.trim() - val p = port.toIntOrNull() ?: 9777 - val n = hostName - scope.launch { sheetState.hide() }.invokeOnCompletion { - showManualSheet = false - connect(h, p, manualName = n) - } - }, - modifier = Modifier.fillMaxWidth(), - ) { Text("Connect ($w×$h@$hz)") } - } - } + AddHostSheet( + hostName = hostName, + onHostNameChange = { hostName = it }, + host = host, + onHostChange = { host = it }, + port = port, + onPortChange = { port = it }, + connecting = connecting, + modeLabel = "$w×$h@$hz", + onDismiss = { showManualSheet = false }, + onConnect = { h2, p, n -> connect(h2, p, manualName = n) }, + ) } pendingTrust?.let { pt -> when (pt.kind) { - PendingTrust.Kind.TRUST_NEW -> AlertDialog( - onDismissRequest = { pendingTrust = null }, - title = { Text("Trust this host?") }, - text = { - Column { - Text("First connection to ${pt.host}:${pt.port}.") - pt.advertisedFp?.let { Text("Fingerprint ${it.take(16)}…") } - Text( - "This host allows trust-on-first-use, but that can't tell an impostor " + - "from the real host. Pairing with a PIN is stronger — it proves both sides.", - ) - } - }, - confirmButton = { - TextButton({ pendingTrust = null; doConnect(pt.host, pt.port, pt.name, null) }) { - Text("Trust (TOFU)") - } - }, - dismissButton = { - Row { - TextButton({ pendingTrust = pt.copy(kind = PendingTrust.Kind.PAIR) }) { - Text("Pair with PIN…") - } - TextButton({ pendingTrust = null }) { Text("Cancel") } - } - }, + PendingTrust.Kind.TRUST_NEW -> TrustNewHostDialog( + pt = pt, + onTrust = { pendingTrust = null; doConnect(pt.host, pt.port, pt.name, null) }, + onPairInstead = { pendingTrust = pt.copy(kind = PendingTrust.Kind.PAIR) }, + onDismiss = { pendingTrust = null }, ) - PendingTrust.Kind.FP_CHANGED -> AlertDialog( - onDismissRequest = { pendingTrust = null }, - title = { Text("Host identity changed") }, - text = { - Text( - "The pinned fingerprint for ${pt.host} no longer matches what it now " + - "advertises. This can mean a host reinstall — or an impostor. Re-pair " + - "with the host's PIN to continue.", - ) - }, - confirmButton = { - TextButton({ pendingTrust = pt.copy(kind = PendingTrust.Kind.PAIR) }) { Text("Re-pair") } - }, - dismissButton = { - TextButton({ pendingTrust = null }) { Text("Cancel") } - }, + PendingTrust.Kind.FP_CHANGED -> FingerprintChangedDialog( + pt = pt, + onRepair = { pendingTrust = pt.copy(kind = PendingTrust.Kind.PAIR) }, + onDismiss = { pendingTrust = null }, ) - // A fresh pair=required (or manual/unknown-policy) host: offer the two ways in. "Request - // access" is the no-PIN path — connect and wait for the operator to click Approve in the - // host's console; "Use a PIN…" switches to the SPAKE2 ceremony. - PendingTrust.Kind.REQUEST_ACCESS -> AlertDialog( - onDismissRequest = { pendingTrust = null }, - title = { Text("Pairing required") }, - text = { - Column { - Text("${pt.host}:${pt.port} requires pairing before it will stream.") - Text( - "Request access and approve this device in the host's console (or web " + - "UI) — no PIN needed. Or pair with the 4-digit PIN the host displays.", - ) - } - }, - confirmButton = { - TextButton({ pendingTrust = null; requestAccess(pt) }) { Text("Request access") } - }, - dismissButton = { - Row { - TextButton({ pendingTrust = pt.copy(kind = PendingTrust.Kind.PAIR) }) { - Text("Use a PIN…") - } - TextButton({ pendingTrust = null }) { Text("Cancel") } - } - }, + PendingTrust.Kind.REQUEST_ACCESS -> RequestAccessDialog( + pt = pt, + onRequestAccess = { pendingTrust = null; requestAccess(pt) }, + onUsePin = { pendingTrust = pt.copy(kind = PendingTrust.Kind.PAIR) }, + onDismiss = { pendingTrust = null }, + ) + PendingTrust.Kind.PAIR -> PairPinDialog( + pt = pt, + identity = identity, + onPaired = { fp -> + // Verified host fp — save as a paired known host, then connect pinned. + knownHostStore.save(KnownHost(pt.host, pt.port, pt.name, fp, paired = true)) + savedHosts = knownHostStore.all() + pendingTrust = null + doConnect(pt.host, pt.port, pt.name, fp) + }, + onDismiss = { pendingTrust = null }, ) - PendingTrust.Kind.PAIR -> { - var pin by remember(pt) { mutableStateOf("") } - var name by remember(pt) { mutableStateOf(Build.MODEL ?: "Android") } - var pairing by remember(pt) { mutableStateOf(false) } - var err by remember(pt) { mutableStateOf(null) } - AlertDialog( - onDismissRequest = { if (!pairing) pendingTrust = null }, - title = { Text("Pair with PIN") }, - text = { - Column { - Text("Enter the 4-digit PIN shown on the host.") - OutlinedTextField( - value = pin, - onValueChange = { v -> pin = v.filter { it.isDigit() }.take(4) }, - label = { Text("PIN") }, - singleLine = true, - keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number), - ) - OutlinedTextField( - value = name, - onValueChange = { name = it }, - label = { Text("This device") }, - singleLine = true, - ) - err?.let { Text(it, color = MaterialTheme.colorScheme.error) } - } - }, - confirmButton = { - TextButton( - enabled = !pairing && pin.length == 4 && identity != null, - onClick = { - val id = identity - if (id != null) { - pairing = true - err = null - scope.launch { - val fp = withContext(Dispatchers.IO) { - NativeBridge.nativePair( - pt.host, pt.port, id.certPem, id.privateKeyPem, pin, name, - ) - } - pairing = false - if (fp.isNotEmpty()) { - // Verified host fp — save as a paired known host. - knownHostStore.save( - KnownHost(pt.host, pt.port, pt.name, fp, paired = true), - ) - savedHosts = knownHostStore.all() - pendingTrust = null - doConnect(pt.host, pt.port, pt.name, fp) - } else { - err = "Pairing failed — wrong PIN, or the host isn't armed." - } - } - } - }, - ) { Text(if (pairing) "Pairing…" else "Pair") } - }, - dismissButton = { - TextButton(enabled = !pairing, onClick = { pendingTrust = null }) { Text("Cancel") } - }, - ) - } } } - // The no-PIN "request access" wait: the connect is parked on the host until the operator - // approves this device. Cancel returns the UI immediately — it trips the per-attempt flag so a - // late approval is torn down silently (see requestAccess) and resumes discovery. awaiting?.let { req -> - fun cancel() { - req.cancelled.set(true) - awaiting = null - connecting = false - discovery.start() // the request may still be pending on the host; keep scanning - } - AlertDialog( - onDismissRequest = { cancel() }, - title = { Text("Waiting for approval") }, - text = { - val deviceName = Build.MODEL ?: "this device" - Column(verticalArrangement = Arrangement.spacedBy(12.dp)) { - Row( - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(12.dp), - ) { - CircularProgressIndicator(modifier = Modifier.size(20.dp), strokeWidth = 2.dp) - Text("Approve this device on ${req.target.name}.") - } - Text( - "Open the host's console (or web UI) and approve “$deviceName”. It connects " + - "automatically once you approve — no PIN needed.", - style = MaterialTheme.typography.bodyMedium, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } - }, - confirmButton = {}, - dismissButton = { - TextButton(onClick = { cancel() }) { Text("Cancel") } + AwaitingApprovalDialog( + hostLabel = req.target.name, + onCancel = { + req.cancelled.set(true) + awaiting = null + connecting = false + discovery.start() // the request may still be pending on the host; keep scanning }, ) } - // Rename a saved host's label (discovered hosts are named by mDNS; this is how you give one a - // friendly name like "Living Room" after pairing). Keyed by the host so reopening resets the field. renameTarget?.let { kh -> - var newName by remember(kh) { mutableStateOf(kh.name) } - AlertDialog( - onDismissRequest = { renameTarget = null }, - title = { Text("Rename host") }, - text = { - OutlinedTextField( - value = newName, - onValueChange = { newName = it }, - label = { Text("Name") }, - placeholder = { Text(kh.address) }, - singleLine = true, - ) - }, - confirmButton = { - TextButton( - enabled = newName.isNotBlank(), - onClick = { - knownHostStore.rename(kh.address, kh.port, newName.trim()) - savedHosts = knownHostStore.all() - renameTarget = null - }, - ) { Text("Save") } - }, - dismissButton = { - TextButton(onClick = { renameTarget = null }) { Text("Cancel") } + RenameHostDialog( + target = kh, + onRename = { newName -> + knownHostStore.rename(kh.address, kh.port, newName) + savedHosts = knownHostStore.all() + renameTarget = null }, + onDismiss = { renameTarget = null }, ) } } diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt new file mode 100644 index 0000000..d56e413 --- /dev/null +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StatsOverlay.kt @@ -0,0 +1,98 @@ +package io.unom.punktfunk + +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import io.unom.punktfunk.kit.NativeBridge +import kotlin.math.roundToInt + +/** + * The live stats overlay — mirrors the Apple client's HUD. Reads the 14-double layout from + * [NativeBridge.nativeVideoStats]: + * `[fps, mbps, latP50Ms, latP95Ms, latValid, skew, w, h, hz, dropped, bitDepth, colorPrimaries, + * colorTransfer, chromaFormatIdc]`. The trailing four (present on a current native lib) describe the + * negotiated video feed and render as a codec/depth/colour/chroma line; older layouts just omit it. + */ +@Composable +internal fun StatsOverlay(s: DoubleArray, modifier: Modifier = Modifier) { + if (s.size < 10) return + val w = s[6].toInt() + val h = s[7].toInt() + val hz = s[8].toInt() + val latValid = s[4] != 0.0 + val skew = s[5] != 0.0 + val dropped = s[9].toLong() + Column( + modifier = modifier + .background(Color.Black.copy(alpha = 0.45f), RoundedCornerShape(6.dp)) + .padding(horizontal = 8.dp, vertical = 4.dp), + ) { + Text( + "$w×$h@$hz ${s[0].roundToInt()} fps ${"%.1f".format(s[1])} Mb/s", + color = Color.White, + fontFamily = FontFamily.Monospace, + fontSize = 12.sp, + ) + videoFeedLine(s)?.let { feed -> + Text( + feed, + color = Color.White, + fontFamily = FontFamily.Monospace, + fontSize = 12.sp, + ) + } + if (latValid) { + val tag = if (skew) "" else " (same-host)" + Text( + "capture→client ${"%.1f".format(s[2])}/${"%.1f".format(s[3])} ms p50/p95$tag", + color = Color.White, + fontFamily = FontFamily.Monospace, + fontSize = 12.sp, + ) + } + if (dropped > 0) { + Text( + "dropped $dropped", + color = Color(0xFFFFB0B0), + fontFamily = FontFamily.Monospace, + fontSize = 12.sp, + ) + } + } +} + +/** + * Format the negotiated video-feed descriptor from the trailing four stats doubles + * `[bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc]`, e.g. + * `HEVC · 10-bit · HDR (BT.2020 PQ) · 4:2:0`. Returns `null` on a pre-video-feed layout (< 14 doubles) + * so the overlay simply omits the line. The codes are CICP / H.273: transfer 16 = PQ, 18 = HLG (else + * SDR); primaries 9 = BT.2020, 1 = BT.709; chroma_format_idc 1 = 4:2:0, 2 = 4:2:2, 3 = 4:4:4. The + * Android decoder is always HEVC (`video/hevc`). + */ +private fun videoFeedLine(s: DoubleArray): String? { + if (s.size < 14) return null + val bitDepth = s[10].toInt() + val primaries = s[11].toInt() + val transfer = s[12].toInt() + val chromaIdc = s[13].toInt() + val depthLabel = if (bitDepth > 0) "$bitDepth-bit" else "8-bit" + val (dynamicRange, colorSpace) = when (transfer) { + 16 -> "HDR" to "BT.2020 PQ" + 18 -> "HDR" to "BT.2020 HLG" + else -> "SDR" to if (primaries == 9) "BT.2020" else "BT.709" + } + val chromaLabel = when (chromaIdc) { + 3 -> "4:4:4" + 2 -> "4:2:2" + else -> "4:2:0" + } + return "HEVC · $depthLabel · $dynamicRange ($colorSpace) · $chromaLabel" +} diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt index a9b0748..f889ac3 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt @@ -7,15 +7,9 @@ import android.view.SurfaceHolder import android.view.SurfaceView import android.view.WindowManager import androidx.activity.compose.BackHandler -import androidx.compose.foundation.background -import androidx.compose.foundation.gestures.awaitEachGesture -import androidx.compose.foundation.gestures.awaitFirstDown import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.shape.RoundedCornerShape -import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect @@ -25,12 +19,9 @@ import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier -import androidx.compose.ui.graphics.Color import androidx.compose.ui.input.pointer.pointerInput import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.text.font.FontFamily import androidx.compose.ui.unit.dp -import androidx.compose.ui.unit.sp import androidx.compose.ui.viewinterop.AndroidView import androidx.core.content.ContextCompat import androidx.core.view.WindowCompat @@ -41,25 +32,6 @@ import io.unom.punktfunk.kit.GamepadFeedback import io.unom.punktfunk.kit.NativeBridge import java.util.concurrent.atomic.AtomicBoolean import kotlinx.coroutines.delay -import kotlin.math.abs -import kotlin.math.hypot -import kotlin.math.roundToInt - -// Touch-gesture tuning (px / ms). TAP_SLOP: movement under this still counts as a tap, not a drag. -// TAP_DRAG_MS: a new touch within this long after a tap starts a left-button drag. SCROLL_DIV: px of -// two-finger pan per wheel notch (smaller = faster scroll). -private const val TAP_SLOP = 12f -private const val TAP_DRAG_MS = 250L -private const val SCROLL_DIV = 4f - -// Trackpad-mode pointer ballistics (relative one-finger motion). POINTER_SENS: base finger-px → -// host-px gain (~1:1, never twitchy). The rest is mild acceleration so a flick crosses the screen -// while a slow drag stays precise: above ACCEL_SPEED_FLOOR px/ms the gain ramps by ACCEL_GAIN per -// px/ms, capped at ACCEL_MAX (so a fast swipe can't fling the cursor uncontrollably). -private const val POINTER_SENS = 1.3f -private const val ACCEL_GAIN = 0.6f -private const val ACCEL_SPEED_FLOOR = 0.3f -private const val ACCEL_MAX = 3.0f @Composable fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { @@ -76,18 +48,25 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { Manifest.permission.RECORD_AUDIO, ) == PackageManager.PERMISSION_GRANTED - // Live decode stats for the HUD. Poll once a second for the whole stream (cheap, and each call - // drains+resets the native window so it never grows unbounded even while the overlay is hidden); - // `showStats` only gates rendering. A 3-finger tap toggles it live; the default comes from Settings. + // Live decode stats for the HUD. `showStats` gates the whole pipeline: the native per-frame + // sampling (nativeSetVideoStatsEnabled — hidden HUD costs one atomic load per frame) AND the + // 1 s poll loop, which only runs while the overlay is visible. Enabling resets the native + // window, so re-showing never renders stale data. A 3-finger tap toggles it live; the default + // comes from Settings. val initialSettings = remember { SettingsStore(context).load() } var stats by remember { mutableStateOf(null) } var showStats by remember { mutableStateOf(initialSettings.statsHudEnabled) } // Touch model is fixed per session (re-keys the gesture handler below if it ever changes). val trackpad = initialSettings.trackpadMode - LaunchedEffect(handle) { - while (true) { - delay(1000) - stats = NativeBridge.nativeVideoStats(handle) + LaunchedEffect(handle, showStats) { + NativeBridge.nativeSetVideoStatsEnabled(handle, showStats) + if (showStats) { + while (true) { + delay(1000) + stats = NativeBridge.nativeVideoStats(handle) + } + } else { + stats = null // drop the last snapshot so a re-show never flashes stale numbers } } @@ -169,240 +148,12 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { if (showStats) { stats?.let { StatsOverlay(it, Modifier.align(Alignment.TopStart).padding(12.dp)) } } - // Touch → mouse. Two models, chosen by the Trackpad-mode setting: - // • trackpad (default): the cursor STAYS where it is on touch-down and moves by the finger's - // relative delta (MouseMove) with mild pointer acceleration — swipe to nudge, lift and - // re-swipe to walk it across, tap to click where it is. This is what makes the cursor - // reachable on a small screen. - // • direct (opt-out): the cursor jumps to the finger and follows it (MouseMoveAbs, - // host-normalized against the overlay size), the old "direct pointing" behaviour. - // Both share the same gesture vocabulary: tap = left click; two-finger tap = right click; - // two-finger drag = scroll; tap-then-press-and-drag = left-drag (text selection / moving - // windows); three-finger tap = toggle the stats HUD. + // Touch → mouse (trackpad vs. direct pointing + the shared gesture vocabulary — see + // streamTouchInput in TouchInput.kt). Box( Modifier.fillMaxSize().pointerInput(handle, trackpad) { - var lastTapUp = 0L - var lastTapX = 0f - var lastTapY = 0f - fun moveAbs(x: Float, y: Float) { - val sw = size.width - val sh = size.height - if (sw <= 0 || sh <= 0) return - NativeBridge.nativeSendPointerAbs( - handle, - x.coerceIn(0f, (sw - 1).toFloat()).roundToInt(), - y.coerceIn(0f, (sh - 1).toFloat()).roundToInt(), - sw, - sh, - ) - } - awaitEachGesture { - val down = awaitFirstDown(requireUnconsumed = false) - val startX = down.position.x - val startY = down.position.y - // A touch landing just after a quick tap nearby = tap-and-drag: hold the left - // button for this whole gesture (laptop-trackpad convention). - val isDrag = down.uptimeMillis - lastTapUp < TAP_DRAG_MS && - abs(startX - lastTapX) < TAP_SLOP && abs(startY - lastTapY) < TAP_SLOP - lastTapUp = 0L // consume the arming either way - // Direct mode jumps the cursor to the finger; trackpad mode leaves it put (the - // whole point — you nudge it with swipes instead). - if (!trackpad) moveAbs(startX, startY) - if (isDrag) NativeBridge.nativeSendPointerButton(handle, 1, true) - - var moved = false - var maxFingers = 1 - var scrolling = false - var prevCx = startX - var prevCy = startY - var upTime = down.uptimeMillis - // Trackpad relative-motion state: the tracked finger, its last position/time, and - // the sub-pixel remainder so a slow drag isn't lost to Int truncation. - var trackId = down.id - var prevX = startX - var prevY = startY - var prevT = down.uptimeMillis - var accX = 0f - var accY = 0f - - while (true) { - val ev = awaitPointerEvent() - val pressed = ev.changes.filter { it.pressed } - if (pressed.isEmpty()) { - upTime = ev.changes.firstOrNull()?.uptimeMillis ?: upTime - break - } - if (pressed.size > maxFingers) maxFingers = pressed.size - - if (pressed.size >= 2) { - // Two fingers → scroll by the centroid delta; never move the cursor. - val cx = (pressed.sumOf { it.position.x.toDouble() } / pressed.size).toFloat() - val cy = (pressed.sumOf { it.position.y.toDouble() } / pressed.size).toFloat() - if (!scrolling) { - scrolling = true - prevCx = cx - prevCy = cy - } - val sy = ((prevCy - cy) / SCROLL_DIV).toInt() // finger up → wheel up - val sx = ((cx - prevCx) / SCROLL_DIV).toInt() - if (sy != 0) { - NativeBridge.nativeSendScroll(handle, 0, sy * 120) - prevCy = cy - moved = true - } - if (sx != 0) { - NativeBridge.nativeSendScroll(handle, 1, sx * 120) - prevCx = cx - moved = true - } - } else if (!scrolling) { - // One finger (skipped once a gesture turned into a scroll, so dropping - // back to one finger doesn't jerk the cursor). - val p = pressed.firstOrNull { it.id == down.id } ?: pressed.first() - if (abs(p.position.x - startX) > TAP_SLOP || - abs(p.position.y - startY) > TAP_SLOP - ) { - moved = true - } - if (trackpad) { - // Relative: move by the finger delta × (sensitivity × acceleration), - // carrying the sub-pixel remainder. Re-anchor (zero delta this frame) - // if the tracked finger changed, so lifting one of several fingers - // never jumps the cursor. - if (p.id != trackId) { - trackId = p.id - prevX = p.position.x - prevY = p.position.y - prevT = p.uptimeMillis - } - val dx = p.position.x - prevX - val dy = p.position.y - prevY - val dt = (p.uptimeMillis - prevT).coerceAtLeast(1L) - prevX = p.position.x - prevY = p.position.y - prevT = p.uptimeMillis - val speed = hypot(dx, dy) / dt // finger px per ms - val accel = (1f + ACCEL_GAIN * (speed - ACCEL_SPEED_FLOOR).coerceAtLeast(0f)) - .coerceAtMost(ACCEL_MAX) - accX += dx * POINTER_SENS * accel - accY += dy * POINTER_SENS * accel - val outX = accX.toInt() // truncates toward zero → remainder kept w/ sign - val outY = accY.toInt() - if (outX != 0 || outY != 0) { - NativeBridge.nativeSendPointerMove(handle, outX, outY) - accX -= outX - accY -= outY - } - } else { - moveAbs(p.position.x, p.position.y) // direct: cursor follows the finger - } - } - ev.changes.forEach { it.consume() } - } - - if (isDrag) { - NativeBridge.nativeSendPointerButton(handle, 1, false) // end the drag - } else if (!moved) { - when { - maxFingers >= 3 -> showStats = !showStats // in-stream HUD toggle - maxFingers == 2 -> { // two-finger tap → right click - NativeBridge.nativeSendPointerButton(handle, 3, true) - NativeBridge.nativeSendPointerButton(handle, 3, false) - } - else -> { // tap → left click (at the cursor's current spot), arm tap-drag - NativeBridge.nativeSendPointerButton(handle, 1, true) - NativeBridge.nativeSendPointerButton(handle, 1, false) - lastTapUp = upTime - lastTapX = startX - lastTapY = startY - } - } - } - } + streamTouchInput(handle, trackpad, onToggleStats = { showStats = !showStats }) }, ) } } - -/** - * The live stats overlay — mirrors the Apple client's HUD. Reads the 14-double layout from - * [NativeBridge.nativeVideoStats]: - * `[fps, mbps, latP50Ms, latP95Ms, latValid, skew, w, h, hz, dropped, bitDepth, colorPrimaries, - * colorTransfer, chromaFormatIdc]`. The trailing four (present on a current native lib) describe the - * negotiated video feed and render as a codec/depth/colour/chroma line; older layouts just omit it. - */ -@Composable -internal fun StatsOverlay(s: DoubleArray, modifier: Modifier = Modifier) { - if (s.size < 10) return - val w = s[6].toInt() - val h = s[7].toInt() - val hz = s[8].toInt() - val latValid = s[4] != 0.0 - val skew = s[5] != 0.0 - val dropped = s[9].toLong() - Column( - modifier = modifier - .background(Color.Black.copy(alpha = 0.45f), RoundedCornerShape(6.dp)) - .padding(horizontal = 8.dp, vertical = 4.dp), - ) { - Text( - "$w×$h@$hz ${s[0].roundToInt()} fps ${"%.1f".format(s[1])} Mb/s", - color = Color.White, - fontFamily = FontFamily.Monospace, - fontSize = 12.sp, - ) - videoFeedLine(s)?.let { feed -> - Text( - feed, - color = Color.White, - fontFamily = FontFamily.Monospace, - fontSize = 12.sp, - ) - } - if (latValid) { - val tag = if (skew) "" else " (same-host)" - Text( - "capture→client ${"%.1f".format(s[2])}/${"%.1f".format(s[3])} ms p50/p95$tag", - color = Color.White, - fontFamily = FontFamily.Monospace, - fontSize = 12.sp, - ) - } - if (dropped > 0) { - Text( - "dropped $dropped", - color = Color(0xFFFFB0B0), - fontFamily = FontFamily.Monospace, - fontSize = 12.sp, - ) - } - } -} - -/** - * Format the negotiated video-feed descriptor from the trailing four stats doubles - * `[bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc]`, e.g. - * `HEVC · 10-bit · HDR (BT.2020 PQ) · 4:2:0`. Returns `null` on a pre-video-feed layout (< 14 doubles) - * so the overlay simply omits the line. The codes are CICP / H.273: transfer 16 = PQ, 18 = HLG (else - * SDR); primaries 9 = BT.2020, 1 = BT.709; chroma_format_idc 1 = 4:2:0, 2 = 4:2:2, 3 = 4:4:4. The - * Android decoder is always HEVC (`video/hevc`). - */ -private fun videoFeedLine(s: DoubleArray): String? { - if (s.size < 14) return null - val bitDepth = s[10].toInt() - val primaries = s[11].toInt() - val transfer = s[12].toInt() - val chromaIdc = s[13].toInt() - val depthLabel = if (bitDepth > 0) "$bitDepth-bit" else "8-bit" - val (dynamicRange, colorSpace) = when (transfer) { - 16 -> "HDR" to "BT.2020 PQ" - 18 -> "HDR" to "BT.2020 HLG" - else -> "SDR" to if (primaries == 9) "BT.2020" else "BT.709" - } - val chromaLabel = when (chromaIdc) { - 3 -> "4:4:4" - 2 -> "4:2:2" - else -> "4:2:0" - } - return "HEVC · $depthLabel · $dynamicRange ($colorSpace) · $chromaLabel" -} diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt new file mode 100644 index 0000000..715128b --- /dev/null +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt @@ -0,0 +1,184 @@ +package io.unom.punktfunk + +import androidx.compose.foundation.gestures.awaitEachGesture +import androidx.compose.foundation.gestures.awaitFirstDown +import androidx.compose.ui.input.pointer.PointerInputScope +import io.unom.punktfunk.kit.NativeBridge +import kotlin.math.abs +import kotlin.math.hypot +import kotlin.math.roundToInt + +// Touch-gesture tuning (px / ms). TAP_SLOP: movement under this still counts as a tap, not a drag. +// TAP_DRAG_MS: a new touch within this long after a tap starts a left-button drag. SCROLL_DIV: px of +// two-finger pan per wheel notch (smaller = faster scroll). +private const val TAP_SLOP = 12f +private const val TAP_DRAG_MS = 250L +private const val SCROLL_DIV = 4f + +// Trackpad-mode pointer ballistics (relative one-finger motion). POINTER_SENS: base finger-px → +// host-px gain (~1:1, never twitchy). The rest is mild acceleration so a flick crosses the screen +// while a slow drag stays precise: above ACCEL_SPEED_FLOOR px/ms the gain ramps by ACCEL_GAIN per +// px/ms, capped at ACCEL_MAX (so a fast swipe can't fling the cursor uncontrollably). +private const val POINTER_SENS = 1.3f +private const val ACCEL_GAIN = 0.6f +private const val ACCEL_SPEED_FLOOR = 0.3f +private const val ACCEL_MAX = 3.0f + +/** + * Touch → mouse, run inside the stream overlay's `pointerInput`. Two models, chosen by the + * Trackpad-mode setting: + * * trackpad (default): the cursor STAYS where it is on touch-down and moves by the finger's + * relative delta (MouseMove) with mild pointer acceleration — swipe to nudge, lift and + * re-swipe to walk it across, tap to click where it is. This is what makes the cursor + * reachable on a small screen. + * * direct (opt-out): the cursor jumps to the finger and follows it (MouseMoveAbs, + * host-normalized against the overlay size), the old "direct pointing" behaviour. + * + * Both share the same gesture vocabulary: tap = left click; two-finger tap = right click; + * two-finger drag = scroll; tap-then-press-and-drag = left-drag (text selection / moving + * windows); three-finger tap = [onToggleStats] (the stats HUD). + */ +internal suspend fun PointerInputScope.streamTouchInput( + handle: Long, + trackpad: Boolean, + onToggleStats: () -> Unit, +) { + var lastTapUp = 0L + var lastTapX = 0f + var lastTapY = 0f + fun moveAbs(x: Float, y: Float) { + val sw = size.width + val sh = size.height + if (sw <= 0 || sh <= 0) return + NativeBridge.nativeSendPointerAbs( + handle, + x.coerceIn(0f, (sw - 1).toFloat()).roundToInt(), + y.coerceIn(0f, (sh - 1).toFloat()).roundToInt(), + sw, + sh, + ) + } + awaitEachGesture { + val down = awaitFirstDown(requireUnconsumed = false) + val startX = down.position.x + val startY = down.position.y + // A touch landing just after a quick tap nearby = tap-and-drag: hold the left + // button for this whole gesture (laptop-trackpad convention). + val isDrag = down.uptimeMillis - lastTapUp < TAP_DRAG_MS && + abs(startX - lastTapX) < TAP_SLOP && abs(startY - lastTapY) < TAP_SLOP + lastTapUp = 0L // consume the arming either way + // Direct mode jumps the cursor to the finger; trackpad mode leaves it put (the + // whole point — you nudge it with swipes instead). + if (!trackpad) moveAbs(startX, startY) + if (isDrag) NativeBridge.nativeSendPointerButton(handle, 1, true) + + var moved = false + var maxFingers = 1 + var scrolling = false + var prevCx = startX + var prevCy = startY + var upTime = down.uptimeMillis + // Trackpad relative-motion state: the tracked finger, its last position/time, and + // the sub-pixel remainder so a slow drag isn't lost to Int truncation. + var trackId = down.id + var prevX = startX + var prevY = startY + var prevT = down.uptimeMillis + var accX = 0f + var accY = 0f + + while (true) { + val ev = awaitPointerEvent() + val pressed = ev.changes.filter { it.pressed } + if (pressed.isEmpty()) { + upTime = ev.changes.firstOrNull()?.uptimeMillis ?: upTime + break + } + if (pressed.size > maxFingers) maxFingers = pressed.size + + if (pressed.size >= 2) { + // Two fingers → scroll by the centroid delta; never move the cursor. + val cx = (pressed.sumOf { it.position.x.toDouble() } / pressed.size).toFloat() + val cy = (pressed.sumOf { it.position.y.toDouble() } / pressed.size).toFloat() + if (!scrolling) { + scrolling = true + prevCx = cx + prevCy = cy + } + val sy = ((prevCy - cy) / SCROLL_DIV).toInt() // finger up → wheel up + val sx = ((cx - prevCx) / SCROLL_DIV).toInt() + if (sy != 0) { + NativeBridge.nativeSendScroll(handle, 0, sy * 120) + prevCy = cy + moved = true + } + if (sx != 0) { + NativeBridge.nativeSendScroll(handle, 1, sx * 120) + prevCx = cx + moved = true + } + } else if (!scrolling) { + // One finger (skipped once a gesture turned into a scroll, so dropping + // back to one finger doesn't jerk the cursor). + val p = pressed.firstOrNull { it.id == down.id } ?: pressed.first() + if (abs(p.position.x - startX) > TAP_SLOP || + abs(p.position.y - startY) > TAP_SLOP + ) { + moved = true + } + if (trackpad) { + // Relative: move by the finger delta × (sensitivity × acceleration), + // carrying the sub-pixel remainder. Re-anchor (zero delta this frame) + // if the tracked finger changed, so lifting one of several fingers + // never jumps the cursor. + if (p.id != trackId) { + trackId = p.id + prevX = p.position.x + prevY = p.position.y + prevT = p.uptimeMillis + } + val dx = p.position.x - prevX + val dy = p.position.y - prevY + val dt = (p.uptimeMillis - prevT).coerceAtLeast(1L) + prevX = p.position.x + prevY = p.position.y + prevT = p.uptimeMillis + val speed = hypot(dx, dy) / dt // finger px per ms + val accel = (1f + ACCEL_GAIN * (speed - ACCEL_SPEED_FLOOR).coerceAtLeast(0f)) + .coerceAtMost(ACCEL_MAX) + accX += dx * POINTER_SENS * accel + accY += dy * POINTER_SENS * accel + val outX = accX.toInt() // truncates toward zero → remainder kept w/ sign + val outY = accY.toInt() + if (outX != 0 || outY != 0) { + NativeBridge.nativeSendPointerMove(handle, outX, outY) + accX -= outX + accY -= outY + } + } else { + moveAbs(p.position.x, p.position.y) // direct: cursor follows the finger + } + } + ev.changes.forEach { it.consume() } + } + + if (isDrag) { + NativeBridge.nativeSendPointerButton(handle, 1, false) // end the drag + } else if (!moved) { + when { + maxFingers >= 3 -> onToggleStats() // in-stream HUD toggle + maxFingers == 2 -> { // two-finger tap → right click + NativeBridge.nativeSendPointerButton(handle, 3, true) + NativeBridge.nativeSendPointerButton(handle, 3, false) + } + else -> { // tap → left click (at the cursor's current spot), arm tap-drag + NativeBridge.nativeSendPointerButton(handle, 1, true) + NativeBridge.nativeSendPointerButton(handle, 1, false) + lastTapUp = upTime + lastTapX = startX + lastTapY = startY + } + } + } + } +} diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index e4bb368..f8e39d5 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -114,6 +114,14 @@ object NativeBridge { */ external fun nativeVideoStats(handle: Long): DoubleArray? + /** + * Gate per-frame stats sampling on the HUD being visible: while disabled the decode thread + * skips the per-AU clock read + lock, so toggle this with the overlay (and only poll + * [nativeVideoStats] while it's on). Enabling resets the measurement window — no stale data. + * Sticky for the session (survives video stop/start). No-op on `0`. + */ + external fun nativeSetVideoStatsEnabled(handle: Long, enabled: Boolean) + /** * Start host→client audio: Opus decode → jitter ring → AAudio (LowLatency), all in Rust. No-op * if already started. Best-effort — a failure leaves video streaming. diff --git a/clients/android/native/Cargo.toml b/clients/android/native/Cargo.toml index c952a96..6d76e39 100644 --- a/clients/android/native/Cargo.toml +++ b/clients/android/native/Cargo.toml @@ -27,8 +27,8 @@ log = "0.4" mdns-sd = "0.20" # Android-only deps. Gated so `cargo build --workspace` on the Linux/macOS dev boxes + CI still -# compiles this crate (as a host cdylib) — the Android-framework glue (logging now; AMediaCodec via -# `ndk` and Oboe/Opus audio later) is only pulled in for the real `*-linux-android` targets. +# compiles this crate (as a host cdylib) — the Android-framework glue (logging, AMediaCodec + AAudio +# via `ndk`, the Opus codec) is only pulled in for the real `*-linux-android` targets. [target.'cfg(target_os = "android")'.dependencies] android_logger = "0.14" # NDK bindings. "media" = AMediaCodec/ANativeWindow (video); "audio" = AAudio (audio playback). diff --git a/clients/android/native/src/audio.rs b/clients/android/native/src/audio.rs index d70b942..e700adf 100644 --- a/clients/android/native/src/audio.rs +++ b/clients/android/native/src/audio.rs @@ -129,109 +129,140 @@ impl AudioPlayback { let jitter_headroom = JITTER_HEADROOM_MS * ms; let hard_cap_max = HARD_CAP_MS * ms; let counters = Arc::new(Counters::default()); - let (tx, rx) = sync_channel::>(RING_CHUNKS); - // Recycle free-list: drained PCM buffers go BACK to the decode thread to be refilled, so the - // realtime callback never frees heap (Android's Scudo allocator has unbounded free() tail - // latency — a free on the audio thread is an XRun = a click) and the decode thread rarely - // allocates. Same depth as the data channel. - let (free_tx, free_rx) = sync_channel::>(RING_CHUNKS); - // Realtime consumer state, owned by the callback (FnMut) — no lock: AAudio calls it from a - // single high-priority thread, and the decode thread only touches `tx`/`free_rx`. - let cb_counters = counters.clone(); - // Pre-reserve the ring so `extend` never reallocates on the realtime thread. Worst transient - // before the trim below = the hard cap plus one full channel of 5 ms (480-f32) frames — the - // punktfunk protocol always sends 5 ms Opus frames (host `audio_thread`); a larger frame - // would force a one-time realloc, asserted (not silently corrupted) in `decode_loop`. - let mut ring: VecDeque = VecDeque::with_capacity(hard_cap_max + RING_CHUNKS * 5 * ms); - let mut primed = false; - let mut empties: u32 = 0; // consecutive empty callbacks (de-prime hysteresis) - let mut cb_count: u32 = 0; // callbacks since open (throttles the XRun grow check) - let mut last_xrun: i32 = 0; // last AAudio XRun count we grew the buffer for - let callback = move |s: &AudioStream, data: *mut c_void, num_frames: i32| { - let want = num_frames as usize * channels; - // SAFETY: AAudio provides `num_frames * channel_count` F32 slots at `data`. - let out = unsafe { std::slice::from_raw_parts_mut(data as *mut f32, want) }; - // Drain decoded chunks into the ring WITHOUT freeing on the RT thread: `drain(..)` empties - // each Vec but keeps its capacity, then the empty buffer is handed back for reuse. The - // only RT-thread free is the rare case where the recycle channel is momentarily full. - while let Ok(mut chunk) = rx.try_recv() { - ring.extend(chunk.drain(..)); - let _ = free_tx.try_send(chunk); - } - // Jitter buffer: prime to ~40 ms (prime_floor) before playing and after a sustained drain; - // drop-oldest only above a wide ~120 ms band. Decoupled from the AAudio burst `want` (tiny - // on the LowLatency MMAP path) so the depth doesn't collapse to a single quantum. - let target = (3 * want).clamp(prime_floor, prime_ceil); - let hard_cap = (target + jitter_headroom).min(hard_cap_max); - while ring.len() > hard_cap { - ring.pop_front(); - } - if !primed && ring.len() >= target { - primed = true; - } - if primed { - for slot in out.iter_mut() { - *slot = ring.pop_front().unwrap_or(0.0); + // One open attempt at a given sharing mode. Everything the realtime callback captures + // (channels, ring, prime state) is rebuilt per attempt — `open_stream` consumes the builder + // AND the callback, so nothing survives a failed try to reuse. + let try_open = |sharing: AudioSharingMode| -> ndk::audio::Result<( + AudioStream, + SyncSender>, + Receiver>, + )> { + let (tx, rx) = sync_channel::>(RING_CHUNKS); + // Recycle free-list: drained PCM buffers go BACK to the decode thread to be refilled, so + // the realtime callback never frees heap (Android's Scudo allocator has unbounded free() + // tail latency — a free on the audio thread is an XRun = a click) and the decode thread + // rarely allocates. Same depth as the data channel. + let (free_tx, free_rx) = sync_channel::>(RING_CHUNKS); + + // Realtime consumer state, owned by the callback (FnMut) — no lock: AAudio calls it from + // a single high-priority thread, and the decode thread only touches `tx`/`free_rx`. + let cb_counters = counters.clone(); + // Pre-reserve the ring so `extend` never reallocates on the realtime thread. Worst + // transient before the trim below = the hard cap plus one full channel of 5 ms (480-f32) + // frames — the punktfunk protocol always sends 5 ms Opus frames (host `audio_thread`); a + // larger frame would force a one-time realloc, asserted (not silently corrupted) in + // `decode_loop`. + let mut ring: VecDeque = + VecDeque::with_capacity(hard_cap_max + RING_CHUNKS * 5 * ms); + let mut primed = false; + let mut empties: u32 = 0; // consecutive empty callbacks (de-prime hysteresis) + let mut cb_count: u32 = 0; // callbacks since open (throttles the XRun grow check) + let mut last_xrun: i32 = 0; // last AAudio XRun count we grew the buffer for + let callback = move |s: &AudioStream, data: *mut c_void, num_frames: i32| { + let want = num_frames as usize * channels; + // SAFETY: AAudio provides `num_frames * channel_count` F32 slots at `data`. + let out = unsafe { std::slice::from_raw_parts_mut(data as *mut f32, want) }; + // Drain decoded chunks into the ring WITHOUT freeing on the RT thread: `drain(..)` + // empties each Vec but keeps its capacity, then the empty buffer is handed back for + // reuse. The only RT-thread free is the rare case where the recycle channel is + // momentarily full. + while let Ok(mut chunk) = rx.try_recv() { + ring.extend(chunk.drain(..)); + let _ = free_tx.try_send(chunk); + } + // Jitter buffer: prime to ~40 ms (prime_floor) before playing and after a sustained + // drain; drop-oldest only above a wide ~120 ms band. Decoupled from the AAudio burst + // `want` (tiny on the LowLatency MMAP path) so the depth doesn't collapse to a single + // quantum. + let target = (3 * want).clamp(prime_floor, prime_ceil); + let hard_cap = (target + jitter_headroom).min(hard_cap_max); + while ring.len() > hard_cap { + ring.pop_front(); + } + if !primed && ring.len() >= target { + primed = true; + } + if primed { + for slot in out.iter_mut() { + *slot = ring.pop_front().unwrap_or(0.0); + } + cb_counters + .pcm_written + .fetch_add(num_frames as u64, Ordering::Relaxed); + } else { + out.fill(0.0); + cb_counters.underruns.fetch_add(1, Ordering::Relaxed); + } + // Re-prime only after a RUN of empty callbacks, not a single transient one — + // otherwise every momentary drain costs a fresh 40 ms silence (the old behaviour, + // self-inflicted crackle on any jitter spike). + if ring.is_empty() { + empties += 1; + if empties >= DEPRIME_AFTER_CALLBACKS { + primed = false; + } + } else { + empties = 0; } cb_counters - .pcm_written - .fetch_add(num_frames as u64, Ordering::Relaxed); - } else { - out.fill(0.0); - cb_counters.underruns.fetch_add(1, Ordering::Relaxed); - } - // Re-prime only after a RUN of empty callbacks, not a single transient one — otherwise - // every momentary drain costs a fresh 40 ms silence (the old behaviour, self-inflicted - // crackle on any jitter spike). - if ring.is_empty() { - empties += 1; - if empties >= DEPRIME_AFTER_CALLBACKS { - primed = false; + .ring_depth + .store(ring.len() as u64, Ordering::Relaxed); + // Google's AAudio anti-glitch technique: when the device reports new XRuns, grow the + // HW buffer by one burst (up to capacity). getXRunCount + setBufferSizeInFrames are + // both callback-safe / non-blocking, and set clamps to capacity so it self-limits. + // Throttled. + cb_count = cb_count.wrapping_add(1); + if cb_count % XRUN_CHECK_EVERY == 0 { + let xr = s.x_run_count(); + if xr > last_xrun { + last_xrun = xr; + let burst = s.frames_per_burst().max(1); + let grown = + (s.buffer_size_in_frames() + burst).min(s.buffer_capacity_in_frames()); + let _ = s.set_buffer_size_in_frames(grown); + } } - } else { - empties = 0; - } - cb_counters - .ring_depth - .store(ring.len() as u64, Ordering::Relaxed); - // Google's AAudio anti-glitch technique: when the device reports new XRuns, grow the HW - // buffer by one burst (up to capacity). getXRunCount + setBufferSizeInFrames are both - // callback-safe / non-blocking, and set clamps to capacity so it self-limits. Throttled. - cb_count = cb_count.wrapping_add(1); - if cb_count % XRUN_CHECK_EVERY == 0 { - let xr = s.x_run_count(); - if xr > last_xrun { - last_xrun = xr; - let burst = s.frames_per_burst().max(1); - let grown = - (s.buffer_size_in_frames() + burst).min(s.buffer_capacity_in_frames()); - let _ = s.set_buffer_size_in_frames(grown); - } - } - AudioCallbackResult::Continue + AudioCallbackResult::Continue + }; + + let stream = AudioStreamBuilder::new()? + .direction(AudioDirection::Output) + .sample_rate(SAMPLE_RATE) + // The wire order (FL FR FC LFE RL RR SL SR) is the standard AAudio/Android channel + // order, so this is an IDENTITY mapping — no permute. AAudio infers the 5.1/7.1 mask + // from `channel_count` (the ndk crate's builder exposes no setChannelMask); the host + // captures + Opus-encodes in exactly this order. + .channel_count(channels as i32) + .format(AudioFormat::PCM_Float) + .performance_mode(AudioPerformanceMode::LowLatency) + .sharing_mode(sharing) + .data_callback(Box::new(callback)) + .error_callback(Box::new(|_s, e| { + log::warn!("audio: AAudio error (device reroute/disconnect?): {e:?}"); + })) + .open_stream()?; + Ok((stream, tx, free_rx)) }; - let stream = AudioStreamBuilder::new() - .map_err(|e| log::error!("audio: AudioStreamBuilder::new: {e}")) - .ok()? - .direction(AudioDirection::Output) - .sample_rate(SAMPLE_RATE) - // The wire order (FL FR FC LFE RL RR SL SR) is the standard AAudio/Android channel - // order, so this is an IDENTITY mapping — no permute. AAudio infers the 5.1/7.1 mask - // from `channel_count` (the ndk crate's builder exposes no setChannelMask); the host - // captures + Opus-encodes in exactly this order. - .channel_count(channels as i32) - .format(AudioFormat::PCM_Float) - .performance_mode(AudioPerformanceMode::LowLatency) - .sharing_mode(AudioSharingMode::Shared) - .data_callback(Box::new(callback)) - .error_callback(Box::new(|_s, e| { - log::warn!("audio: AAudio error (device reroute/disconnect?): {e:?}"); - })) - .open_stream() - .map_err(|e| log::error!("audio: open_stream: {e}")) - .ok()?; + // Exclusive first — MMAP-exclusive is AAudio's lowest-latency path (once proven on-device it + // may also allow lowering the jitter-ring depths above; those stay put pending crackle + // testing) — and fall back to Shared when the device refuses (no MMAP, output claimed, …). + // The started-log below prints the mode the device actually GRANTED (`share=`): AAudio may + // still resolve an Exclusive request to Shared. + let (stream, tx, free_rx) = match try_open(AudioSharingMode::Exclusive) { + Ok(opened) => opened, + Err(e) => { + log::info!("audio: Exclusive open failed ({e}) — retrying Shared"); + match try_open(AudioSharingMode::Shared) { + Ok(opened) => opened, + Err(e) => { + log::error!("audio: open_stream: {e}"); + return None; + } + } + } + }; if let Err(e) = stream.request_start() { log::error!("audio: request_start: {e}"); diff --git a/clients/android/native/src/decode.rs b/clients/android/native/src/decode.rs index 7968ef0..21e030c 100644 --- a/clients/android/native/src/decode.rs +++ b/clients/android/native/src/decode.rs @@ -14,6 +14,7 @@ use ndk::media::media_format::MediaFormat; use ndk::native_window::{FrameRateCompatibility, NativeWindow}; use punktfunk_core::client::NativeClient; use punktfunk_core::error::PunktfunkError; +use punktfunk_core::session::Frame; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -53,6 +54,9 @@ pub fn run( ); // Ask for the low-latency decode path where the decoder supports it (no reordering buffer). format.set_i32("low-latency", 1); + // Best-effort vendor twin of the standard key: older Qualcomm decoders only honor their own + // extension. Unknown keys are ignored by other vendors' codecs, so this is safe to set blind. + format.set_i32("vendor.qti-ext-dec-low-latency.enable", 1); // Advisory low-latency hints (KEY_PRIORITY / KEY_OPERATING_RATE), ignored where unsupported: // realtime priority + the target frame rate, so vendor decoders (e.g. Qualcomm) run at full // clocks instead of a power-saving cadence that adds dequeue latency. @@ -102,6 +106,11 @@ pub fn run( let mut fed: u64 = 0; let mut rendered: u64 = 0; + let mut discarded: u64 = 0; + // The AU waiting for a free codec input buffer. `feed` is non-blocking; on transient input + // pressure the AU stays parked here instead of being dropped (a drop forces a keyframe + // round-trip) and we only pop the next one once it's queued. + let mut pending: Option = None; // Loss recovery: watch the host→client unrecoverable-drop count and ask for an IDR when it // climbs. let mut last_dropped = client.frames_dropped(); @@ -112,29 +121,61 @@ pub fn run( // The dataspace we've signalled on the Surface so far (None = default/SDR). Set reactively once // the decoder reports an HDR stream (see `drain`); avoids re-applying every format event. let mut applied_ds: Option = None; + // One thread feeds AND drains: the NDK AMediaCodec wrapper isn't documented thread-safe for + // cross-thread feed/drain, so instead of splitting threads the loop decouples the two — input + // dequeue is non-blocking (never stalls presentation of already-decoded frames) and the only + // blocking wait is a short output dequeue while input is backed up (decoder progress is exactly + // what frees the next input buffer). while !shutdown.load(Ordering::Relaxed) { - match client.next_frame(Duration::from_millis(5)) { - Ok(frame) => { - if fed == 0 { - let p = &frame.data; - log::info!( - "decode: first AU {} bytes, head {:02x?}", - p.len(), - &p[..p.len().min(6)] - ); + if pending.is_none() { + match client.next_frame(Duration::from_millis(5)) { + Ok(frame) => { + if fed == 0 { + let p = &frame.data; + log::info!( + "decode: first AU {} bytes, head {:02x?}", + p.len(), + &p[..p.len().min(6)] + ); + } + // HUD stat: capture→client-receipt latency = client_now + (host−client) − + // capture_pts. Gated on the HUD being visible — `enabled` first so the hidden + // steady state skips the wall-clock read and the lock entirely. + if stats.enabled() { + let lat_ns = + now_realtime_ns() + clock_offset as i128 - frame.pts_ns as i128; + let lat_us = (lat_ns > 0 && lat_ns < 10_000_000_000) + .then_some((lat_ns / 1000) as u64); + stats.note(frame.data.len(), lat_us, clock_offset != 0); + } + pending = Some(frame); } - fed += 1; - // HUD stat: capture→client-receipt latency = client_now + (host−client) − capture_pts. - let lat_ns = now_realtime_ns() + clock_offset as i128 - frame.pts_ns as i128; - let lat_us = - (lat_ns > 0 && lat_ns < 10_000_000_000).then_some((lat_ns / 1000) as u64); - stats.note(frame.data.len(), lat_us, clock_offset != 0); - feed(&codec, &frame.data, frame.pts_ns / 1000); + Err(PunktfunkError::NoFrame) => {} // timeout — still drain output below + Err(_) => break, // session closed } - Err(PunktfunkError::NoFrame) => {} // timeout — still drain output below - Err(_) => break, // session closed } - rendered += drain(&codec, &window, &mut applied_ds); + if let Some(frame) = pending.take() { + if feed(&codec, &frame.data, frame.pts_ns / 1000) { + fed += 1; + if fed % 300 == 0 { + log::info!("decode: fed={fed} rendered={rendered} discarded={discarded}"); + } + } else { + // No input buffer free — transient back-pressure. Keep the AU and let `drain` block + // briefly below; a released output buffer is what recycles an input slot. + pending = Some(frame); + } + } + // Drain every iteration. When input is blocked, wait ~2 ms on output so the loop rides + // decoder progress instead of busy-spinning against a full input queue. + let wait = if pending.is_some() { + Duration::from_millis(2) + } else { + Duration::ZERO + }; + let (r, d) = drain(&codec, &window, &mut applied_ds, wait); + rendered += r; + discarded += d; // Loss recovery: under infinite GOP the only recovery keyframe is one we request. The // reassembler drops unrecoverable AUs (frames_dropped); the decoder then conceals the @@ -152,14 +193,10 @@ pub fn run( log::debug!("decode: requested keyframe (loss recovery, dropped={dropped})"); } } - - if fed > 0 && fed % 300 == 0 { - log::info!("decode: fed={fed} rendered={rendered}"); - } } let _ = codec.stop(); - log::info!("decode: stopped (fed={fed} rendered={rendered})"); + log::info!("decode: stopped (fed={fed} rendered={rendered} discarded={discarded})"); } /// Wall-clock now in nanoseconds (CLOCK_REALTIME basis), to compare against the host-stamped @@ -189,9 +226,12 @@ fn boost_thread_priority() { } } -/// Copy one access unit into a codec input buffer and queue it. -fn feed(codec: &MediaCodec, au: &[u8], pts_us: u64) { - match codec.dequeue_input_buffer(Duration::from_millis(10)) { +/// Try to copy one access unit into a codec input buffer and queue it, without blocking. Returns +/// `false` only on `TryAgainLater` (no input buffer free) — the caller keeps the AU pending and +/// retries; a hard dequeue/queue error counts as consumed (retrying can't salvage the AU, and +/// parking it forever would wedge the loop on a broken codec). +fn feed(codec: &MediaCodec, au: &[u8], pts_us: u64) -> bool { + match codec.dequeue_input_buffer(Duration::ZERO) { Ok(DequeuedInputBufferResult::Buffer(mut buf)) => { let n = { let dst = buf.buffer_mut(); @@ -203,41 +243,63 @@ fn feed(codec: &MediaCodec, au: &[u8], pts_us: u64) { dst.len() ); } - for (slot, &b) in dst.iter_mut().zip(&au[..n]) { - slot.write(b); + // SAFETY: `au` and `dst` are distinct allocations (wire AU vs. codec buffer), both + // valid for `n` bytes; `MaybeUninit` is layout-identical to `u8`, so the cast + // write initializes exactly `dst[..n]`. + unsafe { + std::ptr::copy_nonoverlapping(au.as_ptr(), dst.as_mut_ptr().cast::(), n); } n }; if let Err(e) = codec.queue_input_buffer(buf, 0, n, pts_us, 0) { log::warn!("decode: queue_input_buffer: {e}"); } + true } - Ok(DequeuedInputBufferResult::TryAgainLater) => { - // No input buffer free right now; the AU is dropped (FEC/keyframes recover). + Ok(DequeuedInputBufferResult::TryAgainLater) => false, // caller keeps the AU pending + Err(e) => { + log::warn!("decode: dequeue_input_buffer: {e}"); + true } - Err(e) => log::warn!("decode: dequeue_input_buffer: {e}"), } } -/// Release any ready output buffers to the surface (render = true), latency-first. Returns the -/// number of frames presented. Also reacts to `OutputFormatChanged` to signal HDR on the Surface. -fn drain(codec: &MediaCodec, window: &NativeWindow, applied_ds: &mut Option) -> u64 { - let mut n = 0; +/// Dequeue every ready output buffer and present only the NEWEST (render = true), discarding the +/// rest (render = false) — when decode falls behind, a back-to-back burst of stale frames on glass +/// is worse than skipping straight to the freshest one (the Apple client's 1-slot newest-ready +/// ring, ported). `first_wait` is the timeout for the first dequeue only: zero normally, ~2 ms when +/// the caller's input is blocked so the loop waits on decoder progress instead of busy-spinning. +/// Returns `(rendered, discarded)`. Also reacts to `OutputFormatChanged` (which can interleave +/// between buffers — handled without losing the held buffer) to signal HDR on the Surface. +fn drain( + codec: &MediaCodec, + window: &NativeWindow, + applied_ds: &mut Option, + first_wait: Duration, +) -> (u64, u64) { + let mut held = None; // newest ready buffer so far, presented after the loop + let mut discarded: u64 = 0; + let mut wait = first_wait; loop { - match codec.dequeue_output_buffer(Duration::from_millis(0)) { + match codec.dequeue_output_buffer(wait) { Ok(DequeuedOutputBufferInfoResult::Buffer(buf)) => { - if let Err(e) = codec.release_output_buffer(buf, true) { - log::warn!("decode: release_output_buffer: {e}"); - break; + wait = Duration::ZERO; // only the first dequeue may block + if let Some(stale) = held.replace(buf) { + // A newer frame is ready — drop the held one without rendering. + if let Err(e) = codec.release_output_buffer(stale, false) { + log::warn!("decode: release_output_buffer(discard): {e}"); + } + discarded += 1; } - n += 1; } Ok(DequeuedOutputBufferInfoResult::OutputFormatChanged) => { // The decoder has parsed the SPS and now reports the stream's real colour signalling // (the AMediaCodec analogue of VideoToolbox's format description on the Apple client). // If it's HDR (BT.2020 PQ/HLG), tell the Surface so the compositor/display switch to // HDR; SDR streams leave the default dataspace alone. The decoder itself picks a - // Main10 path from the SPS — no profile override needed. Keep looping (buffers follow). + // Main10 path from the SPS — no profile override needed. Keep looping (buffers + // follow, and any held buffer stays held across this event). + wait = Duration::ZERO; if let Some(ds) = hdr_dataspace(codec) { if *applied_ds != Some(ds) { match window.set_buffers_data_space(ds) { @@ -252,7 +314,7 @@ fn drain(codec: &MediaCodec, window: &NativeWindow, applied_ds: &mut Option break, Err(e) => { log::warn!("decode: dequeue_output_buffer: {e}"); @@ -260,7 +322,15 @@ fn drain(codec: &MediaCodec, window: &NativeWindow, applied_ds: &mut Option rendered = 1, + Err(e) => log::warn!("decode: release_output_buffer: {e}"), + } + } + (rendered, discarded) } /// Map the decoder's reported output colour to a BT.2020 HDR dataspace, or `None` for SDR. The diff --git a/clients/android/native/src/lib.rs b/clients/android/native/src/lib.rs index 7e222ac..a3594b6 100644 --- a/clients/android/native/src/lib.rs +++ b/clients/android/native/src/lib.rs @@ -16,10 +16,10 @@ //! Wi-Fi `MulticastLock` + permission UX, Keystore identity). //! //! JNI symbols map to `io.unom.punktfunk.kit.NativeBridge` in the `:kit` Gradle module -//! (`clients/android`). The current surface is the scaffold's native-link proof -//! (`abiVersion`/`coreVersion`) plus the session handle lifecycle in [`session`]; the per-plane -//! pumps (video → AMediaCodec, audio → Oboe), input, audio, pairing and mode renegotiation are -//! the next milestone (see the TODOs in [`session`]). +//! (`clients/android`). The surface: the native-link proof (`abiVersion`/`coreVersion`), mDNS host +//! discovery ([`discovery`]), and the session lifecycle in [`session`] — connect/pair + the trust +//! surface, the per-plane pumps (video → AMediaCodec, audio ↔ AAudio, mic uplink), input, and +//! rumble/HID feedback ([`feedback`]). Mode renegotiation is still TODO (see [`session`]). use jni::objects::JObject; use jni::sys::jint; diff --git a/clients/android/native/src/mic.rs b/clients/android/native/src/mic.rs index 68820bf..6ae164e 100644 --- a/clients/android/native/src/mic.rs +++ b/clients/android/native/src/mic.rs @@ -1,9 +1,12 @@ //! Android microphone uplink (android-only): capture mic PCM via AAudio (LowLatency **input**), //! Opus-encode 20 ms stereo frames, and push them to the host over the connector's mic plane //! (`send_mic` → 0xCB datagram). The mirror of [`crate::audio`] in reverse: AAudio's realtime input -//! callback hands captured interleaved f32 to a channel; a worker thread we own does the Opus encode -//! + send (encoding is too heavy for the realtime callback, exactly as decode is on the playback -//! side). Format matches the host decoder + the Linux client: 48 kHz **stereo**, 20 ms, Opus VOIP. +//! callback hands captured interleaved f32 to a channel; a worker thread we own does the Opus +//! encode + send (encoding is too heavy for the realtime callback, exactly as decode is on the +//! playback side). Like the playback path, the realtime callback is allocation-free: captured +//! bursts are copied into pre-allocated buffers from a recycle free-list (pool empty = drop the +//! chunk, never allocate on the capture thread). Format matches the host decoder + the Linux +//! client: 48 kHz **stereo**, 20 ms, Opus VOIP. use ndk::audio::{ AudioCallbackResult, AudioDirection, AudioFormat, AudioPerformanceMode, AudioSharingMode, @@ -13,7 +16,7 @@ use punktfunk_core::client::NativeClient; use std::collections::VecDeque; use std::ffi::c_void; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::mpsc::{sync_channel, Receiver, RecvTimeoutError, TrySendError}; +use std::sync::mpsc::{sync_channel, Receiver, RecvTimeoutError, SyncSender, TrySendError}; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -23,6 +26,10 @@ const SAMPLE_RATE: i32 = 48_000; const FRAME_SAMPLES: usize = 960; /// Captured-chunk hand-off depth (each ~ one burst); drops on overflow (best-effort uplink). const RING_CHUNKS: usize = 64; +/// Free-list buffer capacity, in interleaved f32 samples: comfortably above a LowLatency input +/// burst (typically ≤ ~480 frames). A device with larger bursts costs each buffer a one-time grow +/// on the capture thread, after which the steady state is allocation-free again. +const CHUNK_CAP_SAMPLES: usize = 1920; // 20 ms stereo /// Opus VOIP target bitrate (speech; tunable). const MIC_BITRATE: i32 = 64_000; @@ -38,56 +45,109 @@ impl MicCapture { /// forwards captured PCM to a channel, then spawn the Opus encode + uplink thread. `None` on /// failure (the caller leaves the rest of the session streaming). pub fn start(client: Arc) -> Option { - let (tx, rx) = sync_channel::>(RING_CHUNKS); let captured = Arc::new(AtomicU64::new(0)); - let cb_captured = captured.clone(); + // Chunks discarded on the capture thread (free-list empty / encoder lagging); logged + // throttled from the encode worker. + let dropped = Arc::new(AtomicU64::new(0)); - let callback = move |_s: &AudioStream, data: *mut c_void, num_frames: i32| { - let n = num_frames as usize * CHANNELS; - // SAFETY: for an input stream AAudio provides `num_frames * channel_count` captured F32 - // samples at `data` (read-only for us). - let inp = unsafe { std::slice::from_raw_parts(data as *const f32, n) }; - match tx.try_send(inp.to_vec()) { - Ok(()) | Err(TrySendError::Full(_)) => {} // drop-newest if the encoder lags - Err(TrySendError::Disconnected(_)) => return AudioCallbackResult::Stop, + // One open attempt at a given sharing mode (same pattern as [`crate::audio`]: `open_stream` + // consumes the builder AND the callback, so each try rebuilds the channels it captures). + let try_open = |sharing: AudioSharingMode| -> ndk::audio::Result<( + AudioStream, + Receiver>, + SyncSender>, + )> { + let (tx, rx) = sync_channel::>(RING_CHUNKS); + // Recycle free-list, mirroring the playback path: the realtime capture callback must + // not touch the allocator (Android's Scudo has unbounded malloc/free tail latency — an + // allocation here is a missed burst), so it pops a pre-allocated buffer, copies the + // burst in and sends it; the encode worker returns drained buffers. Pool empty = DROP + // the chunk (counted) rather than allocate. + let (free_tx, free_rx) = sync_channel::>(RING_CHUNKS); + for _ in 0..RING_CHUNKS { + let _ = free_tx.try_send(Vec::with_capacity(CHUNK_CAP_SAMPLES)); } - cb_captured.fetch_add(num_frames as u64, Ordering::Relaxed); - AudioCallbackResult::Continue + let cb_captured = captured.clone(); + let cb_dropped = dropped.clone(); + let cb_free_tx = free_tx.clone(); // returns the buffer when the data channel is full + + let callback = move |_s: &AudioStream, data: *mut c_void, num_frames: i32| { + let n = num_frames as usize * CHANNELS; + // SAFETY: for an input stream AAudio provides `num_frames * channel_count` captured + // F32 samples at `data` (read-only for us). + let inp = unsafe { std::slice::from_raw_parts(data as *const f32, n) }; + cb_captured.fetch_add(num_frames as u64, Ordering::Relaxed); + match free_rx.try_recv() { + Ok(mut buf) => { + buf.clear(); + buf.extend_from_slice(inp); // retained capacity — no realloc past the first + match tx.try_send(buf) { + Ok(()) => {} + Err(TrySendError::Full(buf)) => { + // Encoder lagging: drop the chunk, hand the buffer straight back. + let _ = cb_free_tx.try_send(buf); + cb_dropped.fetch_add(1, Ordering::Relaxed); + } + Err(TrySendError::Disconnected(_)) => return AudioCallbackResult::Stop, + } + } + // Pool empty (every buffer in flight): drop, never allocate on this thread. + Err(_) => { + cb_dropped.fetch_add(1, Ordering::Relaxed); + } + } + AudioCallbackResult::Continue + }; + + let stream = AudioStreamBuilder::new()? + .direction(AudioDirection::Input) + .sample_rate(SAMPLE_RATE) + .channel_count(CHANNELS as i32) + .format(AudioFormat::PCM_Float) + .performance_mode(AudioPerformanceMode::LowLatency) + .sharing_mode(sharing) + .data_callback(Box::new(callback)) + .error_callback(Box::new(|_s, e| { + log::warn!("mic: AAudio error (device reroute/disconnect?): {e:?}"); + })) + .open_stream()?; + Ok((stream, rx, free_tx)) }; - let stream = AudioStreamBuilder::new() - .map_err(|e| log::error!("mic: AudioStreamBuilder::new: {e}")) - .ok()? - .direction(AudioDirection::Input) - .sample_rate(SAMPLE_RATE) - .channel_count(CHANNELS as i32) - .format(AudioFormat::PCM_Float) - .performance_mode(AudioPerformanceMode::LowLatency) - .sharing_mode(AudioSharingMode::Shared) - .data_callback(Box::new(callback)) - .error_callback(Box::new(|_s, e| { - log::warn!("mic: AAudio error (device reroute/disconnect?): {e:?}"); - })) - .open_stream() - .map_err(|e| log::error!("mic: open_stream (RECORD_AUDIO granted?): {e}")) - .ok()?; + // Exclusive first — MMAP-exclusive is AAudio's lowest-latency path — falling back to Shared + // when the device refuses (no MMAP, mic claimed, …). The started-log below prints the mode + // the device actually GRANTED (`share=`). + let (stream, rx, free_tx) = match try_open(AudioSharingMode::Exclusive) { + Ok(opened) => opened, + Err(e) => { + log::info!("mic: Exclusive open failed ({e}) — retrying Shared"); + match try_open(AudioSharingMode::Shared) { + Ok(opened) => opened, + Err(e) => { + log::error!("mic: open_stream (RECORD_AUDIO granted?): {e}"); + return None; + } + } + } + }; if let Err(e) = stream.request_start() { log::error!("mic: request_start: {e}"); return None; } log::info!( - "mic: AAudio input started rate={} ch={} fmt={:?}", + "mic: AAudio input started rate={} ch={} fmt={:?} share={:?}", stream.sample_rate(), stream.channel_count(), stream.format(), + stream.sharing_mode(), ); let shutdown = Arc::new(AtomicBool::new(false)); let sd = shutdown.clone(); let join = std::thread::Builder::new() .name("pf-mic".into()) - .spawn(move || encode_loop(client, rx, sd, captured)) + .spawn(move || encode_loop(client, rx, free_tx, sd, captured, dropped)) .ok(); Some(MicCapture { @@ -109,11 +169,15 @@ impl Drop for MicCapture { } /// Consumer: drain captured f32 → accumulate → Opus `encode_float` 20 ms stereo frames → `send_mic`. +/// Drained chunk buffers go back to the callback's free-list; the encode scratch is reused across +/// frames (only the packet Vec handed to `send_mic` is allocated per frame — it's sent away owned). fn encode_loop( client: Arc, rx: Receiver>, + free_tx: SyncSender>, shutdown: Arc, captured: Arc, + dropped: Arc, ) { let mut enc = match opus::Encoder::new( SAMPLE_RATE as u32, @@ -130,6 +194,7 @@ fn encode_loop( let frame = FRAME_SAMPLES * CHANNELS; let mut ring: VecDeque = VecDeque::with_capacity(frame * 4); + let mut pcm = vec![0f32; frame]; // reusable encode scratch (one 20 ms frame) let mut out = vec![0u8; 4000]; // max Opus packet for a 20 ms frame fits easily let mut seq: u32 = 0; let mut sent: u64 = 0; @@ -137,12 +202,19 @@ fn encode_loop( while !shutdown.load(Ordering::Relaxed) { match rx.recv_timeout(Duration::from_millis(100)) { - Ok(chunk) => ring.extend(chunk), + Ok(mut chunk) => { + // `drain(..)` keeps the Vec's capacity; hand the emptied buffer back to the + // callback's free-list (dropped only if the pool is momentarily full). + ring.extend(chunk.drain(..)); + let _ = free_tx.try_send(chunk); + } Err(RecvTimeoutError::Timeout) => continue, // wake to re-check shutdown Err(RecvTimeoutError::Disconnected) => break, } while ring.len() >= frame { - let pcm: Vec = ring.drain(..frame).collect(); + for (dst, src) in pcm.iter_mut().zip(ring.drain(..frame)) { + *dst = src; + } for &s in &pcm { peak = peak.max(s.abs()); } @@ -157,8 +229,9 @@ fn encode_loop( sent += 1; if sent % 250 == 0 { log::info!( - "mic: sent={sent} captured_frames={} peak={peak:.3}", + "mic: sent={sent} captured_frames={} dropped_chunks={} peak={peak:.3}", captured.load(Ordering::Relaxed), + dropped.load(Ordering::Relaxed), ); peak = 0.0; } @@ -168,7 +241,8 @@ fn encode_loop( } } log::info!( - "mic: stopped (sent={sent} captured_frames={})", + "mic: stopped (sent={sent} captured_frames={} dropped_chunks={})", captured.load(Ordering::Relaxed), + dropped.load(Ordering::Relaxed), ); } diff --git a/clients/android/native/src/session.rs b/clients/android/native/src/session.rs deleted file mode 100644 index 7de2abf..0000000 --- a/clients/android/native/src/session.rs +++ /dev/null @@ -1,762 +0,0 @@ -//! Session lifecycle + plane wiring over JNI. -//! -//! A connected session is a [`SessionHandle`] — an `Arc` plus the decode thread it -//! feeds — boxed and handed to Kotlin as an opaque `jlong`. The connector is `Sync`, so the decode -//! thread pulls the video plane (`next_frame`) directly while Kotlin still holds the handle. -//! -//! Wired: connect/close, the video plane (HEVC `next_frame` → NDK AMediaCodec → the SurfaceView's -//! `ANativeWindow`, see [`crate::decode`]), host→client audio ([`crate::audio`]), input -//! (`send_input` — mouse/keyboard/gamepad), rumble/DualSense HID feedback ([`crate::feedback`]), -//! and the trust surface: `nativeGenerateIdentity` (persistent identity, Keystore-wrapped on the -//! Kotlin side), `nativeConnect` with identity + pin (TOFU / pinned), and `nativePair` (SPAKE2 PIN). -//! -//! TODO(M4 Android stage 1): client→host DualSense rich input (`send_rich_input`), mode -//! renegotiation. Port the remaining orchestration from `clients/linux`. - -use jni::objects::{JObject, JString}; -use jni::sys::{jboolean, jdoubleArray, jint, jlong, jsize}; -use jni::JNIEnv; -use punktfunk_core::client::NativeClient; -use punktfunk_core::config::{CompositorPref, GamepadPref, Mode}; -use punktfunk_core::input::{InputEvent, InputKind}; -use std::panic::AssertUnwindSafe; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; -use std::thread::JoinHandle; -use std::time::Duration; - -/// Run a JNI body, catching any panic at the FFI boundary and returning `default` instead. -/// -/// A panic unwinding out of an `extern "system"` function aborts the whole process on Rust ≥ 1.81 — -/// a hard crash of the embedding Android app with no logcat trace. This mirrors the discipline the C -/// ABI already enforces (`punktfunk_core::abi` wraps every entry point in `catch_unwind`); the -/// `panic = "unwind"` profile in the workspace `Cargo.toml` exists precisely so these guards work. -/// We apply it to the teardown + background-thread shims (the "leaving a stream" path), where an -/// unexpected panic (e.g. a poisoned `Mutex` during concurrent teardown) must degrade to a logged -/// no-op rather than kill the app. -pub(crate) fn jni_guard(default: T, f: impl FnOnce() -> T) -> T { - std::panic::catch_unwind(AssertUnwindSafe(f)).unwrap_or_else(|_| { - log::error!("punktfunk JNI: caught a panic at the FFI boundary (returning default)"); - default - }) -} - -/// A live session behind the `jlong` handle: the connector + the decode thread it feeds. -pub(crate) struct SessionHandle { - // Read only by the android decode path (`nativeStartVideo` → `crate::decode`); on the host - // build (CI's workspace clippy/build) those readers are cfg'd out, so it's intentionally unused. - #[cfg_attr(not(target_os = "android"), allow(dead_code))] - pub client: Arc, - video: Mutex>, - #[cfg(target_os = "android")] - audio: Mutex>, - #[cfg(target_os = "android")] - mic: Mutex>, -} - -struct VideoThread { - shutdown: Arc, - join: Option>, - /// Live decode stats, written by the decode thread and drained ~1 Hz by `nativeVideoStats`. - stats: Arc, -} - -impl SessionHandle { - /// Signal the decode thread to stop and join it. Idempotent. - fn stop_video(&self) { - if let Some(mut vt) = self.video.lock().unwrap().take() { - vt.shutdown.store(true, Ordering::SeqCst); - if let Some(j) = vt.join.take() { - let _ = j.join(); - } - } - } - - /// Stop + close audio playback. Dropping the [`crate::audio::AudioPlayback`] joins its decode - /// thread and closes the AAudio stream. Idempotent. - #[cfg(target_os = "android")] - fn stop_audio(&self) { - let _ = self.audio.lock().unwrap().take(); - } - - /// Stop mic uplink. Dropping the [`crate::mic::MicCapture`] joins its encode thread and closes - /// the AAudio input stream. Idempotent. - #[cfg(target_os = "android")] - fn stop_mic(&self) { - let _ = self.mic.lock().unwrap().take(); - } -} - -impl Drop for SessionHandle { - fn drop(&mut self) { - self.stop_video(); - #[cfg(target_os = "android")] - self.stop_audio(); - #[cfg(target_os = "android")] - self.stop_mic(); - } -} - -/// SHA-256 fingerprint → 64 lowercase hex chars (matches the host log + client-rs). -fn hex32(fp: &[u8; 32]) -> String { - use std::fmt::Write; - fp.iter().fold(String::with_capacity(64), |mut s, b| { - let _ = write!(s, "{b:02x}"); - s - }) -} - -/// 64-hex → [u8; 32]; `None` on bad length/char. -fn parse_hex32(s: &str) -> Option<[u8; 32]> { - if s.len() != 64 { - return None; - } - let mut out = [0u8; 32]; - for (i, b) in out.iter_mut().enumerate() { - *b = u8::from_str_radix(&s[2 * i..2 * i + 2], 16).ok()?; - } - Some(out) -} - -/// `NativeBridge.nativeGenerateIdentity(): String` — mint a fresh persistent self-signed identity. -/// Returns `"\n-----PUNKTFUNK-KEY-----\n"`, or `""` on failure (logged). Kotlin -/// persists it (Keystore-wrapped) and only calls this again when the store is genuinely empty. -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeGenerateIdentity<'local>( - env: JNIEnv<'local>, - _this: JObject<'local>, -) -> jni::sys::jstring { - let out = match punktfunk_core::quic::endpoint::generate_identity() { - Ok((cert, key)) => format!("{cert}\n-----PUNKTFUNK-KEY-----\n{key}"), - Err(e) => { - log::error!("nativeGenerateIdentity failed: {e}"); - String::new() - } - }; - match env.new_string(out) { - Ok(s) => s.into_raw(), - Err(_) => JObject::null().into_raw(), - } -} - -/// `NativeBridge.nativeConnect(host, port, w, h, hz, certPem, keyPem, pinHex, bitrateKbps, -/// compositorPref, gamepadPref, hdrEnabled, audioChannels, timeoutMs): Long`. `certPem`/`keyPem` -/// empty = anonymous, else presented as the persistent identity. `pinHex` empty = TOFU (read -/// `nativeHostFingerprint` after), else 64-hex SHA-256 to pin the host (mismatch → 0). `bitrateKbps` -/// 0 = host default. `compositorPref`/`gamepadPref` are `CompositorPref`/`GamepadPref` wire bytes -/// (0 = Auto; unknown → Auto). `audioChannels` is the requested surround layout (2/6/8; normalized, -/// anything else → stereo) — the host clamps it and the resolved count drives playback. `timeoutMs` -/// is the handshake budget: the normal path passes a short value, the no-PIN "request access" path a -/// long one (≥ the host's approval-park window) so a slow operator approval lands on this same parked -/// connection rather than timing the client out first. Returns an opaque handle, or 0 on failure. -#[no_mangle] -#[allow(clippy::too_many_arguments)] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'local>( - mut env: JNIEnv<'local>, - _this: JObject<'local>, - host: JString<'local>, - port: jint, - width: jint, - height: jint, - refresh_hz: jint, - cert_pem: JString<'local>, - key_pem: JString<'local>, - pin_hex: JString<'local>, - bitrate_kbps: jint, - compositor_pref: jint, - gamepad_pref: jint, - hdr_enabled: jboolean, - audio_channels: jint, - preferred_codec: jint, - timeout_ms: jint, -) -> jlong { - let host: String = match env.get_string(&host) { - Ok(s) => s.into(), - Err(_) => return 0, - }; - let cert: String = env - .get_string(&cert_pem) - .map(Into::into) - .unwrap_or_default(); - let key: String = env.get_string(&key_pem).map(Into::into).unwrap_or_default(); - let pin_hex: String = env.get_string(&pin_hex).map(Into::into).unwrap_or_default(); - - let identity: Option<(String, String)> = if cert.is_empty() || key.is_empty() { - None - } else { - Some((cert, key)) - }; - let pin: Option<[u8; 32]> = if pin_hex.is_empty() { - None - } else { - match parse_hex32(&pin_hex) { - Some(fp) => Some(fp), - None => { - log::error!("nativeConnect: bad pin hex (len {})", pin_hex.len()); - return 0; - } - } - }; - let mode = Mode { - width: width as u32, - height: height as u32, - refresh_hz: refresh_hz as u32, - }; - match NativeClient::connect( - &host, - port as u16, - mode, - CompositorPref::from_u8(compositor_pref.clamp(0, u8::MAX as jint) as u8), - GamepadPref::from_u8(gamepad_pref.clamp(0, u8::MAX as jint) as u8), - bitrate_kbps.max(0) as u32, // 0 = host default - // Advertise 10-bit + HDR ONLY when this device's display can actually present it (Kotlin - // checks Display.getHdrCapabilities() and passes the result): the host (e.g. Windows) then - // upgrades to a Main10 / BT.2020 PQ encode. On an SDR display we advertise 0 so the host - // sends a proper 8-bit BT.709 stream rather than PQ the panel would mis-tone-map. AMediaCodec - // decodes Main10 from the SPS and the decode loop signals the Surface HDR dataspace + static - // metadata (see crate::decode). - if hdr_enabled != 0 { - punktfunk_core::quic::VIDEO_CAP_10BIT | punktfunk_core::quic::VIDEO_CAP_HDR - } else { - 0 - }, - // Requested surround layout (2 = stereo / 6 = 5.1 / 8 = 7.1). The host clamps to what it can - // capture and echoes the resolved count in `connector.audio_channels`, which drives the - // decoder + AAudio layout (read in `crate::audio::AudioPlayback::start`). Anything else - // normalizes to stereo here. - punktfunk_core::audio::normalize_channels(audio_channels.clamp(0, u8::MAX as jint) as u8), - // Codecs this device can decode — AMediaCodec decodes both HEVC and H.264 (AV1 isn't wired; - // hosts don't emit it on the native path yet). The host resolves the emitted codec from these - // + the soft `preferred_codec` and echoes it in `connector.codec`, which drives the mime below. - punktfunk_core::quic::CODEC_H264 | punktfunk_core::quic::CODEC_HEVC, - preferred_codec.clamp(0, u8::MAX as jint) as u8, - None, // launch: default app - pin, // Some → Crypto on host-fp mismatch - identity, // owned (cert, key) PEM, or None (anonymous) - // Handshake budget from Kotlin: ~10 s for a normal connect, ~185 s for "request access" - // (the host parks the connection until the operator approves the device — see ConnectScreen). - Duration::from_millis(timeout_ms.max(0) as u64), - ) { - Ok(client) => { - let handle = SessionHandle { - client: Arc::new(client), - video: Mutex::new(None), - #[cfg(target_os = "android")] - audio: Mutex::new(None), - #[cfg(target_os = "android")] - mic: Mutex::new(None), - }; - Box::into_raw(Box::new(handle)) as jlong - } - Err(e) => { - log::error!("nativeConnect to {host}:{port} failed: {e}"); - 0 - } - } -} - -/// `NativeBridge.nativeClose(handle)` — drop the session (stops the decode thread, then RAII-tears -/// down the connector). No-op on `0`. -/// -/// # Safety contract -/// `handle` must be `0` or a live handle from [`Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect`], -/// closed exactly once and not concurrently with other calls on the same handle (Kotlin owns this). -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeClose( - _env: JNIEnv, - _this: JObject, - handle: jlong, -) { - jni_guard((), || { - if handle != 0 { - // SAFETY: per the contract, `handle` is a live `Box` pointer. - unsafe { drop(Box::from_raw(handle as *mut SessionHandle)) }; - } - }) -} - -/// `NativeBridge.nativeHostFingerprint(handle): String` — the SHA-256 (64-hex) of the cert the host -/// presented on this connection. Valid after a successful `nativeConnect`; Kotlin pins it on a TOFU -/// connect. `""` on a `0` handle. -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeHostFingerprint<'local>( - env: JNIEnv<'local>, - _this: JObject<'local>, - handle: jlong, -) -> jni::sys::jstring { - let out = if handle == 0 { - String::new() - } else { - // SAFETY: live handle per the nativeConnect/nativeClose contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - hex32(&h.client.host_fingerprint) - }; - match env.new_string(out) { - Ok(s) => s.into_raw(), - Err(_) => JObject::null().into_raw(), - } -} - -/// `NativeBridge.nativePair(host, port, certPem, keyPem, pin, name): String` — run the SPAKE2 PIN -/// ceremony, presenting our persistent identity. On success returns the host's verified fingerprint -/// (64-hex) to persist + pin; on any failure (wrong PIN / MITM / host reject / unreachable) returns -/// `""` (logged). Blocking — Kotlin calls it off the UI thread. -#[no_mangle] -#[allow(clippy::too_many_arguments)] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativePair<'local>( - mut env: JNIEnv<'local>, - _this: JObject<'local>, - host: JString<'local>, - port: jint, - cert_pem: JString<'local>, - key_pem: JString<'local>, - pin: JString<'local>, - name: JString<'local>, -) -> jni::sys::jstring { - let g = |e: &mut JNIEnv<'local>, j: &JString<'local>| -> String { - e.get_string(j).map(Into::into).unwrap_or_default() - }; - let host = g(&mut env, &host); - let cert = g(&mut env, &cert_pem); - let key = g(&mut env, &key_pem); - let pin = g(&mut env, &pin); - let name = g(&mut env, &name); - - let out = if host.is_empty() || cert.is_empty() || key.is_empty() { - log::error!("nativePair: missing host/identity"); - String::new() - } else { - match NativeClient::pair( - &host, - port as u16, - (&cert, &key), // borrowed identity - &pin, - &name, - Duration::from_secs(60), - ) { - Ok(host_fp) => hex32(&host_fp), - Err(e) => { - // Crypto error == wrong PIN / MITM; anything else == transport/host reject. - log::error!("nativePair to {host}:{port} failed: {e}"); - String::new() - } - } - }; - match env.new_string(out) { - Ok(s) => s.into_raw(), - Err(_) => JObject::null().into_raw(), - } -} - -/// `NativeBridge.nativeStartVideo(handle, surface)` — wrap the SurfaceView's `Surface` as an -/// `ANativeWindow` and start the HEVC decode thread rendering onto it. No-op if already started. -#[cfg(target_os = "android")] -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartVideo( - env: JNIEnv, - _this: JObject, - handle: jlong, - surface: JObject, -) { - if handle == 0 { - return; - } - // SAFETY: live handle per the nativeConnect/nativeClose contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - let mut guard = h.video.lock().unwrap(); - if guard.is_some() { - return; // already streaming - } - // SAFETY: `env`/`surface` are valid JNI pointers for this call. `as *mut _` bridges any - // jni-sys version skew between the `jni` and `ndk` crates (both are raw `*mut _` pointers). - let window = match unsafe { - ndk::native_window::NativeWindow::from_surface( - env.get_native_interface() as *mut _, - surface.as_raw() as *mut _, - ) - } { - Some(w) => w, - None => { - log::error!("nativeStartVideo: no ANativeWindow from Surface"); - return; - } - }; - let shutdown = Arc::new(AtomicBool::new(false)); - let stats = Arc::new(crate::stats::VideoStats::new()); - let client = h.client.clone(); - let sd = shutdown.clone(); - let st = stats.clone(); - let join = std::thread::Builder::new() - .name("pf-decode".into()) - .spawn(move || crate::decode::run(client, window, sd, st)) - .ok(); - *guard = Some(VideoThread { - shutdown, - join, - stats, - }); -} - -/// `NativeBridge.nativeStopVideo(handle)` — stop + join the decode thread (without closing the -/// session). No-op on `0`. -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopVideo( - _env: JNIEnv, - _this: JObject, - handle: jlong, -) { - jni_guard((), || { - if handle != 0 { - // SAFETY: live handle per the contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - h.stop_video(); - } - }) -} - -/// `NativeBridge.nativeVideoStats(handle): DoubleArray?` — drain ~1 s of decode stats for the HUD. -/// Returns 14 doubles -/// `[fps, mbps, latP50Ms, latP95Ms, latValid, skewCorrected, width, height, refreshHz, framesDropped, -/// bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc]` -/// (the two flags are 1.0/0.0; the trailing four describe the negotiated video feed — see below), or -/// `null` when no decode thread is running. Poll ~1 Hz from the UI; each call resets the measurement -/// window. Not android-gated — pure `jni` + connector reads, so it links on the host build too -/// (Kotlin only ever calls it on device). -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats( - env: JNIEnv, - _this: JObject, - handle: jlong, -) -> jdoubleArray { - jni_guard(std::ptr::null_mut(), || { - if handle == 0 { - return std::ptr::null_mut(); - } - // SAFETY: live handle per the nativeConnect/nativeClose contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - let snap = match h.video.lock().unwrap().as_ref() { - Some(vt) => vt.stats.drain(), - None => return std::ptr::null_mut(), // not streaming → no stats - }; - let mode = h.client.mode(); - let color = h.client.color; - let buf: [f64; 14] = [ - snap.fps, - snap.mbps, - snap.lat_p50_ms, - snap.lat_p95_ms, - if snap.lat_valid { 1.0 } else { 0.0 }, - if snap.skew_corrected { 1.0 } else { 0.0 }, - mode.width as f64, - mode.height as f64, - mode.refresh_hz as f64, - h.client.frames_dropped() as f64, - // Video-feed properties the host resolved at the handshake (Welcome): encode bit depth - // (8 / 10), the CICP colour primaries + transfer code points (Kotlin maps these to a - // colour-space / HDR label — transfer 16 = PQ, 18 = HLG ⇒ HDR), and the HEVC - // chroma_format_idc (1 = 4:2:0, 3 = 4:4:4). Static for the session unless renegotiated. - h.client.bit_depth as f64, - color.primaries as f64, - color.transfer as f64, - h.client.chroma_format as f64, - ]; - let arr = match env.new_double_array(buf.len() as jsize) { - Ok(a) => a, - Err(_) => return std::ptr::null_mut(), - }; - if env.set_double_array_region(&arr, 0, &buf).is_err() { - return std::ptr::null_mut(); - } - arr.into_raw() - }) -} - -/// `NativeBridge.nativeStartAudio(handle)` — start the Opus→AAudio playback thread. No-op if already -/// started or on a `0` handle. Best-effort: a failure leaves video streaming. -#[cfg(target_os = "android")] -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartAudio( - _env: JNIEnv, - _this: JObject, - handle: jlong, -) { - if handle == 0 { - return; - } - // SAFETY: live handle per the nativeConnect/nativeClose contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - let mut guard = h.audio.lock().unwrap(); - if guard.is_some() { - return; // already playing - } - match crate::audio::AudioPlayback::start(h.client.clone()) { - Some(p) => *guard = Some(p), - None => log::error!("nativeStartAudio: playback init failed (video unaffected)"), - } -} - -/// `NativeBridge.nativeStopAudio(handle)` — stop + join the audio thread and close AAudio (without -/// closing the session). No-op on `0`. -#[cfg(target_os = "android")] -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopAudio( - _env: JNIEnv, - _this: JObject, - handle: jlong, -) { - jni_guard((), || { - if handle != 0 { - // SAFETY: live handle per the contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - h.stop_audio(); - } - }) -} - -/// `NativeBridge.nativeStartMic(handle)` — start mic capture (AAudio input → Opus → host `send_mic`). -/// No-op if already running or on a `0` handle. Caller MUST hold RECORD_AUDIO; a failure (e.g. no -/// permission) leaves the rest of the session streaming. -#[cfg(target_os = "android")] -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartMic( - _env: JNIEnv, - _this: JObject, - handle: jlong, -) { - if handle == 0 { - return; - } - // SAFETY: live handle per the nativeConnect/nativeClose contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - let mut guard = h.mic.lock().unwrap(); - if guard.is_some() { - return; // already capturing - } - match crate::mic::MicCapture::start(h.client.clone()) { - Some(m) => *guard = Some(m), - None => log::error!("nativeStartMic: mic init failed (RECORD_AUDIO? — session unaffected)"), - } -} - -/// `NativeBridge.nativeStopMic(handle)` — stop + join the mic thread and close the AAudio input -/// stream (without closing the session). No-op on `0`. -#[cfg(target_os = "android")] -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopMic( - _env: JNIEnv, - _this: JObject, - handle: jlong, -) { - jni_guard((), || { - if handle != 0 { - // SAFETY: live handle per the contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - h.stop_mic(); - } - }) -} - -// ---- Input plane: Kotlin capture → NativeClient::send_input ---------------------------------- -// All four are `&self` on the `Sync` connector (send_input is a non-blocking datagram push), safe -// from the Kotlin UI thread. NOT android-gated — send_input exists on the host build too, so these -// compile everywhere (parity with nativeConnect/nativeClose). The wire codes are the GameStream -// conventions: buttons 1=left/2=middle/3=right/4=X1/5=X2; scroll axis 0=vertical/1=horizontal, -// signed 120-unit delta, +=up/right; keys are Windows VK (mapped from KEYCODE_* on the Kotlin side). - -/// `NativeBridge.nativeSendPointerMove(handle, dx, dy)` — relative mouse motion (screen +y down). -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendPointerMove( - _env: JNIEnv, - _this: JObject, - handle: jlong, - dx: jint, - dy: jint, -) { - if handle == 0 { - return; - } - // SAFETY: live handle per the nativeConnect/nativeClose contract; send_input is &self. - let h = unsafe { &*(handle as *const SessionHandle) }; - let _ = h.client.send_input(&InputEvent { - kind: InputKind::MouseMove, - _pad: [0; 3], - code: 0, - x: dx, - y: dy, - flags: 0, - }); -} - -/// `NativeBridge.nativeSendPointerAbs(handle, x, y, surfaceWidth, surfaceHeight)` — absolute cursor -/// position: the host moves the pointer to `x`/`y` in a `surfaceWidth`×`surfaceHeight` pixel space, -/// normalizing against the size packed into `flags` as `(w << 16) | h` and mapping into the output -/// region (it drops the event if that size is zero). This is the touch "direct pointing" path — the -/// cursor jumps to the finger — and matches the Apple client's absolute touch forwarding. -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendPointerAbs( - _env: JNIEnv, - _this: JObject, - handle: jlong, - x: jint, - y: jint, - surface_width: jint, - surface_height: jint, -) { - if handle == 0 { - return; - } - // SAFETY: live handle per the contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - let w = (surface_width.max(0) as u32) & 0xffff; - let ht = (surface_height.max(0) as u32) & 0xffff; - let _ = h.client.send_input(&InputEvent { - kind: InputKind::MouseMoveAbs, - _pad: [0; 3], - code: 0, - x, - y, - flags: (w << 16) | ht, - }); -} - -/// `NativeBridge.nativeSendPointerButton(handle, button, down)` — one button transition. -/// `button`: GameStream id (1=left, 2=middle, 3=right, 4=X1, 5=X2). `down`: 1=press, 0=release. -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendPointerButton( - _env: JNIEnv, - _this: JObject, - handle: jlong, - button: jint, - down: jboolean, -) { - if handle == 0 { - return; - } - // SAFETY: live handle per the contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - let _ = h.client.send_input(&InputEvent { - kind: if down != 0 { - InputKind::MouseButtonDown - } else { - InputKind::MouseButtonUp - }, - _pad: [0; 3], - code: button as u32, - x: 0, - y: 0, - flags: 0, - }); -} - -/// `NativeBridge.nativeSendScroll(handle, axis, delta)` — one scroll step. `axis`: 0=vertical, -/// 1=horizontal. `delta`: signed, WHEEL_DELTA(120)-scaled, +=up/right. -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendScroll( - _env: JNIEnv, - _this: JObject, - handle: jlong, - axis: jint, - delta: jint, -) { - if handle == 0 { - return; - } - // SAFETY: live handle per the contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - let _ = h.client.send_input(&InputEvent { - kind: InputKind::MouseScroll, - _pad: [0; 3], - code: axis as u32, - x: delta, - y: 0, - flags: 0, - }); -} - -/// `NativeBridge.nativeSendKey(handle, vk, down, mods)` — one key transition. `vk`: Windows -/// Virtual-Key code (0 = unmapped → dropped). `down`: 1=press, 0=release. `mods`: VK modifier -/// bitmask (0 for now — the host folds modifiers from the L/R modifier key events themselves). -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendKey( - _env: JNIEnv, - _this: JObject, - handle: jlong, - vk: jint, - down: jboolean, - mods: jint, -) { - if handle == 0 || vk == 0 { - return; - } - // SAFETY: live handle per the contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - let _ = h.client.send_input(&InputEvent { - kind: if down != 0 { - InputKind::KeyDown - } else { - InputKind::KeyUp - }, - _pad: [0; 3], - code: vk as u32, - x: 0, - y: 0, - flags: mods as u32, - }); -} - -// ---- Gamepad: Kotlin captures (KeyEvent/MotionEvent) → NativeClient::send_input --------------- -// Single-pad model: exactly one controller, forwarded as pad 0 (flags = 0). Buttons carry the -// gamepad::BTN_* bit in `code` and pressed/released in `x` (1/0); axes carry the gamepad::AXIS_* id -// in `code` and the value in `x` (sticks i16 −32768..32767, +y = up; triggers 0..255). The host -// accumulates the incremental events into its virtual xpad. Wire contract: input.rs::gamepad. - -/// `NativeBridge.nativeSendGamepadButton(handle, bit, down)` — one gamepad button transition. -/// `bit`: a `gamepad::BTN_*` bit (e.g. BTN_A = 0x1000). `down`: 1=press, 0=release. -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendGamepadButton( - _env: JNIEnv, - _this: JObject, - handle: jlong, - bit: jint, - down: jboolean, -) { - if handle == 0 { - return; - } - // SAFETY: live handle per the contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - let _ = h.client.send_input(&InputEvent { - kind: InputKind::GamepadButton, - _pad: [0; 3], - code: bit as u32, - x: i32::from(down != 0), - y: 0, - flags: 0, // pad index 0 — single-pad model - }); -} - -/// `NativeBridge.nativeSendGamepadAxis(handle, axisId, value)` — one gamepad axis update. -/// `axisId`: a `gamepad::AXIS_*` id (LS_X=0..RT=5). `value`: stick i16 (−32768..32767, +y=up) or -/// trigger 0..255. -#[no_mangle] -pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendGamepadAxis( - _env: JNIEnv, - _this: JObject, - handle: jlong, - axis_id: jint, - value: jint, -) { - if handle == 0 { - return; - } - // SAFETY: live handle per the contract. - let h = unsafe { &*(handle as *const SessionHandle) }; - let _ = h.client.send_input(&InputEvent { - kind: InputKind::GamepadAxis, - _pad: [0; 3], - code: axis_id as u32, - x: value, - y: 0, - flags: 0, // pad index 0 — single-pad model - }); -} diff --git a/clients/android/native/src/session/connect.rs b/clients/android/native/src/session/connect.rs new file mode 100644 index 0000000..f406ffa --- /dev/null +++ b/clients/android/native/src/session/connect.rs @@ -0,0 +1,244 @@ +//! Connect lifecycle + the trust surface: identity mint, connect (TOFU / pinned), close, +//! host-fingerprint read, and the SPAKE2 PIN pairing ceremony. + +use jni::objects::{JObject, JString}; +use jni::sys::{jboolean, jint, jlong}; +use jni::JNIEnv; +use punktfunk_core::client::NativeClient; +use punktfunk_core::config::{CompositorPref, GamepadPref, Mode}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use super::{hex32, jni_guard, parse_hex32, SessionHandle}; + +/// `NativeBridge.nativeGenerateIdentity(): String` — mint a fresh persistent self-signed identity. +/// Returns `"\n-----PUNKTFUNK-KEY-----\n"`, or `""` on failure (logged). Kotlin +/// persists it (Keystore-wrapped) and only calls this again when the store is genuinely empty. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeGenerateIdentity<'local>( + env: JNIEnv<'local>, + _this: JObject<'local>, +) -> jni::sys::jstring { + let out = match punktfunk_core::quic::endpoint::generate_identity() { + Ok((cert, key)) => format!("{cert}\n-----PUNKTFUNK-KEY-----\n{key}"), + Err(e) => { + log::error!("nativeGenerateIdentity failed: {e}"); + String::new() + } + }; + match env.new_string(out) { + Ok(s) => s.into_raw(), + Err(_) => JObject::null().into_raw(), + } +} + +/// `NativeBridge.nativeConnect(host, port, w, h, hz, certPem, keyPem, pinHex, bitrateKbps, +/// compositorPref, gamepadPref, hdrEnabled, audioChannels, preferredCodec, timeoutMs): Long`. +/// `certPem`/`keyPem` empty = anonymous, else presented as the persistent identity. `pinHex` empty +/// = TOFU (read `nativeHostFingerprint` after), else 64-hex SHA-256 to pin the host (mismatch → 0). +/// `bitrateKbps` 0 = host default. `compositorPref`/`gamepadPref` are `CompositorPref`/`GamepadPref` +/// wire bytes (0 = Auto; unknown → Auto). `audioChannels` is the requested surround layout (2/6/8; +/// normalized, anything else → stereo) — the host clamps it and the resolved count drives playback. +/// `preferredCodec` is the soft codec preference wire byte (0 = Auto). `timeoutMs` is the handshake +/// budget: the normal path passes a short value, the no-PIN "request access" path a long one (≥ the +/// host's approval-park window) so a slow operator approval lands on this same parked connection +/// rather than timing the client out first. Returns an opaque handle, or 0 on failure. +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'local>( + mut env: JNIEnv<'local>, + _this: JObject<'local>, + host: JString<'local>, + port: jint, + width: jint, + height: jint, + refresh_hz: jint, + cert_pem: JString<'local>, + key_pem: JString<'local>, + pin_hex: JString<'local>, + bitrate_kbps: jint, + compositor_pref: jint, + gamepad_pref: jint, + hdr_enabled: jboolean, + audio_channels: jint, + preferred_codec: jint, + timeout_ms: jint, +) -> jlong { + let host: String = match env.get_string(&host) { + Ok(s) => s.into(), + Err(_) => return 0, + }; + let cert: String = env + .get_string(&cert_pem) + .map(Into::into) + .unwrap_or_default(); + let key: String = env.get_string(&key_pem).map(Into::into).unwrap_or_default(); + let pin_hex: String = env.get_string(&pin_hex).map(Into::into).unwrap_or_default(); + + let identity: Option<(String, String)> = if cert.is_empty() || key.is_empty() { + None + } else { + Some((cert, key)) + }; + let pin: Option<[u8; 32]> = if pin_hex.is_empty() { + None + } else { + match parse_hex32(&pin_hex) { + Some(fp) => Some(fp), + None => { + log::error!("nativeConnect: bad pin hex (len {})", pin_hex.len()); + return 0; + } + } + }; + let mode = Mode { + width: width as u32, + height: height as u32, + refresh_hz: refresh_hz as u32, + }; + match NativeClient::connect( + &host, + port as u16, + mode, + CompositorPref::from_u8(compositor_pref.clamp(0, u8::MAX as jint) as u8), + GamepadPref::from_u8(gamepad_pref.clamp(0, u8::MAX as jint) as u8), + bitrate_kbps.max(0) as u32, // 0 = host default + // Advertise 10-bit + HDR ONLY when this device's display can actually present it (Kotlin + // checks Display.getHdrCapabilities() and passes the result): the host (e.g. Windows) then + // upgrades to a Main10 / BT.2020 PQ encode. On an SDR display we advertise 0 so the host + // sends a proper 8-bit BT.709 stream rather than PQ the panel would mis-tone-map. AMediaCodec + // decodes Main10 from the SPS and the decode loop signals the Surface HDR dataspace + static + // metadata (see crate::decode). + if hdr_enabled != 0 { + punktfunk_core::quic::VIDEO_CAP_10BIT | punktfunk_core::quic::VIDEO_CAP_HDR + } else { + 0 + }, + // Requested surround layout (2 = stereo / 6 = 5.1 / 8 = 7.1). The host clamps to what it can + // capture and echoes the resolved count in `connector.audio_channels`, which drives the + // decoder + AAudio layout (read in `crate::audio::AudioPlayback::start`). Anything else + // normalizes to stereo here. + punktfunk_core::audio::normalize_channels(audio_channels.clamp(0, u8::MAX as jint) as u8), + // Codecs this device can decode — AMediaCodec decodes both HEVC and H.264 (AV1 isn't wired; + // hosts don't emit it on the native path yet). The host resolves the emitted codec from these + // + the soft `preferred_codec` and echoes it in `connector.codec`, which drives the mime below. + punktfunk_core::quic::CODEC_H264 | punktfunk_core::quic::CODEC_HEVC, + preferred_codec.clamp(0, u8::MAX as jint) as u8, + None, // launch: default app + pin, // Some → Crypto on host-fp mismatch + identity, // owned (cert, key) PEM, or None (anonymous) + // Handshake budget from Kotlin: ~10 s for a normal connect, ~185 s for "request access" + // (the host parks the connection until the operator approves the device — see ConnectScreen). + Duration::from_millis(timeout_ms.max(0) as u64), + ) { + Ok(client) => { + let handle = SessionHandle { + client: Arc::new(client), + stats: Arc::new(crate::stats::VideoStats::new()), + video: Mutex::new(None), + #[cfg(target_os = "android")] + audio: Mutex::new(None), + #[cfg(target_os = "android")] + mic: Mutex::new(None), + }; + Box::into_raw(Box::new(handle)) as jlong + } + Err(e) => { + log::error!("nativeConnect to {host}:{port} failed: {e}"); + 0 + } + } +} + +/// `NativeBridge.nativeClose(handle)` — drop the session (stops the decode thread, then RAII-tears +/// down the connector). No-op on `0`. +/// +/// # Safety contract +/// `handle` must be `0` or a live handle from [`Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect`], +/// closed exactly once and not concurrently with other calls on the same handle (Kotlin owns this). +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeClose( + _env: JNIEnv, + _this: JObject, + handle: jlong, +) { + jni_guard((), || { + if handle != 0 { + // SAFETY: per the contract, `handle` is a live `Box` pointer. + unsafe { drop(Box::from_raw(handle as *mut SessionHandle)) }; + } + }) +} + +/// `NativeBridge.nativeHostFingerprint(handle): String` — the SHA-256 (64-hex) of the cert the host +/// presented on this connection. Valid after a successful `nativeConnect`; Kotlin pins it on a TOFU +/// connect. `""` on a `0` handle. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeHostFingerprint<'local>( + env: JNIEnv<'local>, + _this: JObject<'local>, + handle: jlong, +) -> jni::sys::jstring { + let out = if handle == 0 { + String::new() + } else { + // SAFETY: live handle per the nativeConnect/nativeClose contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + hex32(&h.client.host_fingerprint) + }; + match env.new_string(out) { + Ok(s) => s.into_raw(), + Err(_) => JObject::null().into_raw(), + } +} + +/// `NativeBridge.nativePair(host, port, certPem, keyPem, pin, name): String` — run the SPAKE2 PIN +/// ceremony, presenting our persistent identity. On success returns the host's verified fingerprint +/// (64-hex) to persist + pin; on any failure (wrong PIN / MITM / host reject / unreachable) returns +/// `""` (logged). Blocking — Kotlin calls it off the UI thread. +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativePair<'local>( + mut env: JNIEnv<'local>, + _this: JObject<'local>, + host: JString<'local>, + port: jint, + cert_pem: JString<'local>, + key_pem: JString<'local>, + pin: JString<'local>, + name: JString<'local>, +) -> jni::sys::jstring { + let g = |e: &mut JNIEnv<'local>, j: &JString<'local>| -> String { + e.get_string(j).map(Into::into).unwrap_or_default() + }; + let host = g(&mut env, &host); + let cert = g(&mut env, &cert_pem); + let key = g(&mut env, &key_pem); + let pin = g(&mut env, &pin); + let name = g(&mut env, &name); + + let out = if host.is_empty() || cert.is_empty() || key.is_empty() { + log::error!("nativePair: missing host/identity"); + String::new() + } else { + match NativeClient::pair( + &host, + port as u16, + (&cert, &key), // borrowed identity + &pin, + &name, + Duration::from_secs(60), + ) { + Ok(host_fp) => hex32(&host_fp), + Err(e) => { + // Crypto error == wrong PIN / MITM; anything else == transport/host reject. + log::error!("nativePair to {host}:{port} failed: {e}"); + String::new() + } + } + }; + match env.new_string(out) { + Ok(s) => s.into_raw(), + Err(_) => JObject::null().into_raw(), + } +} diff --git a/clients/android/native/src/session/input.rs b/clients/android/native/src/session/input.rs new file mode 100644 index 0000000..b5be196 --- /dev/null +++ b/clients/android/native/src/session/input.rs @@ -0,0 +1,159 @@ +//! Input plane: Kotlin capture → `NativeClient::send_input`. +//! +//! All shims are `&self` on the `Sync` connector (send_input is a non-blocking datagram push), safe +//! from the Kotlin UI thread. NOT android-gated — send_input exists on the host build too, so these +//! compile everywhere (parity with nativeConnect/nativeClose). The wire codes are the GameStream +//! conventions: buttons 1=left/2=middle/3=right/4=X1/5=X2; scroll axis 0=vertical/1=horizontal, +//! signed 120-unit delta, +=up/right; keys are Windows VK (mapped from KEYCODE_* on the Kotlin side). + +use jni::objects::JObject; +use jni::sys::{jboolean, jint, jlong}; +use jni::JNIEnv; +use punktfunk_core::input::{InputEvent, InputKind}; + +use super::SessionHandle; + +/// Shared shim body: guard against a `0` handle, deref, and push one [`InputEvent`]. +fn send_event(handle: jlong, kind: InputKind, code: u32, x: i32, y: i32, flags: u32) { + if handle == 0 { + return; + } + // SAFETY: live handle per the nativeConnect/nativeClose contract; send_input is &self. + let h = unsafe { &*(handle as *const SessionHandle) }; + let _ = h.client.send_input(&InputEvent { + kind, + _pad: [0; 3], + code, + x, + y, + flags, + }); +} + +/// `NativeBridge.nativeSendPointerMove(handle, dx, dy)` — relative mouse motion (screen +y down). +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendPointerMove( + _env: JNIEnv, + _this: JObject, + handle: jlong, + dx: jint, + dy: jint, +) { + send_event(handle, InputKind::MouseMove, 0, dx, dy, 0); +} + +/// `NativeBridge.nativeSendPointerAbs(handle, x, y, surfaceWidth, surfaceHeight)` — absolute cursor +/// position: the host moves the pointer to `x`/`y` in a `surfaceWidth`×`surfaceHeight` pixel space, +/// normalizing against the size packed into `flags` as `(w << 16) | h` and mapping into the output +/// region (it drops the event if that size is zero). This is the touch "direct pointing" path — the +/// cursor jumps to the finger — and matches the Apple client's absolute touch forwarding. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendPointerAbs( + _env: JNIEnv, + _this: JObject, + handle: jlong, + x: jint, + y: jint, + surface_width: jint, + surface_height: jint, +) { + let w = (surface_width.max(0) as u32) & 0xffff; + let ht = (surface_height.max(0) as u32) & 0xffff; + send_event(handle, InputKind::MouseMoveAbs, 0, x, y, (w << 16) | ht); +} + +/// `NativeBridge.nativeSendPointerButton(handle, button, down)` — one button transition. +/// `button`: GameStream id (1=left, 2=middle, 3=right, 4=X1, 5=X2). `down`: 1=press, 0=release. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendPointerButton( + _env: JNIEnv, + _this: JObject, + handle: jlong, + button: jint, + down: jboolean, +) { + let kind = if down != 0 { + InputKind::MouseButtonDown + } else { + InputKind::MouseButtonUp + }; + send_event(handle, kind, button as u32, 0, 0, 0); +} + +/// `NativeBridge.nativeSendScroll(handle, axis, delta)` — one scroll step. `axis`: 0=vertical, +/// 1=horizontal. `delta`: signed, WHEEL_DELTA(120)-scaled, +=up/right. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendScroll( + _env: JNIEnv, + _this: JObject, + handle: jlong, + axis: jint, + delta: jint, +) { + send_event(handle, InputKind::MouseScroll, axis as u32, delta, 0, 0); +} + +/// `NativeBridge.nativeSendKey(handle, vk, down, mods)` — one key transition. `vk`: Windows +/// Virtual-Key code (0 = unmapped → dropped). `down`: 1=press, 0=release. `mods`: VK modifier +/// bitmask (0 for now — the host folds modifiers from the L/R modifier key events themselves). +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendKey( + _env: JNIEnv, + _this: JObject, + handle: jlong, + vk: jint, + down: jboolean, + mods: jint, +) { + if vk == 0 { + return; + } + let kind = if down != 0 { + InputKind::KeyDown + } else { + InputKind::KeyUp + }; + send_event(handle, kind, vk as u32, 0, 0, mods as u32); +} + +// ---- Gamepad: Kotlin captures (KeyEvent/MotionEvent) → NativeClient::send_input --------------- +// Single-pad model: exactly one controller, forwarded as pad 0 (flags = 0). Buttons carry the +// gamepad::BTN_* bit in `code` and pressed/released in `x` (1/0); axes carry the gamepad::AXIS_* id +// in `code` and the value in `x` (sticks i16 −32768..32767, +y = up; triggers 0..255). The host +// accumulates the incremental events into its virtual xpad. Wire contract: input.rs::gamepad. + +/// `NativeBridge.nativeSendGamepadButton(handle, bit, down)` — one gamepad button transition. +/// `bit`: a `gamepad::BTN_*` bit (e.g. BTN_A = 0x1000). `down`: 1=press, 0=release. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendGamepadButton( + _env: JNIEnv, + _this: JObject, + handle: jlong, + bit: jint, + down: jboolean, +) { + // flags = 0: pad index 0 — single-pad model. + send_event( + handle, + InputKind::GamepadButton, + bit as u32, + i32::from(down != 0), + 0, + 0, + ); +} + +/// `NativeBridge.nativeSendGamepadAxis(handle, axisId, value)` — one gamepad axis update. +/// `axisId`: a `gamepad::AXIS_*` id (LS_X=0..RT=5). `value`: stick i16 (−32768..32767, +y=up) or +/// trigger 0..255. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendGamepadAxis( + _env: JNIEnv, + _this: JObject, + handle: jlong, + axis_id: jint, + value: jint, +) { + // flags = 0: pad index 0 — single-pad model. + send_event(handle, InputKind::GamepadAxis, axis_id as u32, value, 0, 0); +} diff --git a/clients/android/native/src/session/mod.rs b/clients/android/native/src/session/mod.rs new file mode 100644 index 0000000..8ca7cc5 --- /dev/null +++ b/clients/android/native/src/session/mod.rs @@ -0,0 +1,124 @@ +//! Session lifecycle + plane wiring over JNI. +//! +//! A connected session is a [`SessionHandle`] — an `Arc` plus the decode thread it +//! feeds — boxed and handed to Kotlin as an opaque `jlong`. The connector is `Sync`, so the decode +//! thread pulls the video plane (`next_frame`) directly while Kotlin still holds the handle. +//! +//! Wired: connect/close, the video plane (HEVC `next_frame` → NDK AMediaCodec → the SurfaceView's +//! `ANativeWindow`, see [`crate::decode`]), host→client audio ([`crate::audio`]), input +//! (`send_input` — mouse/keyboard/gamepad), rumble/DualSense HID feedback ([`crate::feedback`]), +//! and the trust surface: `nativeGenerateIdentity` (persistent identity, Keystore-wrapped on the +//! Kotlin side), `nativeConnect` with identity + pin (TOFU / pinned), and `nativePair` (SPAKE2 PIN). +//! +//! Split by concern: [`connect`] (identity + connect/close + the trust surface), [`planes`] +//! (video/audio/mic start/stop + the stats drain), [`input`] (the input-plane shims). This module +//! keeps the shared infrastructure they all deref through. +//! +//! TODO(M4 Android stage 1): client→host DualSense rich input (`send_rich_input`), mode +//! renegotiation. Port the remaining orchestration from `clients/linux`. + +mod connect; +mod input; +mod planes; + +use punktfunk_core::client::NativeClient; +use std::panic::AssertUnwindSafe; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; + +/// Run a JNI body, catching any panic at the FFI boundary and returning `default` instead. +/// +/// A panic unwinding out of an `extern "system"` function aborts the whole process on Rust ≥ 1.81 — +/// a hard crash of the embedding Android app with no logcat trace. This mirrors the discipline the C +/// ABI already enforces (`punktfunk_core::abi` wraps every entry point in `catch_unwind`); the +/// `panic = "unwind"` profile in the workspace `Cargo.toml` exists precisely so these guards work. +/// We apply it to the teardown + background-thread shims (the "leaving a stream" path), where an +/// unexpected panic (e.g. a poisoned `Mutex` during concurrent teardown) must degrade to a logged +/// no-op rather than kill the app. +pub(crate) fn jni_guard(default: T, f: impl FnOnce() -> T) -> T { + std::panic::catch_unwind(AssertUnwindSafe(f)).unwrap_or_else(|_| { + log::error!("punktfunk JNI: caught a panic at the FFI boundary (returning default)"); + default + }) +} + +/// A live session behind the `jlong` handle: the connector + the decode thread it feeds. +pub(crate) struct SessionHandle { + // Read only by the android decode path (`nativeStartVideo` → `crate::decode`); on the host + // build (CI's workspace clippy/build) those readers are cfg'd out, so it's intentionally unused. + #[cfg_attr(not(target_os = "android"), allow(dead_code))] + pub client: Arc, + /// Live decode stats, written by the decode thread and drained ~1 Hz by `nativeVideoStats`. + /// Session-lifetime (not per `VideoThread`) so the HUD's enable gate set via + /// `nativeSetVideoStatsEnabled` survives surface teardown/recreate and can land before + /// `nativeStartVideo` — enabling resets the window, so no stale data leaks across restarts. + pub stats: Arc, + video: Mutex>, + #[cfg(target_os = "android")] + audio: Mutex>, + #[cfg(target_os = "android")] + mic: Mutex>, +} + +struct VideoThread { + shutdown: Arc, + join: Option>, +} + +impl SessionHandle { + /// Signal the decode thread to stop and join it. Idempotent. + fn stop_video(&self) { + if let Some(mut vt) = self.video.lock().unwrap().take() { + vt.shutdown.store(true, Ordering::SeqCst); + if let Some(j) = vt.join.take() { + let _ = j.join(); + } + } + } + + /// Stop + close audio playback. Dropping the [`crate::audio::AudioPlayback`] joins its decode + /// thread and closes the AAudio stream. Idempotent. + #[cfg(target_os = "android")] + fn stop_audio(&self) { + let _ = self.audio.lock().unwrap().take(); + } + + /// Stop mic uplink. Dropping the [`crate::mic::MicCapture`] joins its encode thread and closes + /// the AAudio input stream. Idempotent. + #[cfg(target_os = "android")] + fn stop_mic(&self) { + let _ = self.mic.lock().unwrap().take(); + } +} + +impl Drop for SessionHandle { + fn drop(&mut self) { + self.stop_video(); + #[cfg(target_os = "android")] + self.stop_audio(); + #[cfg(target_os = "android")] + self.stop_mic(); + } +} + +/// SHA-256 fingerprint → 64 lowercase hex chars (matches the host log + client-rs). +fn hex32(fp: &[u8; 32]) -> String { + use std::fmt::Write; + fp.iter().fold(String::with_capacity(64), |mut s, b| { + let _ = write!(s, "{b:02x}"); + s + }) +} + +/// 64-hex → [u8; 32]; `None` on bad length/char. +fn parse_hex32(s: &str) -> Option<[u8; 32]> { + if s.len() != 64 { + return None; + } + let mut out = [0u8; 32]; + for (i, b) in out.iter_mut().enumerate() { + *b = u8::from_str_radix(&s[2 * i..2 * i + 2], 16).ok()?; + } + Some(out) +} diff --git a/clients/android/native/src/session/planes.rs b/clients/android/native/src/session/planes.rs new file mode 100644 index 0000000..3c63ace --- /dev/null +++ b/clients/android/native/src/session/planes.rs @@ -0,0 +1,236 @@ +//! Plane start/stop: video (HEVC decode → Surface), host→client audio, mic uplink — plus the +//! ~1 Hz decode-stats drain for the HUD. + +use jni::objects::JObject; +use jni::sys::{jboolean, jdoubleArray, jlong, jsize}; +use jni::JNIEnv; + +use super::{jni_guard, SessionHandle}; + +/// `NativeBridge.nativeStartVideo(handle, surface)` — wrap the SurfaceView's `Surface` as an +/// `ANativeWindow` and start the HEVC decode thread rendering onto it. No-op if already started. +#[cfg(target_os = "android")] +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartVideo( + env: JNIEnv, + _this: JObject, + handle: jlong, + surface: JObject, +) { + use super::VideoThread; + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + + if handle == 0 { + return; + } + // SAFETY: live handle per the nativeConnect/nativeClose contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + let mut guard = h.video.lock().unwrap(); + if guard.is_some() { + return; // already streaming + } + // SAFETY: `env`/`surface` are valid JNI pointers for this call. `as *mut _` bridges any + // jni-sys version skew between the `jni` and `ndk` crates (both are raw `*mut _` pointers). + let window = match unsafe { + ndk::native_window::NativeWindow::from_surface( + env.get_native_interface() as *mut _, + surface.as_raw() as *mut _, + ) + } { + Some(w) => w, + None => { + log::error!("nativeStartVideo: no ANativeWindow from Surface"); + return; + } + }; + let shutdown = Arc::new(AtomicBool::new(false)); + let client = h.client.clone(); + let sd = shutdown.clone(); + let st = h.stats.clone(); // session-lifetime stats (gate survives surface recreate) + let join = std::thread::Builder::new() + .name("pf-decode".into()) + .spawn(move || crate::decode::run(client, window, sd, st)) + .ok(); + *guard = Some(VideoThread { shutdown, join }); +} + +/// `NativeBridge.nativeStopVideo(handle)` — stop + join the decode thread (without closing the +/// session). No-op on `0`. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopVideo( + _env: JNIEnv, + _this: JObject, + handle: jlong, +) { + jni_guard((), || { + if handle != 0 { + // SAFETY: live handle per the contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + h.stop_video(); + } + }) +} + +/// `NativeBridge.nativeVideoStats(handle): DoubleArray?` — drain ~1 s of decode stats for the HUD. +/// Returns 14 doubles +/// `[fps, mbps, latP50Ms, latP95Ms, latValid, skewCorrected, width, height, refreshHz, framesDropped, +/// bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc]` +/// (the two flags are 1.0/0.0; the trailing four describe the negotiated video feed — see below), or +/// `null` when no decode thread is running. Poll ~1 Hz from the UI; each call resets the measurement +/// window. Not android-gated — pure `jni` + connector reads, so it links on the host build too +/// (Kotlin only ever calls it on device). +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats( + env: JNIEnv, + _this: JObject, + handle: jlong, +) -> jdoubleArray { + jni_guard(std::ptr::null_mut(), || { + if handle == 0 { + return std::ptr::null_mut(); + } + // SAFETY: live handle per the nativeConnect/nativeClose contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + if h.video.lock().unwrap().is_none() { + return std::ptr::null_mut(); // not streaming → no stats + } + let snap = h.stats.drain(); + let mode = h.client.mode(); + let color = h.client.color; + let buf: [f64; 14] = [ + snap.fps, + snap.mbps, + snap.lat_p50_ms, + snap.lat_p95_ms, + if snap.lat_valid { 1.0 } else { 0.0 }, + if snap.skew_corrected { 1.0 } else { 0.0 }, + mode.width as f64, + mode.height as f64, + mode.refresh_hz as f64, + h.client.frames_dropped() as f64, + // Video-feed properties the host resolved at the handshake (Welcome): encode bit depth + // (8 / 10), the CICP colour primaries + transfer code points (Kotlin maps these to a + // colour-space / HDR label — transfer 16 = PQ, 18 = HLG ⇒ HDR), and the HEVC + // chroma_format_idc (1 = 4:2:0, 3 = 4:4:4). Static for the session unless renegotiated. + h.client.bit_depth as f64, + color.primaries as f64, + color.transfer as f64, + h.client.chroma_format as f64, + ]; + let arr = match env.new_double_array(buf.len() as jsize) { + Ok(a) => a, + Err(_) => return std::ptr::null_mut(), + }; + if env.set_double_array_region(&arr, 0, &buf).is_err() { + return std::ptr::null_mut(); + } + arr.into_raw() + }) +} + +/// `NativeBridge.nativeSetVideoStatsEnabled(handle, enabled)` — gate per-frame stats sampling on the +/// HUD actually being visible: while disabled the decode thread skips the clock read + lock per AU. +/// Enabling resets the measurement window so a later show never reports stale data. Sticky for the +/// session (survives video stop/start across surface recreation). No-op on `0`. Not android-gated — +/// pure `jni` + an atomic store, so it links on the host build too. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSetVideoStatsEnabled( + _env: JNIEnv, + _this: JObject, + handle: jlong, + enabled: jboolean, +) { + jni_guard((), || { + if handle != 0 { + // SAFETY: live handle per the nativeConnect/nativeClose contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + h.stats.set_enabled(enabled != 0); + } + }) +} + +/// `NativeBridge.nativeStartAudio(handle)` — start the Opus→AAudio playback thread. No-op if already +/// started or on a `0` handle. Best-effort: a failure leaves video streaming. +#[cfg(target_os = "android")] +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartAudio( + _env: JNIEnv, + _this: JObject, + handle: jlong, +) { + if handle == 0 { + return; + } + // SAFETY: live handle per the nativeConnect/nativeClose contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + let mut guard = h.audio.lock().unwrap(); + if guard.is_some() { + return; // already playing + } + match crate::audio::AudioPlayback::start(h.client.clone()) { + Some(p) => *guard = Some(p), + None => log::error!("nativeStartAudio: playback init failed (video unaffected)"), + } +} + +/// `NativeBridge.nativeStopAudio(handle)` — stop + join the audio thread and close AAudio (without +/// closing the session). No-op on `0`. +#[cfg(target_os = "android")] +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopAudio( + _env: JNIEnv, + _this: JObject, + handle: jlong, +) { + jni_guard((), || { + if handle != 0 { + // SAFETY: live handle per the contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + h.stop_audio(); + } + }) +} + +/// `NativeBridge.nativeStartMic(handle)` — start mic capture (AAudio input → Opus → host `send_mic`). +/// No-op if already running or on a `0` handle. Caller MUST hold RECORD_AUDIO; a failure (e.g. no +/// permission) leaves the rest of the session streaming. +#[cfg(target_os = "android")] +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartMic( + _env: JNIEnv, + _this: JObject, + handle: jlong, +) { + if handle == 0 { + return; + } + // SAFETY: live handle per the nativeConnect/nativeClose contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + let mut guard = h.mic.lock().unwrap(); + if guard.is_some() { + return; // already capturing + } + match crate::mic::MicCapture::start(h.client.clone()) { + Some(m) => *guard = Some(m), + None => log::error!("nativeStartMic: mic init failed (RECORD_AUDIO? — session unaffected)"), + } +} + +/// `NativeBridge.nativeStopMic(handle)` — stop + join the mic thread and close the AAudio input +/// stream (without closing the session). No-op on `0`. +#[cfg(target_os = "android")] +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopMic( + _env: JNIEnv, + _this: JObject, + handle: jlong, +) { + jni_guard((), || { + if handle != 0 { + // SAFETY: live handle per the contract. + let h = unsafe { &*(handle as *const SessionHandle) }; + h.stop_mic(); + } + }) +} diff --git a/clients/android/native/src/stats.rs b/clients/android/native/src/stats.rs index 07dd8e4..770dce0 100644 --- a/clients/android/native/src/stats.rs +++ b/clients/android/native/src/stats.rs @@ -1,15 +1,22 @@ //! Live decode stats for the on-stream HUD (mirrors the Apple client's stats overlay): FPS, //! receive throughput, and capture→client-receipt latency (p50/p95). The decode thread is the sole //! writer (`note` per access unit); the JNI accessor `nativeVideoStats` drains a snapshot ~1 Hz and -//! resets the window. Pure `std` so it compiles on the host build too (the decode thread is -//! android-only, but `VideoThread` holds the shared handle unconditionally). +//! resets the window. Sampling is gated on the HUD actually being visible (`set_enabled`, driven by +//! `nativeSetVideoStatsEnabled`) so the hidden steady state costs one relaxed atomic load per frame. +//! Pure `std` so it compiles on the host build too (the decode thread is android-only, but +//! `SessionHandle` holds the shared handle unconditionally). +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Mutex; use std::time::Instant; /// Rolling per-window accumulator. Rates are computed over the actual elapsed wall-time at drain /// (robust to poll jitter), so a poll that lands at 0.9 s or 1.1 s still reports the right FPS. pub struct VideoStats { + /// HUD gate: `note` runs on the per-frame decode path, so while the overlay is hidden it (and + /// the caller's latency computation — see `enabled`) early-outs on this flag alone. Off until + /// Kotlin shows the HUD. + enabled: AtomicBool, inner: Mutex, } @@ -35,11 +42,9 @@ pub struct Snapshot { } impl VideoStats { - // `new`/`note` are driven only by the android-only decode thread; `drain` (the JNI accessor) is - // ungated, so on the host build these two are unreferenced — that's expected, not dead code. - #[cfg_attr(not(target_os = "android"), allow(dead_code))] pub fn new() -> VideoStats { VideoStats { + enabled: AtomicBool::new(false), inner: Mutex::new(Inner { window_start: Instant::now(), frames: 0, @@ -50,10 +55,44 @@ impl VideoStats { } } + /// Whether the HUD wants samples. The decode thread checks this BEFORE building a latency + /// sample, so the per-frame wall-clock read is skipped too while hidden. + // Read only by the android-only decode thread; unreferenced on the host build — expected. + #[cfg_attr(not(target_os = "android"), allow(dead_code))] + pub fn enabled(&self) -> bool { + self.enabled.load(Ordering::Relaxed) + } + + /// Toggle sampling. Enabling resets the window, so the first HUD poll after a show never mixes + /// in counters (or a window start) from before the overlay was visible. + pub fn set_enabled(&self, on: bool) { + let was = self.enabled.swap(on, Ordering::Relaxed); + if on && !was { + let mut g = self + .inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + g.window_start = Instant::now(); + g.frames = 0; + g.bytes = 0; + g.lat_us.clear(); + } + } + /// Record one decoded access unit: its wire size and (if in range) its capture→client latency. + // Driven only by the android-only decode thread; unreferenced on the host build — expected. #[cfg_attr(not(target_os = "android"), allow(dead_code))] pub fn note(&self, bytes: usize, lat_us: Option, skew_corrected: bool) { - let mut g = self.inner.lock().unwrap(); + if !self.enabled.load(Ordering::Relaxed) { + return; // HUD hidden — skip the lock (the caller already skipped the clock read) + } + // Poison-proof: `note` runs per-frame on the decode thread, which has no catch_unwind — + // a panic elsewhere must not turn every later lock into a second panic (the counters + // stay consistent regardless). + let mut g = self + .inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); g.frames += 1; g.bytes += bytes as u64; g.skew_corrected = skew_corrected; @@ -64,7 +103,11 @@ impl VideoStats { /// Compute the window's rates + latency percentiles, then reset for the next window. pub fn drain(&self) -> Snapshot { - let mut g = self.inner.lock().unwrap(); + // Poison-proof for the same reason as `note` — a poisoned window still drains fine. + let mut g = self + .inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let elapsed = g.window_start.elapsed().as_secs_f64().max(1e-3); let fps = g.frames as f64 / elapsed; let mbps = g.bytes as f64 * 8.0 / 1_000_000.0 / elapsed;