feat(android): mic uplink + connect-screen redesign
ci / web (push) Successful in 29s
android / android (push) Successful in 1m50s
ci / bench (push) Successful in 1m42s
apple / swift (push) Successful in 53s
ci / rust (push) Successful in 1m4s
ci / docs-site (push) Successful in 31s
decky / build-publish (push) Successful in 13s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 5s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 4s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 3s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 4s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 4s
deb / build-publish (push) Successful in 3m21s
rpm / build-publish (bazzite, punktfunk-fedora-rpm) (push) Successful in 5m15s
docker / deploy-docs (push) Successful in 17s
rpm / build-publish (fedora-44, punktfunk-fedora44-rpm) (push) Successful in 5m1s
ci / web (push) Successful in 29s
android / android (push) Successful in 1m50s
ci / bench (push) Successful in 1m42s
apple / swift (push) Successful in 53s
ci / rust (push) Successful in 1m4s
ci / docs-site (push) Successful in 31s
decky / build-publish (push) Successful in 13s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 5s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 4s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 3s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 4s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 4s
deb / build-publish (push) Successful in 3m21s
rpm / build-publish (bazzite, punktfunk-fedora-rpm) (push) Successful in 5m15s
docker / deploy-docs (push) Successful in 17s
rpm / build-publish (fedora-44, punktfunk-fedora44-rpm) (push) Successful in 5m1s
Microphone uplink (client → host's virtual mic, 0xCB) and a cleaner connect screen.
Mic (Rust-heavy, mirrors the audio playback path in reverse):
- crates/punktfunk-android/src/mic.rs: AAudio LowLatency **input** → realtime callback hands
captured f32 to a channel → a worker thread Opus-encodes 20 ms stereo frames (48 kHz, VOIP,
64 kbps) and calls NativeClient::send_mic. MicCapture owns the stream + encode thread (RAII stop).
- session.rs: SessionHandle gains a `mic` slot; nativeStartMic/nativeStopMic JNI (mirror of audio);
stopped in Drop. NativeBridge: the two externs.
- Settings: a `micEnabled` flag + a Microphone toggle in SettingsScreen that requests RECORD_AUDIO
(denied → stays off). StreamScreen starts the mic only if enabled AND the permission is held.
Connect-screen redesign:
- One scrollable Column (was a fixed centered layout that could clip with the new tab bar);
host rows render via forEach (no nested LazyColumn). Colored section labels ("Saved hosts",
"Discovered on the network", "Connect manually"), full-width host cards / fields / Connect button,
a header + subtitle, and a muted footer.
Verified live (emulator pf_phone -> home-worker-2): toggling mic requests RECORD_AUDIO; with it
granted, a session sends mic frames (client "mic: sent=250 … peak=0.439" — real audio) and the host
logs "client datagram stream ended … mic=276". Redesigned screen confirmed via screenshots.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
package io.unom.punktfunk
|
||||
|
||||
import android.Manifest
|
||||
import android.content.pm.PackageManager
|
||||
import android.os.Build
|
||||
import android.os.Bundle
|
||||
import android.view.InputDevice
|
||||
@@ -15,26 +17,23 @@ import androidx.activity.enableEdgeToEdge
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.gestures.awaitEachGesture
|
||||
import androidx.compose.foundation.gestures.awaitFirstDown
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.heightIn
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.lazy.LazyColumn
|
||||
import androidx.compose.foundation.lazy.items
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.foundation.text.KeyboardOptions
|
||||
import androidx.compose.foundation.verticalScroll
|
||||
import androidx.compose.material3.AlertDialog
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.Card
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.filled.Home
|
||||
import androidx.compose.material.icons.filled.Settings
|
||||
import androidx.compose.material3.HorizontalDivider
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.NavigationBar
|
||||
@@ -62,6 +61,7 @@ import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.text.input.KeyboardType
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.viewinterop.AndroidView
|
||||
import androidx.core.content.ContextCompat
|
||||
import io.unom.punktfunk.kit.Gamepad
|
||||
import io.unom.punktfunk.kit.GamepadFeedback
|
||||
import io.unom.punktfunk.kit.Keymap
|
||||
@@ -178,7 +178,7 @@ private fun App() {
|
||||
|
||||
if (streamHandle != 0L) {
|
||||
// Immersive: the stream takes the whole screen, no bottom bar.
|
||||
StreamScreen(streamHandle, onDisconnect = { streamHandle = 0L })
|
||||
StreamScreen(streamHandle, micEnabled = settings.micEnabled, onDisconnect = { streamHandle = 0L })
|
||||
} else {
|
||||
Scaffold(
|
||||
bottomBar = {
|
||||
@@ -309,19 +309,23 @@ private fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) {
|
||||
}
|
||||
|
||||
Column(
|
||||
modifier = Modifier.fillMaxSize().padding(24.dp),
|
||||
modifier = Modifier
|
||||
.fillMaxSize()
|
||||
.verticalScroll(rememberScrollState())
|
||||
.padding(horizontal = 24.dp, vertical = 16.dp),
|
||||
horizontalAlignment = Alignment.CenterHorizontally,
|
||||
verticalArrangement = Arrangement.Center,
|
||||
) {
|
||||
Text("punktfunk", style = MaterialTheme.typography.headlineMedium)
|
||||
Text("Android client", style = MaterialTheme.typography.bodyMedium)
|
||||
Spacer(Modifier.height(24.dp))
|
||||
Text("punktfunk", style = MaterialTheme.typography.headlineLarge)
|
||||
Text(
|
||||
"stream a remote desktop",
|
||||
style = MaterialTheme.typography.bodyMedium,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
Spacer(Modifier.height(28.dp))
|
||||
|
||||
if (savedHosts.isNotEmpty()) {
|
||||
Text("Saved hosts", style = MaterialTheme.typography.labelLarge)
|
||||
Spacer(Modifier.height(8.dp))
|
||||
LazyColumn(modifier = Modifier.fillMaxWidth().heightIn(max = 220.dp)) {
|
||||
items(savedHosts, key = { "${it.address}:${it.port}" }) { kh ->
|
||||
SectionLabel("Saved hosts")
|
||||
savedHosts.forEach { kh ->
|
||||
SavedHostRow(
|
||||
kh,
|
||||
enabled = !connecting,
|
||||
@@ -336,34 +340,28 @@ private fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) {
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
Spacer(Modifier.height(16.dp))
|
||||
HorizontalDivider()
|
||||
Spacer(Modifier.height(16.dp))
|
||||
Spacer(Modifier.height(20.dp))
|
||||
}
|
||||
|
||||
if (discovered.isNotEmpty()) {
|
||||
Text("Discovered hosts", style = MaterialTheme.typography.labelLarge)
|
||||
Spacer(Modifier.height(8.dp))
|
||||
LazyColumn(modifier = Modifier.fillMaxWidth().heightIn(max = 220.dp)) {
|
||||
items(discovered, key = { it.key }) { dh ->
|
||||
SectionLabel("Discovered on the network")
|
||||
discovered.forEach { dh ->
|
||||
DiscoveredHostRow(dh, enabled = !connecting) {
|
||||
host = dh.host
|
||||
port = dh.port.toString()
|
||||
connect(dh.host, dh.port, dh)
|
||||
}
|
||||
}
|
||||
}
|
||||
Spacer(Modifier.height(16.dp))
|
||||
HorizontalDivider()
|
||||
Spacer(Modifier.height(16.dp))
|
||||
Spacer(Modifier.height(20.dp))
|
||||
}
|
||||
|
||||
SectionLabel("Connect manually")
|
||||
OutlinedTextField(
|
||||
value = host,
|
||||
onValueChange = { host = it },
|
||||
label = { Text("Host") },
|
||||
singleLine = true,
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
)
|
||||
Spacer(Modifier.height(8.dp))
|
||||
OutlinedTextField(
|
||||
@@ -372,18 +370,28 @@ private fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) {
|
||||
label = { Text("Port") },
|
||||
singleLine = true,
|
||||
keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number),
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
)
|
||||
Spacer(Modifier.height(16.dp))
|
||||
Button(
|
||||
enabled = !connecting && host.isNotBlank() && port.isNotBlank(),
|
||||
onClick = { connect(host.trim(), port.toInt()) },
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
) { Text(if (connecting) "Connecting…" else "Connect ($w×$h@$hz)") }
|
||||
status?.let {
|
||||
Spacer(Modifier.height(12.dp))
|
||||
Text(it, style = MaterialTheme.typography.bodySmall)
|
||||
Text(
|
||||
it,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.error,
|
||||
)
|
||||
}
|
||||
Spacer(Modifier.height(24.dp))
|
||||
Text("core ABI v$abi", style = MaterialTheme.typography.labelSmall)
|
||||
Spacer(Modifier.height(28.dp))
|
||||
Text(
|
||||
"core ABI v$abi",
|
||||
style = MaterialTheme.typography.labelSmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
|
||||
pendingTrust?.let { pt ->
|
||||
@@ -499,6 +507,17 @@ private fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Left-aligned section header above each block of the connect screen. */
|
||||
@Composable
|
||||
private fun SectionLabel(text: String) {
|
||||
Text(
|
||||
text,
|
||||
style = MaterialTheme.typography.titleSmall,
|
||||
color = MaterialTheme.colorScheme.primary,
|
||||
modifier = Modifier.fillMaxWidth().padding(bottom = 8.dp),
|
||||
)
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun DiscoveredHostRow(dh: DiscoveredHost, enabled: Boolean, onTap: () -> Unit) {
|
||||
Card(
|
||||
@@ -547,10 +566,15 @@ private fun SavedHostRow(
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun StreamScreen(handle: Long, onDisconnect: () -> Unit) {
|
||||
private fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) {
|
||||
val context = LocalContext.current
|
||||
val activity = context as? MainActivity
|
||||
val window = activity?.window
|
||||
// Start mic only if the user enabled it AND granted RECORD_AUDIO (else the AAudio input fails).
|
||||
val micWanted = micEnabled && ContextCompat.checkSelfPermission(
|
||||
context,
|
||||
Manifest.permission.RECORD_AUDIO,
|
||||
) == PackageManager.PERMISSION_GRANTED
|
||||
|
||||
DisposableEffect(handle) {
|
||||
window?.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
|
||||
@@ -564,7 +588,8 @@ private fun StreamScreen(handle: Long, onDisconnect: () -> Unit) {
|
||||
activity?.axisMapper = null
|
||||
activity?.streamHandle = 0L
|
||||
window?.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
|
||||
// Leaving the stream: stop the audio + decode threads and tear down the session.
|
||||
// Leaving the stream: stop the mic + audio + decode threads and tear down the session.
|
||||
NativeBridge.nativeStopMic(handle)
|
||||
NativeBridge.nativeStopAudio(handle)
|
||||
NativeBridge.nativeStopVideo(handle)
|
||||
NativeBridge.nativeClose(handle)
|
||||
@@ -582,11 +607,13 @@ private fun StreamScreen(handle: Long, onDisconnect: () -> Unit) {
|
||||
override fun surfaceCreated(holder: SurfaceHolder) {
|
||||
NativeBridge.nativeStartVideo(handle, holder.surface)
|
||||
NativeBridge.nativeStartAudio(handle)
|
||||
if (micWanted) NativeBridge.nativeStartMic(handle)
|
||||
}
|
||||
|
||||
override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {}
|
||||
|
||||
override fun surfaceDestroyed(holder: SurfaceHolder) {
|
||||
NativeBridge.nativeStopMic(handle)
|
||||
NativeBridge.nativeStopAudio(handle)
|
||||
NativeBridge.nativeStopVideo(handle)
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ data class Settings(
|
||||
val bitrateKbps: Int = 0,
|
||||
val compositor: Int = 0,
|
||||
val gamepad: Int = 0,
|
||||
val micEnabled: Boolean = false,
|
||||
)
|
||||
|
||||
/** Loads/saves [Settings] in the app-private `punktfunk_settings` prefs. */
|
||||
@@ -29,6 +30,7 @@ class SettingsStore(context: Context) {
|
||||
bitrateKbps = prefs.getInt(K_BITRATE, 0),
|
||||
compositor = prefs.getInt(K_COMPOSITOR, 0),
|
||||
gamepad = prefs.getInt(K_GAMEPAD, 0),
|
||||
micEnabled = prefs.getBoolean(K_MIC, false),
|
||||
)
|
||||
|
||||
fun save(s: Settings) {
|
||||
@@ -39,6 +41,7 @@ class SettingsStore(context: Context) {
|
||||
.putInt(K_BITRATE, s.bitrateKbps)
|
||||
.putInt(K_COMPOSITOR, s.compositor)
|
||||
.putInt(K_GAMEPAD, s.gamepad)
|
||||
.putBoolean(K_MIC, s.micEnabled)
|
||||
.apply()
|
||||
}
|
||||
|
||||
@@ -49,6 +52,7 @@ class SettingsStore(context: Context) {
|
||||
const val K_BITRATE = "bitrate_kbps"
|
||||
const val K_COMPOSITOR = "compositor"
|
||||
const val K_GAMEPAD = "gamepad"
|
||||
const val K_MIC = "mic_enabled"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
package io.unom.punktfunk
|
||||
|
||||
import android.Manifest
|
||||
import android.content.pm.PackageManager
|
||||
import androidx.activity.compose.BackHandler
|
||||
import androidx.activity.compose.rememberLauncherForActivityResult
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
@@ -17,6 +22,7 @@ import androidx.compose.material3.ExposedDropdownMenuAnchorType
|
||||
import androidx.compose.material3.ExposedDropdownMenuDefaults
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.OutlinedTextField
|
||||
import androidx.compose.material3.Switch
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.material3.TextButton
|
||||
import androidx.compose.runtime.Composable
|
||||
@@ -24,9 +30,11 @@ import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.core.content.ContextCompat
|
||||
|
||||
/**
|
||||
* Stream settings. Edits are persisted immediately via [onChange]; [onBack] returns to the connect
|
||||
@@ -82,6 +90,31 @@ fun SettingsScreen(initial: Settings, onChange: (Settings) -> Unit, onBack: () -
|
||||
selected = s.gamepad,
|
||||
) { g -> update(s.copy(gamepad = g)) }
|
||||
|
||||
// Mic uplink — turning it on requests RECORD_AUDIO; if denied, the toggle stays off.
|
||||
val micLauncher = rememberLauncherForActivityResult(
|
||||
ActivityResultContracts.RequestPermission(),
|
||||
) { granted -> update(s.copy(micEnabled = granted)) }
|
||||
Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically) {
|
||||
Column(Modifier.weight(1f)) {
|
||||
Text("Microphone", style = MaterialTheme.typography.bodyLarge)
|
||||
Text(
|
||||
"Send your mic to the host's virtual microphone",
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
)
|
||||
}
|
||||
Switch(
|
||||
checked = s.micEnabled,
|
||||
onCheckedChange = { on ->
|
||||
when {
|
||||
!on -> update(s.copy(micEnabled = false))
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
|
||||
PackageManager.PERMISSION_GRANTED -> update(s.copy(micEnabled = true))
|
||||
else -> micLauncher.launch(Manifest.permission.RECORD_AUDIO)
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Spacer(Modifier.height(8.dp))
|
||||
TextButton(onClick = onBack) { Text("Done") }
|
||||
}
|
||||
|
||||
@@ -84,6 +84,16 @@ object NativeBridge {
|
||||
/** Stop + join the audio thread and close AAudio, without closing the session. No-op on `0`. */
|
||||
external fun nativeStopAudio(handle: Long)
|
||||
|
||||
/**
|
||||
* Start mic uplink: AAudio input → Opus (48 kHz stereo, 20 ms) → host (`send_mic` / 0xCB), all in
|
||||
* Rust. No-op if already running. The caller MUST hold RECORD_AUDIO; otherwise the AAudio input
|
||||
* stream fails to open and the rest of the session keeps streaming.
|
||||
*/
|
||||
external fun nativeStartMic(handle: Long)
|
||||
|
||||
/** Stop + join the mic thread and close the AAudio input stream. No-op on `0`. */
|
||||
external fun nativeStopMic(handle: Long)
|
||||
|
||||
// ---- Input: Kotlin captures, Rust forwards to the host (send_input) ----
|
||||
|
||||
/** Relative mouse move; dx/dy are device-pixel deltas (screen +y down). */
|
||||
|
||||
@@ -26,6 +26,8 @@ mod audio;
|
||||
#[cfg(target_os = "android")]
|
||||
mod decode;
|
||||
mod feedback;
|
||||
#[cfg(target_os = "android")]
|
||||
mod mic;
|
||||
mod session;
|
||||
|
||||
/// Initialize `android_logger` once when the JVM loads the library. Logs land in logcat under the
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
//! Android microphone uplink (android-only): capture mic PCM via AAudio (LowLatency **input**),
|
||||
//! Opus-encode 20 ms stereo frames, and push them to the host over the connector's mic plane
|
||||
//! (`send_mic` → 0xCB datagram). The mirror of [`crate::audio`] in reverse: AAudio's realtime input
|
||||
//! callback hands captured interleaved f32 to a channel; a worker thread we own does the Opus encode
|
||||
//! + send (encoding is too heavy for the realtime callback, exactly as decode is on the playback
|
||||
//! side). Format matches the host decoder + the Linux client: 48 kHz **stereo**, 20 ms, Opus VOIP.
|
||||
|
||||
use ndk::audio::{
|
||||
AudioCallbackResult, AudioDirection, AudioFormat, AudioPerformanceMode, AudioSharingMode,
|
||||
AudioStream, AudioStreamBuilder,
|
||||
};
|
||||
use punktfunk_core::client::NativeClient;
|
||||
use std::collections::VecDeque;
|
||||
use std::ffi::c_void;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::mpsc::{sync_channel, Receiver, RecvTimeoutError, TrySendError};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
const CHANNELS: usize = 2;
|
||||
const SAMPLE_RATE: i32 = 48_000;
|
||||
/// 20 ms per channel @ 48 kHz — the Linux client's frame; the host accepts ≤ 120 ms.
|
||||
const FRAME_SAMPLES: usize = 960;
|
||||
/// Captured-chunk hand-off depth (each ~ one burst); drops on overflow (best-effort uplink).
|
||||
const RING_CHUNKS: usize = 64;
|
||||
/// Opus VOIP target bitrate (speech; tunable).
|
||||
const MIC_BITRATE: i32 = 64_000;
|
||||
|
||||
/// Owned by [`crate::session::SessionHandle`]: the live AAudio input stream + the encode thread.
|
||||
pub struct MicCapture {
|
||||
_stream: AudioStream, // dropping it stops + closes the AAudio input stream
|
||||
shutdown: Arc<AtomicBool>,
|
||||
join: Option<std::thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl MicCapture {
|
||||
/// Open AAudio (LowLatency, 48 kHz/stereo/f32) for **input** with a realtime callback that
|
||||
/// forwards captured PCM to a channel, then spawn the Opus encode + uplink thread. `None` on
|
||||
/// failure (the caller leaves the rest of the session streaming).
|
||||
pub fn start(client: Arc<NativeClient>) -> Option<MicCapture> {
|
||||
let (tx, rx) = sync_channel::<Vec<f32>>(RING_CHUNKS);
|
||||
let captured = Arc::new(AtomicU64::new(0));
|
||||
let cb_captured = captured.clone();
|
||||
|
||||
let callback = move |_s: &AudioStream, data: *mut c_void, num_frames: i32| {
|
||||
let n = num_frames as usize * CHANNELS;
|
||||
// SAFETY: for an input stream AAudio provides `num_frames * channel_count` captured F32
|
||||
// samples at `data` (read-only for us).
|
||||
let inp = unsafe { std::slice::from_raw_parts(data as *const f32, n) };
|
||||
match tx.try_send(inp.to_vec()) {
|
||||
Ok(()) | Err(TrySendError::Full(_)) => {} // drop-newest if the encoder lags
|
||||
Err(TrySendError::Disconnected(_)) => return AudioCallbackResult::Stop,
|
||||
}
|
||||
cb_captured.fetch_add(num_frames as u64, Ordering::Relaxed);
|
||||
AudioCallbackResult::Continue
|
||||
};
|
||||
|
||||
let stream = AudioStreamBuilder::new()
|
||||
.map_err(|e| log::error!("mic: AudioStreamBuilder::new: {e}"))
|
||||
.ok()?
|
||||
.direction(AudioDirection::Input)
|
||||
.sample_rate(SAMPLE_RATE)
|
||||
.channel_count(CHANNELS as i32)
|
||||
.format(AudioFormat::PCM_Float)
|
||||
.performance_mode(AudioPerformanceMode::LowLatency)
|
||||
.sharing_mode(AudioSharingMode::Shared)
|
||||
.data_callback(Box::new(callback))
|
||||
.error_callback(Box::new(|_s, e| {
|
||||
log::warn!("mic: AAudio error (device reroute/disconnect?): {e:?}");
|
||||
}))
|
||||
.open_stream()
|
||||
.map_err(|e| log::error!("mic: open_stream (RECORD_AUDIO granted?): {e}"))
|
||||
.ok()?;
|
||||
|
||||
if let Err(e) = stream.request_start() {
|
||||
log::error!("mic: request_start: {e}");
|
||||
return None;
|
||||
}
|
||||
log::info!(
|
||||
"mic: AAudio input started rate={} ch={} fmt={:?}",
|
||||
stream.sample_rate(),
|
||||
stream.channel_count(),
|
||||
stream.format(),
|
||||
);
|
||||
|
||||
let shutdown = Arc::new(AtomicBool::new(false));
|
||||
let sd = shutdown.clone();
|
||||
let join = std::thread::Builder::new()
|
||||
.name("pf-mic".into())
|
||||
.spawn(move || encode_loop(client, rx, sd, captured))
|
||||
.ok();
|
||||
|
||||
Some(MicCapture {
|
||||
_stream: stream,
|
||||
shutdown,
|
||||
join,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MicCapture {
|
||||
fn drop(&mut self) {
|
||||
self.shutdown.store(true, Ordering::SeqCst);
|
||||
if let Some(j) = self.join.take() {
|
||||
let _ = j.join();
|
||||
}
|
||||
// `_stream` drops here → AAudio request_stop + close.
|
||||
}
|
||||
}
|
||||
|
||||
/// Consumer: drain captured f32 → accumulate → Opus `encode_float` 20 ms stereo frames → `send_mic`.
|
||||
fn encode_loop(
|
||||
client: Arc<NativeClient>,
|
||||
rx: Receiver<Vec<f32>>,
|
||||
shutdown: Arc<AtomicBool>,
|
||||
captured: Arc<AtomicU64>,
|
||||
) {
|
||||
let mut enc = match opus::Encoder::new(
|
||||
SAMPLE_RATE as u32,
|
||||
opus::Channels::Stereo,
|
||||
opus::Application::Voip,
|
||||
) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
log::error!("mic: opus encoder init: {e} — mic disabled");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let _ = enc.set_bitrate(opus::Bitrate::Bits(MIC_BITRATE));
|
||||
|
||||
let frame = FRAME_SAMPLES * CHANNELS;
|
||||
let mut ring: VecDeque<f32> = VecDeque::with_capacity(frame * 4);
|
||||
let mut out = vec![0u8; 4000]; // max Opus packet for a 20 ms frame fits easily
|
||||
let mut seq: u32 = 0;
|
||||
let mut sent: u64 = 0;
|
||||
let mut peak = 0f32; // loudest |sample| since the last log — tells speech from silence
|
||||
|
||||
while !shutdown.load(Ordering::Relaxed) {
|
||||
match rx.recv_timeout(Duration::from_millis(100)) {
|
||||
Ok(chunk) => ring.extend(chunk),
|
||||
Err(RecvTimeoutError::Timeout) => continue, // wake to re-check shutdown
|
||||
Err(RecvTimeoutError::Disconnected) => break,
|
||||
}
|
||||
while ring.len() >= frame {
|
||||
let pcm: Vec<f32> = ring.drain(..frame).collect();
|
||||
for &s in &pcm {
|
||||
peak = peak.max(s.abs());
|
||||
}
|
||||
match enc.encode_float(&pcm, &mut out) {
|
||||
Ok(len) => {
|
||||
let pts = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_nanos() as u64)
|
||||
.unwrap_or(0);
|
||||
let _ = client.send_mic(seq, pts, out[..len].to_vec());
|
||||
seq = seq.wrapping_add(1);
|
||||
sent += 1;
|
||||
if sent % 250 == 0 {
|
||||
log::info!(
|
||||
"mic: sent={sent} captured_frames={} peak={peak:.3}",
|
||||
captured.load(Ordering::Relaxed),
|
||||
);
|
||||
peak = 0.0;
|
||||
}
|
||||
}
|
||||
Err(e) => log::debug!("mic: opus encode: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
log::info!(
|
||||
"mic: stopped (sent={sent} captured_frames={})",
|
||||
captured.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
@@ -33,6 +33,8 @@ pub(crate) struct SessionHandle {
|
||||
video: Mutex<Option<VideoThread>>,
|
||||
#[cfg(target_os = "android")]
|
||||
audio: Mutex<Option<crate::audio::AudioPlayback>>,
|
||||
#[cfg(target_os = "android")]
|
||||
mic: Mutex<Option<crate::mic::MicCapture>>,
|
||||
}
|
||||
|
||||
struct VideoThread {
|
||||
@@ -57,6 +59,13 @@ impl SessionHandle {
|
||||
fn stop_audio(&self) {
|
||||
let _ = self.audio.lock().unwrap().take();
|
||||
}
|
||||
|
||||
/// Stop mic uplink. Dropping the [`crate::mic::MicCapture`] joins its encode thread and closes
|
||||
/// the AAudio input stream. Idempotent.
|
||||
#[cfg(target_os = "android")]
|
||||
fn stop_mic(&self) {
|
||||
let _ = self.mic.lock().unwrap().take();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SessionHandle {
|
||||
@@ -64,6 +73,8 @@ impl Drop for SessionHandle {
|
||||
self.stop_video();
|
||||
#[cfg(target_os = "android")]
|
||||
self.stop_audio();
|
||||
#[cfg(target_os = "android")]
|
||||
self.stop_mic();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,6 +193,8 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo
|
||||
video: Mutex::new(None),
|
||||
#[cfg(target_os = "android")]
|
||||
audio: Mutex::new(None),
|
||||
#[cfg(target_os = "android")]
|
||||
mic: Mutex::new(None),
|
||||
};
|
||||
Box::into_raw(Box::new(handle)) as jlong
|
||||
}
|
||||
@@ -381,6 +394,47 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopAudio(
|
||||
}
|
||||
}
|
||||
|
||||
/// `NativeBridge.nativeStartMic(handle)` — start mic capture (AAudio input → Opus → host `send_mic`).
|
||||
/// No-op if already running or on a `0` handle. Caller MUST hold RECORD_AUDIO; a failure (e.g. no
|
||||
/// permission) leaves the rest of the session streaming.
|
||||
#[cfg(target_os = "android")]
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartMic(
|
||||
_env: JNIEnv,
|
||||
_this: JObject,
|
||||
handle: jlong,
|
||||
) {
|
||||
if handle == 0 {
|
||||
return;
|
||||
}
|
||||
// SAFETY: live handle per the nativeConnect/nativeClose contract.
|
||||
let h = unsafe { &*(handle as *const SessionHandle) };
|
||||
let mut guard = h.mic.lock().unwrap();
|
||||
if guard.is_some() {
|
||||
return; // already capturing
|
||||
}
|
||||
match crate::mic::MicCapture::start(h.client.clone()) {
|
||||
Some(m) => *guard = Some(m),
|
||||
None => log::error!("nativeStartMic: mic init failed (RECORD_AUDIO? — session unaffected)"),
|
||||
}
|
||||
}
|
||||
|
||||
/// `NativeBridge.nativeStopMic(handle)` — stop + join the mic thread and close the AAudio input
|
||||
/// stream (without closing the session). No-op on `0`.
|
||||
#[cfg(target_os = "android")]
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopMic(
|
||||
_env: JNIEnv,
|
||||
_this: JObject,
|
||||
handle: jlong,
|
||||
) {
|
||||
if handle != 0 {
|
||||
// SAFETY: live handle per the contract.
|
||||
let h = unsafe { &*(handle as *const SessionHandle) };
|
||||
h.stop_mic();
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Input plane: Kotlin capture → NativeClient::send_input ----------------------------------
|
||||
// All four are `&self` on the `Sync` connector (send_input is a non-blocking datagram push), safe
|
||||
// from the Kotlin UI thread. NOT android-gated — send_input exists on the host build too, so these
|
||||
|
||||
Reference in New Issue
Block a user