Compare commits
42
Commits
Generated
+32
-32
@@ -947,7 +947,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cursor-probe"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"pf-capture",
|
||||
@@ -1036,7 +1036,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "display-disturb"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"windows 0.62.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
@@ -2221,7 +2221,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "latency-probe"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
@@ -2326,7 +2326,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libvpl-sys"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cmake",
|
||||
@@ -2361,7 +2361,7 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
|
||||
|
||||
[[package]]
|
||||
name = "loss-harness"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"punktfunk-core",
|
||||
]
|
||||
@@ -2850,7 +2850,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "pf-capture"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"ashpd",
|
||||
@@ -2871,7 +2871,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-client-core"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"ash",
|
||||
@@ -2897,7 +2897,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-clipboard"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"ashpd",
|
||||
@@ -2915,7 +2915,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-console-ui"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"ash",
|
||||
@@ -2936,7 +2936,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-encode"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"ash",
|
||||
@@ -2960,7 +2960,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-ffvk"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"ash",
|
||||
"bindgen",
|
||||
@@ -2969,7 +2969,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-frame"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"libc",
|
||||
@@ -2981,7 +2981,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-gpu"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"pf-host-config",
|
||||
@@ -2995,11 +2995,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-host-config"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
|
||||
[[package]]
|
||||
name = "pf-inject"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"ashpd",
|
||||
@@ -3028,14 +3028,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-paths"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pf-presenter"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"ash",
|
||||
@@ -3050,7 +3050,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-update"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -3058,7 +3058,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-update-check"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
@@ -3070,7 +3070,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-vdisplay"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"ashpd",
|
||||
@@ -3103,7 +3103,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-win-display"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"pf-paths",
|
||||
@@ -3115,7 +3115,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pf-zerocopy"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"ash",
|
||||
@@ -3323,7 +3323,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "punktfunk-cli"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"pf-client-core",
|
||||
"punktfunk-core",
|
||||
@@ -3334,7 +3334,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "punktfunk-client-android"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"android_logger",
|
||||
"jni",
|
||||
@@ -3350,7 +3350,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "punktfunk-client-linux"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-channel",
|
||||
@@ -3367,7 +3367,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "punktfunk-client-session"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"pf-client-core",
|
||||
@@ -3382,7 +3382,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "punktfunk-client-windows"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"async-channel",
|
||||
"ffmpeg-next",
|
||||
@@ -3402,7 +3402,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "punktfunk-core"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -3434,7 +3434,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "punktfunk-host"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"aes-gcm",
|
||||
@@ -3519,7 +3519,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "punktfunk-probe"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"mdns-sd",
|
||||
@@ -3533,7 +3533,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "punktfunk-tray"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"ksni",
|
||||
@@ -3556,7 +3556,7 @@ checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
|
||||
|
||||
[[package]]
|
||||
name = "pyrowave-sys"
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cmake",
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@ exclude = [
|
||||
ndk = { path = "clients/android/native/vendor/ndk" }
|
||||
|
||||
[workspace.package]
|
||||
version = "0.22.3"
|
||||
version = "0.23.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.82"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
+7
-2
@@ -3495,7 +3495,7 @@
|
||||
"operationId": "forceUpdateCheck",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Refreshed update-check state (`last_error` carries a failed check)",
|
||||
"description": "Refreshed update-check state (`last_error` carries a failed check; `not_published` an empty channel, which is not one)",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
@@ -7372,7 +7372,8 @@
|
||||
"apply",
|
||||
"channel_hint",
|
||||
"check_disabled",
|
||||
"available"
|
||||
"available",
|
||||
"not_published"
|
||||
],
|
||||
"properties": {
|
||||
"apply": {
|
||||
@@ -7452,6 +7453,10 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"not_published": {
|
||||
"type": "boolean",
|
||||
"description": "The check reached the feed and found this channel has **no release published yet** —\nan expected state (a channel nobody has announced to answers with a 404), not a\nfailure. Mutually exclusive with `last_error`, so a UI can say \"nothing published yet\"\ninstead of painting an empty feed as a broken host. Never set once a manifest has been\nseen for this channel: a feed that loses a document it used to serve stays an error."
|
||||
},
|
||||
"opt_in_hint": {
|
||||
"type": [
|
||||
"string",
|
||||
|
||||
@@ -395,6 +395,11 @@ private fun buildSettingsRows(
|
||||
"mic", null, "Microphone", "Send this device's microphone to the host's virtual mic.",
|
||||
s.micEnabled,
|
||||
) { update(s.copy(micEnabled = it)) },
|
||||
toggle(
|
||||
"echoCancel", null, "Echo cancellation",
|
||||
"Filter the stream's own audio out of the mic pickup. Applies while the microphone is on.",
|
||||
s.echoCancel,
|
||||
) { update(s.copy(echoCancel = it)) },
|
||||
|
||||
choice(
|
||||
"padType", "Controllers", "Controller type",
|
||||
|
||||
@@ -2,6 +2,7 @@ package io.unom.punktfunk
|
||||
|
||||
import android.content.Context
|
||||
import android.os.Build
|
||||
import android.util.Log
|
||||
import io.unom.punktfunk.kit.Gamepad
|
||||
import io.unom.punktfunk.kit.NativeBridge
|
||||
import io.unom.punktfunk.kit.VideoDecoders
|
||||
@@ -45,18 +46,40 @@ suspend fun connectToHost(
|
||||
// Transport-level half of "Low-latency mode (experimental)" (DSCP marking on the media
|
||||
// sockets) — must be applied before connect, since sockets are tagged at creation.
|
||||
NativeBridge.nativeSetLowLatencyMode(settings.lowLatencyMode)
|
||||
val multiSlice = VideoDecoders.multiSliceTolerant()
|
||||
val partialFrame = VideoDecoders.partialFrameCapable()
|
||||
// Slice-progressive delivery: decoder truth AND the async decode loop — the legacy
|
||||
// sync loop feeds whole AUs only, so parts must never arrive when it is selected.
|
||||
val frameParts = settings.lowLatencyMode && partialFrame
|
||||
val codecBits = VideoDecoders.decodableCodecBits()
|
||||
// Automatic codec (P5, measured NP3 ↔ RTX 4090): AV1 beat HEVC by ~1.2 ms end-to-end at
|
||||
// identical conditions, so under "Automatic" this device prefers AV1 when it hardware-
|
||||
// decodes it (the AV1 bit is only ever set for a real, non-blocked hardware decoder) AND
|
||||
// it lacks FEATURE_PartialFrame — a partial-frame device keeps HEVC, whose slice overlap
|
||||
// AV1 cannot ride (AV1 has no slices; the host's chunked poll never arms). The host
|
||||
// honors the preference only inside the probed shared codec set, so an AV1-less encoder
|
||||
// still resolves HEVC. An explicit user choice always wins unchanged.
|
||||
val preferredCodec = settings.preferredCodec().takeIf { it != 0 }
|
||||
?: if (codecBits and 4 != 0 && !partialFrame) 4 else 0
|
||||
// The connect-time capability readout (`adb logcat -s pf.caps`): the P2 slice pipeline
|
||||
// is client-inert unless BOTH probes pass — this line says which decoder failed one.
|
||||
Log.i(
|
||||
"pf.caps",
|
||||
VideoDecoders.capsReport() +
|
||||
" → multiSlice=$multiSlice parts=$frameParts prefer=$preferredCodec" +
|
||||
" (lowLatency=${settings.lowLatencyMode})",
|
||||
)
|
||||
NativeBridge.nativeConnect(
|
||||
host, port, w, h, hz,
|
||||
identity.certPem, identity.privateKeyPem, pinHex,
|
||||
settings.bitrateKbps, settings.compositor, gamepadPref,
|
||||
hdrEnabled, VideoDecoders.multiSliceTolerant(),
|
||||
// Slice-progressive delivery: decoder truth AND the async decode loop — the legacy
|
||||
// sync loop feeds whole AUs only, so parts must never arrive when it is selected.
|
||||
settings.lowLatencyMode && VideoDecoders.partialFrameCapable(),
|
||||
hdrEnabled, multiSlice,
|
||||
frameParts,
|
||||
settings.audioChannels,
|
||||
// What this device can decode (H.264|HEVC always, AV1 when a real decoder exists) +
|
||||
// the user's soft codec preference — the host resolves the emitted codec from both.
|
||||
VideoDecoders.decodableCodecBits(), settings.preferredCodec(), timeoutMs,
|
||||
// the soft codec preference (user choice, or the Automatic AV1 rule above) — the
|
||||
// host resolves the emitted codec from both.
|
||||
codecBits, preferredCodec, timeoutMs,
|
||||
launch,
|
||||
// The host's approval-list / trust-store label for this device — the same
|
||||
// Build.MODEL convention the pairing dialogs use for nativePair.
|
||||
|
||||
@@ -38,6 +38,7 @@ data class SettingsOverlay(
|
||||
val compositor: Int? = null,
|
||||
val audioChannels: Int? = null,
|
||||
val micEnabled: Boolean? = null,
|
||||
val echoCancel: Boolean? = null,
|
||||
val touchMode: TouchMode? = null,
|
||||
val mouseMode: MouseMode? = null,
|
||||
val invertScroll: Boolean? = null,
|
||||
@@ -70,6 +71,7 @@ data class SettingsOverlay(
|
||||
compositor = compositor ?: base.compositor,
|
||||
audioChannels = audioChannels ?: base.audioChannels,
|
||||
micEnabled = micEnabled ?: base.micEnabled,
|
||||
echoCancel = echoCancel ?: base.echoCancel,
|
||||
touchMode = touchMode ?: base.touchMode,
|
||||
mouseMode = mouseMode ?: base.mouseMode,
|
||||
invertScroll = invertScroll ?: base.invertScroll,
|
||||
@@ -103,6 +105,7 @@ data class SettingsOverlay(
|
||||
compositor = if (after.compositor != before.compositor) after.compositor else compositor,
|
||||
audioChannels = if (after.audioChannels != before.audioChannels) after.audioChannels else audioChannels,
|
||||
micEnabled = if (after.micEnabled != before.micEnabled) after.micEnabled else micEnabled,
|
||||
echoCancel = if (after.echoCancel != before.echoCancel) after.echoCancel else echoCancel,
|
||||
touchMode = if (after.touchMode != before.touchMode) after.touchMode else touchMode,
|
||||
mouseMode = if (after.mouseMode != before.mouseMode) after.mouseMode else mouseMode,
|
||||
invertScroll = if (after.invertScroll != before.invertScroll) after.invertScroll else invertScroll,
|
||||
@@ -128,6 +131,7 @@ data class SettingsOverlay(
|
||||
"compositor" -> copy(compositor = null)
|
||||
"audio_channels" -> copy(audioChannels = null)
|
||||
"mic_enabled" -> copy(micEnabled = null)
|
||||
"echo_cancel" -> copy(echoCancel = null)
|
||||
"touch_mode" -> copy(touchMode = null)
|
||||
"mouse_mode" -> copy(mouseMode = null)
|
||||
"invert_scroll" -> copy(invertScroll = null)
|
||||
@@ -150,6 +154,7 @@ data class SettingsOverlay(
|
||||
if (compositor != null) add("compositor")
|
||||
if (audioChannels != null) add("audio_channels")
|
||||
if (micEnabled != null) add("mic_enabled")
|
||||
if (echoCancel != null) add("echo_cancel")
|
||||
if (touchMode != null) add("touch_mode")
|
||||
if (mouseMode != null) add("mouse_mode")
|
||||
if (invertScroll != null) add("invert_scroll")
|
||||
@@ -180,6 +185,7 @@ data class SettingsOverlay(
|
||||
compositor?.let { j.put("compositor", it) }
|
||||
audioChannels?.let { j.put("audio_channels", it) }
|
||||
micEnabled?.let { j.put("mic_enabled", it) }
|
||||
echoCancel?.let { j.put("echo_cancel", it) }
|
||||
touchMode?.let { j.put("touch_mode", it.name) }
|
||||
mouseMode?.let { j.put("mouse_mode", it.storedName) }
|
||||
invertScroll?.let { j.put("invert_scroll", it) }
|
||||
@@ -198,9 +204,9 @@ data class SettingsOverlay(
|
||||
/** Keys this build models; everything else in a stored overlay is carried through. */
|
||||
private val KNOWN = setOf(
|
||||
"width", "height", "refresh_hz", "bitrate_kbps", "render_scale", "codec",
|
||||
"hdr_enabled", "compositor", "audio_channels", "mic_enabled", "touch_mode",
|
||||
"mouse_mode", "invert_scroll", "gamepad", "stats_verbosity", "low_latency_mode",
|
||||
"present_priority", "smooth_buffer",
|
||||
"hdr_enabled", "compositor", "audio_channels", "mic_enabled", "echo_cancel",
|
||||
"touch_mode", "mouse_mode", "invert_scroll", "gamepad", "stats_verbosity",
|
||||
"low_latency_mode", "present_priority", "smooth_buffer",
|
||||
)
|
||||
|
||||
internal fun fromJson(j: JSONObject): SettingsOverlay = SettingsOverlay(
|
||||
@@ -214,6 +220,7 @@ data class SettingsOverlay(
|
||||
compositor = j.optIntOrNull("compositor"),
|
||||
audioChannels = j.optIntOrNull("audio_channels"),
|
||||
micEnabled = j.optBooleanOrNull("mic_enabled"),
|
||||
echoCancel = j.optBooleanOrNull("echo_cancel"),
|
||||
touchMode = j.optStringOrNull("touch_mode")
|
||||
?.let { n -> TouchMode.entries.firstOrNull { it.name == n } },
|
||||
mouseMode = j.optStringOrNull("mouse_mode")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package io.unom.punktfunk
|
||||
|
||||
import android.content.Context
|
||||
import android.hardware.display.DisplayManager
|
||||
import android.os.Build
|
||||
import android.util.Log
|
||||
import android.view.Display
|
||||
@@ -41,6 +42,15 @@ data class Settings(
|
||||
* the host resolves (AV1 is only advertised/offered when the device has a real AV1 decoder). */
|
||||
val codec: String = "auto",
|
||||
val micEnabled: Boolean = false,
|
||||
/**
|
||||
* Cancel acoustic echo on the mic uplink (plus noise suppression): the capture opens under
|
||||
* the VoiceCommunication preset so the HAL's own AEC/NS process it, with the Java effects
|
||||
* attached as a backstop where available. On by default — a phone/tablet plays the game audio
|
||||
* out of the same device its mic hears, so without this the host hears its own stream back.
|
||||
* Turn off for a headset-only setup where the untouched full-band capture sounds better.
|
||||
* Only meaningful while [micEnabled] is on.
|
||||
*/
|
||||
val echoCancel: Boolean = true,
|
||||
/**
|
||||
* How much the in-stream stats overlay shows — see [StatsVerbosity]. Defaults to
|
||||
* [StatsVerbosity.NORMAL] (the res/fps line + latency headline + reliability counters); the full
|
||||
@@ -209,6 +219,7 @@ class SettingsStore(context: Context) {
|
||||
audioChannels = prefs.getInt(K_AUDIO_CH, 2),
|
||||
codec = prefs.getString(K_CODEC, "auto") ?: "auto",
|
||||
micEnabled = prefs.getBoolean(K_MIC, false),
|
||||
echoCancel = prefs.getBoolean(K_ECHO_CANCEL, true),
|
||||
statsVerbosity = prefs.getString(K_STATS_VERBOSITY, null)
|
||||
?.let { name -> StatsVerbosity.entries.firstOrNull { it.name == name } }
|
||||
// Migration from the pre-tier Boolean "stats_hud_enabled": an explicit OFF stays off;
|
||||
@@ -254,6 +265,7 @@ class SettingsStore(context: Context) {
|
||||
.putInt(K_AUDIO_CH, s.audioChannels)
|
||||
.putString(K_CODEC, s.codec)
|
||||
.putBoolean(K_MIC, s.micEnabled)
|
||||
.putBoolean(K_ECHO_CANCEL, s.echoCancel)
|
||||
.putString(K_STATS_VERBOSITY, s.statsVerbosity.name)
|
||||
.putString(K_TOUCH_MODE, s.touchMode.name)
|
||||
.putBoolean(K_GAMEPAD_UI, s.gamepadUiEnabled)
|
||||
@@ -282,6 +294,7 @@ class SettingsStore(context: Context) {
|
||||
const val K_AUDIO_CH = "audio_channels"
|
||||
const val K_CODEC = "codec"
|
||||
const val K_MIC = "mic_enabled"
|
||||
const val K_ECHO_CANCEL = "echo_cancel"
|
||||
const val K_STATS_VERBOSITY = "stats_verbosity"
|
||||
|
||||
/** Pre-tier Boolean the [K_STATS_VERBOSITY] enum replaced — read once for migration, never
|
||||
@@ -319,14 +332,31 @@ class SettingsStore(context: Context) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The display to probe for capability/mode queries: the context's own display when it is already
|
||||
* associated with one, else the DEFAULT display via [DisplayManager]. A `punktfunk://` deep-link
|
||||
* COLD start can reach the connect before the activity is attached to its display —
|
||||
* `context.display` then throws, and the old `false`/1080p60 fallbacks silently downgraded the
|
||||
* whole session (no HDR advertised / non-native mode) with nothing in the log. The default
|
||||
* display IS the panel on phones and TVs; the activity-display distinction only matters on
|
||||
* multi-display setups, where the attached path still wins whenever it is available.
|
||||
*/
|
||||
private fun probeDisplay(context: Context): Display? =
|
||||
runCatching { context.display }.getOrNull()
|
||||
?: runCatching {
|
||||
context.getSystemService(DisplayManager::class.java)
|
||||
?.getDisplay(Display.DEFAULT_DISPLAY)
|
||||
}.getOrNull().also {
|
||||
if (it != null) Log.i("punktfunk", "display probe: context unattached — using DEFAULT_DISPLAY")
|
||||
}
|
||||
|
||||
/**
|
||||
* The device's native display mode as a landscape `(width, height, hz)` — the long edge is the
|
||||
* width, since we stream a desktop. Falls back to 1920×1080@60 if the display can't be read.
|
||||
* [context] must be a visual (Activity) context.
|
||||
* width, since we stream a desktop. Falls back to 1920×1080@60 if no display can be read at all
|
||||
* (see [probeDisplay] for the cold-start fallback that makes that a last resort).
|
||||
*/
|
||||
fun nativeDisplayMode(context: Context): Triple<Int, Int, Int> {
|
||||
// getDisplay() throws on a non-visual context rather than returning null — guard it.
|
||||
val display = runCatching { context.display }.getOrNull() ?: return Triple(1920, 1080, 60)
|
||||
val display = probeDisplay(context) ?: return Triple(1920, 1080, 60)
|
||||
val mode = display.mode
|
||||
val w = mode.physicalWidth
|
||||
val h = mode.physicalHeight
|
||||
@@ -341,7 +371,12 @@ fun nativeDisplayMode(context: Context): Triple<Int, Int, Int> {
|
||||
* capability gate the Apple/Windows clients apply.
|
||||
*/
|
||||
fun displaySupportsHdr(context: Context): Boolean {
|
||||
val display = runCatching { context.display }.getOrNull() ?: return false
|
||||
val display = probeDisplay(context)
|
||||
if (display == null) {
|
||||
// Distinguishable from a real SDR verdict — a silent `false` here cost an HDR session.
|
||||
Log.w("punktfunk", "display HDR probe: no display reachable — advertising SDR")
|
||||
return false
|
||||
}
|
||||
val types = buildSet {
|
||||
// API 34+: the sanctioned per-mode query (Display.Mode.getSupportedHdrTypes). The
|
||||
// deprecated Display-level hdrCapabilities can return EMPTY on Android 14+ devices
|
||||
|
||||
@@ -673,12 +673,22 @@ private fun DisplaySettings(s: Settings, update: (Settings) -> Unit, context: an
|
||||
// Only codecs this device can actually decode are offered — a preference the client never
|
||||
// advertises would be a dead setting (see [codecOptionsFor]).
|
||||
val av1Capable = remember { VideoDecoders.pickDecoder("video/av01") != null }
|
||||
// Mirror the Automatic AV1 rule in HostConnect (hardware AV1 AND no partial-frame
|
||||
// support) so the picker says what "Automatic" actually does on THIS device.
|
||||
val autoPrefersAv1 = remember {
|
||||
VideoDecoders.decodableCodecBits() and 4 != 0 && !VideoDecoders.partialFrameCapable()
|
||||
}
|
||||
SettingDropdown(
|
||||
label = "Video codec",
|
||||
options = codecOptionsFor(s.codec, av1Capable),
|
||||
selected = s.codec,
|
||||
field = "codec",
|
||||
caption = "A preference — the host falls back if it can't encode this one.",
|
||||
caption = if (autoPrefersAv1) {
|
||||
"A preference — the host falls back if it can't encode this one. " +
|
||||
"Automatic prefers AV1 on this device."
|
||||
} else {
|
||||
"A preference — the host falls back if it can't encode this one."
|
||||
},
|
||||
) { c -> update(s.copy(codec = c)) }
|
||||
|
||||
// HDR is only meaningful on a panel that can present HDR10; on an SDR display the toggle is
|
||||
@@ -794,6 +804,14 @@ private fun AudioSettings(s: Settings, update: (Settings) -> Unit, onMicChange:
|
||||
field = "mic_enabled",
|
||||
onCheckedChange = onMicChange,
|
||||
)
|
||||
ToggleRow(
|
||||
title = "Echo cancellation",
|
||||
subtitle = "Filters the stream's own audio out of the mic pickup",
|
||||
checked = s.echoCancel,
|
||||
enabled = s.micEnabled,
|
||||
field = "echo_cancel",
|
||||
onCheckedChange = { on -> update(s.copy(echoCancel = on)) },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,11 +18,13 @@ import kotlin.math.roundToInt
|
||||
* The live stats overlay — the unified HUD (`design/stats-unification.md`): headline is
|
||||
* `capture→displayed` tiled by `host+network` + `decode` + `display` when the platform delivered
|
||||
* OnFrameRendered render callbacks this window (`dispValid`), falling back to the v1
|
||||
* `capture→decoded` headline without the `display` term when it didn't. Reads the 26-double
|
||||
* layout from [NativeBridge.nativeVideoStats]:
|
||||
* `capture→decoded` headline without the `display` term when it didn't. Reads the 33-double
|
||||
* layout from [NativeBridge.nativeVideoStats] (that KDoc is the authoritative index list):
|
||||
* `[fps, mbps, e2eP50Ms, e2eP95Ms, latValid, skew, w, h, hz, lostTotal, bitDepth, colorPrimaries,
|
||||
* colorTransfer, chromaFormatIdc, hostNetP50Ms, decodeP50Ms, hostP50Ms, netP50Ms, lost, skipped,
|
||||
* fec, frames, dispValid, displayP50Ms, e2eDispP50Ms, e2eDispP95Ms]`.
|
||||
* fec, frames, dispValid, displayP50Ms, e2eDispP50Ms, e2eDispP95Ms, paceP50Ms, latchP50Ms,
|
||||
* presentsWindow, presenterActive, feedP50Ms, codecP50Ms, skippedOverflowWindow]`. Every read
|
||||
* is length-guarded, so an older native lib simply omits the lines it can't feed.
|
||||
*
|
||||
* [verbosity] selects how many lines render (each tier a superset of the last — see
|
||||
* [StatsVerbosity]):
|
||||
@@ -129,10 +131,30 @@ internal fun StatsOverlay(
|
||||
} else {
|
||||
""
|
||||
}
|
||||
// P3 decode split (s[30]/s[31]): `feed` = received→queued (hand-off + input-slot
|
||||
// wait) + `codec` = queued→decoded (codec-pure) — rendered when a sample landed.
|
||||
val decodeTerm = if (s.size >= 33 && (s[30] > 0 || s[31] > 0)) {
|
||||
"decode ${"%.1f".format(s[15])} " +
|
||||
"(feed ${"%.1f".format(s[30])} + codec ${"%.1f".format(s[31])})"
|
||||
} else {
|
||||
"decode ${"%.1f".format(s[15])}"
|
||||
}
|
||||
statLine(
|
||||
"= $hostTerms + decode ${"%.1f".format(s[15])}$displayTerm$presents",
|
||||
"= $hostTerms + $decodeTerm$displayTerm$presents",
|
||||
Color.White,
|
||||
)
|
||||
// Metric fairness: the Apple client's HUD shaves ~2 refresh periods of OS
|
||||
// pipeline floor off its shown display/end-to-end; Android shows raw. This twin
|
||||
// applies the same shave so iPhone↔Android HUD numbers compare directly.
|
||||
if (dispValid && hz > 0) {
|
||||
val shave = 2000.0 / hz
|
||||
statLine(
|
||||
"≈ Apple-HUD equiv: end-to-end " +
|
||||
"${"%.1f".format((s[24] - shave).coerceAtLeast(0.0))} · display " +
|
||||
"${"%.1f".format((s[23] - shave).coerceAtLeast(0.0))} (−2 refresh)",
|
||||
Color(0xFFA8D8B8),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
counterLine(s, lost)?.let { statLine(it, Color(0xFFFFB0B0)) }
|
||||
@@ -178,12 +200,17 @@ private fun counterLine(s: DoubleArray, lostTotal: Long): String? {
|
||||
val fec = s[20].toLong()
|
||||
val frames = s[21].toLong()
|
||||
if (lost == 0L && skipped == 0L && fec == 0L) return null
|
||||
// The overflow subset of `skipped` (s[32]): whole AUs dropped before feeding — the decoder
|
||||
// fell behind. Absent (0 / old layout) the plain count keeps meaning benign pacing drops.
|
||||
val overflow = if (s.size >= 33) s[32].toLong() else 0L
|
||||
return buildList {
|
||||
if (lost > 0) {
|
||||
val pct = 100.0 * lost / (frames + lost).coerceAtLeast(1)
|
||||
add("lost $lost (${"%.1f".format(pct)}%)")
|
||||
}
|
||||
if (skipped > 0) add("skipped $skipped")
|
||||
if (skipped > 0) {
|
||||
add(if (overflow > 0) "skipped $skipped (⚠ $overflow overflow)" else "skipped $skipped")
|
||||
}
|
||||
if (fec > 0) add("FEC $fec")
|
||||
}.joinToString(" · ")
|
||||
}
|
||||
|
||||
@@ -9,11 +9,15 @@ import android.content.IntentFilter
|
||||
import android.content.pm.ActivityInfo
|
||||
import android.content.pm.PackageManager
|
||||
import android.hardware.usb.UsbManager
|
||||
import android.media.audiofx.AcousticEchoCanceler
|
||||
import android.media.audiofx.AudioEffect
|
||||
import android.media.audiofx.NoiseSuppressor
|
||||
import android.net.wifi.WifiManager
|
||||
import android.os.Build
|
||||
import android.text.InputType
|
||||
import android.util.Log
|
||||
import android.view.KeyEvent
|
||||
import android.view.Surface
|
||||
import android.view.SurfaceHolder
|
||||
import android.view.SurfaceView
|
||||
import android.view.View
|
||||
@@ -25,12 +29,20 @@ import android.view.inputmethod.InputMethodManager
|
||||
import android.widget.Toast
|
||||
import androidx.activity.compose.BackHandler
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.aspectRatio
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.filled.Mic
|
||||
import androidx.compose.material.icons.filled.MicOff
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.DisposableEffect
|
||||
@@ -41,6 +53,7 @@ import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.draw.clip
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.input.pointer.pointerInput
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
@@ -97,6 +110,38 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) {
|
||||
Manifest.permission.RECORD_AUDIO,
|
||||
) == PackageManager.PERMISSION_GRANTED
|
||||
|
||||
// The Java AEC/NS pair backstopping the native VoiceCommunication capture preset, hung off the
|
||||
// audio session id `nativeStartMic` returns. Attached in surfaceCreated (where the mic starts)
|
||||
// and released on every path that stops the mic — the surface teardown AND the final dispose —
|
||||
// so a surface recreate re-attaches to the fresh stream instead of leaking effect engines.
|
||||
// All three touch points run on the main thread; a plain list is race-free.
|
||||
val micEffects = remember { mutableListOf<AudioEffect>() }
|
||||
|
||||
// In-stream mic mute. Per SESSION and never persisted (no setting backs it): a new stream
|
||||
// always starts unmuted. The authoritative flag lives on the native handle, which is why a mute
|
||||
// survives the mic stop/start a surface recreate performs — this state is the UI's mirror of
|
||||
// it, and survives the same recreate because the composition outlives the surface.
|
||||
var micMuted by remember(handle) { mutableStateOf(false) }
|
||||
// Whether a capture is actually RUNNING, not merely wanted — set from surfaceCreated on what
|
||||
// nativeMicActive reports. A device that refused every AAudio input rung gets no mute control
|
||||
// rather than one that lies about a mic being heard.
|
||||
var micRunning by remember(handle) { mutableStateOf(false) }
|
||||
// Transient confirmation of a mic-chord toggle (null = nothing showing). Only the gamepad path
|
||||
// needs it: the touch button confirms itself by changing under the finger, but a chord has no
|
||||
// on-screen state of its own, and "did that register?" is exactly the doubt to answer.
|
||||
var micHint by remember { mutableStateOf<String?>(null) }
|
||||
LaunchedEffect(micHint) {
|
||||
if (micHint != null) {
|
||||
delay(1600)
|
||||
micHint = null
|
||||
}
|
||||
}
|
||||
// The one place mute is toggled — Compose state + the native flag, always together.
|
||||
val setMicMuted = { muted: Boolean ->
|
||||
micMuted = muted
|
||||
NativeBridge.nativeSetMicMuted(handle, muted)
|
||||
}
|
||||
|
||||
// Live decode stats for the HUD. `statsOn` (verbosity != OFF) gates the whole native pipeline:
|
||||
// the per-frame sampling (nativeSetVideoStatsEnabled — a hidden HUD costs one atomic load per
|
||||
// frame) AND the 1 s poll loop, which only runs while the overlay is visible. Enabling resets
|
||||
@@ -287,6 +332,16 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) {
|
||||
// Show a "hold to quit" hint the moment the chord completes (the router debounces the actual
|
||||
// exit); it clears when the buttons release early or the hold elapses. Runs on the main thread.
|
||||
router.onExitArmed = { armed -> exitArming = armed }
|
||||
// Select + Y toggles the mic — the couch reach for the on-screen mute button, which a
|
||||
// gamepad/TV user has no pointer for. Ignored when no capture is running (there is nothing
|
||||
// to mute, and claiming otherwise would be the lie the control exists to avoid).
|
||||
router.onMicChord = {
|
||||
if (micRunning) {
|
||||
val next = !micMuted
|
||||
setMicMuted(next)
|
||||
micHint = if (next) "Microphone muted" else "Microphone live"
|
||||
}
|
||||
}
|
||||
// Physical mouse: uncaptured hover/click/wheel forwards as absolute pointing; captured
|
||||
// (setting or the Ctrl+Alt+Shift+Q chord) raw deltas forward as relative mouse-look.
|
||||
// The local cursor is hidden over the stream — the host's own cursor, composited into
|
||||
@@ -483,6 +538,7 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) {
|
||||
dsUsbReceiver?.let { runCatching { context.unregisterReceiver(it) } }
|
||||
ds?.stop() // rumble-stop on the physical pad + release the USB link + free the wire slot
|
||||
router.onExitArmed = null // don't poke Compose state from release()'s disarm while tearing down
|
||||
router.onMicChord = null // same: no mute toggle on buttons released during teardown
|
||||
router.release() // flush every slot (nothing sticks host-side) + drop the hot-plug listener
|
||||
activity?.gamepadRouter = null
|
||||
// Mouse/remote-pointer teardown: lift held buttons, drop the grab, restore the cursor.
|
||||
@@ -519,6 +575,7 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) {
|
||||
activity?.requestedOrientation =
|
||||
priorOrientation ?: ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED
|
||||
// Leaving the stream: stop the mic + audio + decode threads and tear down the session.
|
||||
releaseMicEffects(micEffects)
|
||||
NativeBridge.nativeStopMic(handle)
|
||||
NativeBridge.nativeStopAudio(handle)
|
||||
NativeBridge.nativeStopVideo(handle)
|
||||
@@ -609,10 +666,38 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) {
|
||||
.roundToInt(),
|
||||
)
|
||||
NativeBridge.nativeStartAudio(handle, lowLatencyMode)
|
||||
if (micWanted) NativeBridge.nativeStartMic(handle)
|
||||
if (micWanted) {
|
||||
val sessionId =
|
||||
NativeBridge.nativeStartMic(handle, initialSettings.echoCancel)
|
||||
if (initialSettings.echoCancel) {
|
||||
attachMicEffects(sessionId, micEffects)
|
||||
}
|
||||
// Did a capture actually open? That — not the setting — is what
|
||||
// puts the mute control on screen. A restart after a surface
|
||||
// recreate comes back already muted if the user muted: the flag
|
||||
// lives on the session handle, so nothing has to be re-applied.
|
||||
micRunning = NativeBridge.nativeMicActive(handle)
|
||||
}
|
||||
}
|
||||
|
||||
override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {}
|
||||
override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {
|
||||
// Re-assert the frame-rate vote: a buffer-geometry change can reset
|
||||
// the surface's frame-rate setting on some OEM builds, silently
|
||||
// dropping the 120 Hz pin mid-stream. Mirrors the native hint's
|
||||
// policy (FIXED_SOURCE; ALWAYS only on the TV low-latency path —
|
||||
// phones stay seamless so a re-hint can never force a mode flicker).
|
||||
if (streamHz > 0) runCatching {
|
||||
holder.surface.setFrameRate(
|
||||
streamHz.toFloat(),
|
||||
Surface.FRAME_RATE_COMPATIBILITY_FIXED_SOURCE,
|
||||
if (isTv && lowLatencyMode) {
|
||||
Surface.CHANGE_FRAME_RATE_ALWAYS
|
||||
} else {
|
||||
Surface.CHANGE_FRAME_RATE_ONLY_IF_SEAMLESS
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override fun surfaceDestroyed(holder: SurfaceHolder) {
|
||||
// Surface gone (backgrounding, or on the way out). Stop the threads that
|
||||
@@ -620,7 +705,12 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) {
|
||||
// DisposableEffect has closed it, the handle is freed; dereferencing it
|
||||
// here is the use-after-free that crashed on back-navigation.
|
||||
if (!closed.get()) {
|
||||
releaseMicEffects(micEffects)
|
||||
NativeBridge.nativeStopMic(handle)
|
||||
// No capture, no control — but the MUTE state is deliberately left
|
||||
// standing (native keeps it on the handle), so the restart in
|
||||
// surfaceCreated brings the user's choice back with it.
|
||||
micRunning = false
|
||||
NativeBridge.nativeStopAudio(handle)
|
||||
NativeBridge.nativeStopVideo(handle)
|
||||
}
|
||||
@@ -695,9 +785,106 @@ fun StreamScreen(session: ActiveSession, onDisconnect: () -> Unit) {
|
||||
}
|
||||
},
|
||||
)
|
||||
// Mic mute, LAST in the stack — the one in-stream control, so unlike the purely visual
|
||||
// overlays above it has to sit on top of the gesture layer to receive its own taps (it
|
||||
// costs the stream that small corner of touch area, which is why it exists only while a
|
||||
// capture actually runs). On TV it is the indicator alone: the Select + Y chord is the
|
||||
// control there, and a focusable button would fight the game for the D-pad.
|
||||
if (micRunning && (micMuted || !isTv)) {
|
||||
MicMuteControl(
|
||||
muted = micMuted,
|
||||
onToggle = if (isTv) null else ({ setMicMuted(!micMuted) }),
|
||||
modifier = Modifier.align(Alignment.TopEnd).padding(12.dp),
|
||||
)
|
||||
}
|
||||
// Chord confirmation (gamepad/TV) — the counterpart to the button changing under a finger.
|
||||
micHint?.let { MicChordHint(it, Modifier.align(Alignment.TopCenter).padding(top = 16.dp)) }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach the Java echo-canceller + noise-suppressor pair to the mic stream's audio session — the
|
||||
* backstop for HALs whose VoiceCommunication capture path doesn't cancel on its own (the native
|
||||
* side already opened the stream under that preset). [sessionId] `<= 0` means native allocated no
|
||||
* session (echo cancellation off, or the preset fell back to the plain open), so there is nothing
|
||||
* to hang an effect on. Created effects land in [into] for [releaseMicEffects]; `create()`
|
||||
* returning null (unsupported / claimed) is quietly nothing — the HAL preset still does its part.
|
||||
* Needs no extra permission: the effect APIs attach to our own recording session.
|
||||
*/
|
||||
private fun attachMicEffects(sessionId: Int, into: MutableList<AudioEffect>) {
|
||||
if (sessionId <= 0) return
|
||||
if (AcousticEchoCanceler.isAvailable()) {
|
||||
AcousticEchoCanceler.create(sessionId)?.let { it.setEnabled(true); into.add(it) }
|
||||
}
|
||||
if (NoiseSuppressor.isAvailable()) {
|
||||
NoiseSuppressor.create(sessionId)?.let { it.setEnabled(true); into.add(it) }
|
||||
}
|
||||
}
|
||||
|
||||
/** Release every attached mic effect engine. Idempotent — the list is cleared, and both stop
|
||||
* paths (surface teardown, final dispose) may call it in either order. */
|
||||
private fun releaseMicEffects(effects: MutableList<AudioEffect>) {
|
||||
effects.forEach { runCatching { it.release() } }
|
||||
effects.clear()
|
||||
}
|
||||
|
||||
/**
|
||||
* The in-stream mic control and its muted indicator, in one element: a dim mic glyph while the
|
||||
* uplink is live, a red **Muted** badge while it isn't — so the state that matters is the loud one,
|
||||
* readable at couch distance and impossible to mistake for the stream's own picture.
|
||||
*
|
||||
* [onToggle] `null` makes it a pure indicator (the TV/gamepad surface, where the Select + Y chord
|
||||
* is the control); non-null makes the badge itself the touch target. Rendering it at all is the
|
||||
* caller's decision — it means a capture is genuinely running.
|
||||
*/
|
||||
@Composable
|
||||
private fun MicMuteControl(muted: Boolean, onToggle: (() -> Unit)?, modifier: Modifier = Modifier) {
|
||||
val shape = RoundedCornerShape(10.dp)
|
||||
Row(
|
||||
modifier = modifier
|
||||
.clip(shape)
|
||||
.background(if (muted) Color(0xE0B3261E) else Color.Black.copy(alpha = 0.45f))
|
||||
.then(if (onToggle != null) Modifier.clickable(onClick = onToggle) else Modifier)
|
||||
.padding(horizontal = 12.dp, vertical = 10.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
Icon(
|
||||
imageVector = if (muted) Icons.Filled.MicOff else Icons.Filled.Mic,
|
||||
// Spoken state first, then the action — a talkback user needs to know they are muted
|
||||
// before they need to know how to stop being muted.
|
||||
contentDescription = if (muted) {
|
||||
"Microphone muted. Activate to unmute."
|
||||
} else {
|
||||
"Microphone live. Activate to mute."
|
||||
},
|
||||
tint = Color.White,
|
||||
modifier = Modifier.size(20.dp),
|
||||
)
|
||||
if (muted) {
|
||||
Spacer(Modifier.width(6.dp))
|
||||
Text("Muted", color = Color.White, fontSize = 14.sp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transient confirmation that the mic chord (Select + Y) registered. The badge above already says
|
||||
* *muted*, but nothing on screen says *un*muted — and "did that press do anything?" is the whole
|
||||
* doubt a chord with no button under the finger creates. Same pill vocabulary as the other
|
||||
* in-stream cues; the caller clears it after a beat.
|
||||
*/
|
||||
@Composable
|
||||
private fun MicChordHint(text: String, modifier: Modifier = Modifier) {
|
||||
Text(
|
||||
text,
|
||||
modifier = modifier
|
||||
.background(Color.Black.copy(alpha = 0.55f), RoundedCornerShape(8.dp))
|
||||
.padding(horizontal = 14.dp, vertical = 8.dp),
|
||||
color = Color.White,
|
||||
fontSize = 15.sp,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* The "hold to quit" cue shown while the gamepad exit chord (Select + Start + L1 + R1) is held. The
|
||||
* chord no longer quits on a quick press — the router debounces it on a ~1 s hold — so this confirms
|
||||
|
||||
@@ -105,8 +105,20 @@ internal suspend fun PointerInputScope.streamTouchPassthrough(handle: Long, styl
|
||||
NativeBridge.nativeSendTouch(handle, it, 2, 0, 0, sw, sh)
|
||||
}
|
||||
c.positionChanged() ->
|
||||
ids[c.id]?.let {
|
||||
NativeBridge.nativeSendTouch(handle, it, 1, x, y, sw, sh)
|
||||
ids[c.id]?.let { id ->
|
||||
// Batched MotionEvents coalesce intermediate points into the
|
||||
// historical list — forward them in order so a fast swipe keeps
|
||||
// its real curvature on the host (usually empty during a stream:
|
||||
// unbuffered dispatch is requested, so this costs nothing).
|
||||
for (hs in c.historical) {
|
||||
NativeBridge.nativeSendTouch(
|
||||
handle, id, 1,
|
||||
hs.position.x.roundToInt().coerceIn(0, sw - 1),
|
||||
hs.position.y.roundToInt().coerceIn(0, sh - 1),
|
||||
sw, sh,
|
||||
)
|
||||
}
|
||||
NativeBridge.nativeSendTouch(handle, id, 1, x, y, sw, sh)
|
||||
}
|
||||
}
|
||||
c.consume()
|
||||
@@ -289,7 +301,10 @@ internal suspend fun PointerInputScope.streamTouchInput(
|
||||
accY -= outY
|
||||
}
|
||||
} else {
|
||||
moveAbs(p.position.x, p.position.y) // direct: cursor follows the finger
|
||||
// Direct: cursor follows the finger — historical points first (batched
|
||||
// MotionEvent samples), so the host cursor traces the finger's real path.
|
||||
for (hs in p.historical) moveAbs(hs.position.x, hs.position.y)
|
||||
moveAbs(p.position.x, p.position.y)
|
||||
}
|
||||
}
|
||||
ev.changes.forEach { it.consume() }
|
||||
|
||||
@@ -65,6 +65,16 @@ class GamepadRouter(context: Context, private val handle: Long, private val sett
|
||||
*/
|
||||
var onExitArmed: ((armed: Boolean) -> Unit)? = null
|
||||
|
||||
/**
|
||||
* Invoked (main thread) each time the mic-mute chord ([MIC_CHORD], Select + Y) is COMPLETED on
|
||||
* a pad — the couch equivalent of the stream's on-screen mute button, which a gamepad user
|
||||
* cannot reach. `StreamScreen` wires it to the mute toggle. Unlike the exit chord this fires
|
||||
* immediately: muting is the kind of thing you want to have already happened, and the on-screen
|
||||
* indicator makes an accidental toggle self-evident. The buttons still go to the host — the
|
||||
* chord adds a meaning to them rather than swallowing them, exactly as the exit chord does.
|
||||
*/
|
||||
var onMicChord: (() -> Unit)? = null
|
||||
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
/** The pending exit-chord hold timer, or null when the chord isn't currently armed. */
|
||||
private var pendingExit: Runnable? = null
|
||||
@@ -108,14 +118,23 @@ class GamepadRouter(context: Context, private val handle: Long, private val sett
|
||||
|
||||
/**
|
||||
* One button transition on [slot] — the shared body behind [onButton] and an [ExternalPad]'s
|
||||
* transitions: forward the wire event, track held state, and arm/disarm the exit chord.
|
||||
* transitions: forward the wire event, track held state, arm/disarm the exit chord, and fire
|
||||
* the mic-mute chord ([MIC_CHORD]).
|
||||
*/
|
||||
private fun slotButton(slot: Slot, bit: Int, down: Boolean, send: Boolean) {
|
||||
if (down) {
|
||||
if (send) NativeBridge.nativeSendGamepadButton(handle, bit, true, slot.index)
|
||||
val wasHeld = slot.held
|
||||
slot.held = slot.held or bit
|
||||
// Full chord now held on this pad → start the hold countdown (idempotent while held).
|
||||
if (slot.held and EXIT_CHORD == EXIT_CHORD) armExit()
|
||||
// Mic mute, edge-triggered on the button that COMPLETES the chord: a genuine press
|
||||
// (`wasHeld` lacks the bit, so an auto-repeat DOWN can't re-fire it) of a chord member
|
||||
// that leaves the whole chord held. Any other button pressed while Select + Y are down
|
||||
// fails the middle test, so the toggle happens once per chord, not once per press.
|
||||
if (wasHeld and bit == 0 && bit and MIC_CHORD != 0 && slot.held and MIC_CHORD == MIC_CHORD) {
|
||||
onMicChord?.invoke()
|
||||
}
|
||||
} else {
|
||||
if (send) NativeBridge.nativeSendGamepadButton(handle, bit, false, slot.index)
|
||||
slot.held = slot.held and bit.inv()
|
||||
@@ -351,6 +370,14 @@ class GamepadRouter(context: Context, private val handle: Long, private val sett
|
||||
*/
|
||||
const val EXIT_HOLD_MS = 1000L
|
||||
|
||||
/**
|
||||
* Mic-mute chord: Select + Y. Y is deliberately NOT one of [EXIT_CHORD]'s buttons, so no
|
||||
* way of reaching the exit chord can pass through this one on the way (and vice versa) —
|
||||
* and Select is a menu button rather than a twitch action, which makes the pair unlikely
|
||||
* to occur inside real play.
|
||||
*/
|
||||
const val MIC_CHORD = Gamepad.BTN_BACK or Gamepad.BTN_Y
|
||||
|
||||
/** Synthetic slot-key base for [ExternalPad]s — below every real (positive) InputDevice id. */
|
||||
const val EXTERNAL_ID_BASE = -1000
|
||||
}
|
||||
|
||||
@@ -248,11 +248,12 @@ object NativeBridge {
|
||||
|
||||
/**
|
||||
* Drain ~1 s of live decode stats for the on-stream HUD, or `null` when no decode thread runs.
|
||||
* Returns 30 doubles (unified stats spec, `design/stats-unification.md`):
|
||||
* Returns 33 doubles (unified stats spec, `design/stats-unification.md`):
|
||||
* `[fps, mbps, e2eP50Ms, e2eP95Ms, latValid, skewCorrected, width, height, refreshHz, framesLost,
|
||||
* bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc, hostNetP50Ms, decodeP50Ms, hostP50Ms,
|
||||
* netP50Ms, lostWindow, skippedWindow, fecWindow, framesWindow, dispValid, displayP50Ms,
|
||||
* e2eDispP50Ms, e2eDispP95Ms, paceP50Ms, latchP50Ms, presentsWindow, presenterActive]`
|
||||
* e2eDispP50Ms, e2eDispP95Ms, paceP50Ms, latchP50Ms, presentsWindow, presenterActive,
|
||||
* feedP50Ms, codecP50Ms, skippedOverflowWindow]`
|
||||
* (the flags are 1.0/0.0; indexes 2/3 are the end-to-end capture→decoded headline; 10–13
|
||||
* describe the negotiated video feed — bit depth 8/10, CICP primaries/transfer, and the HEVC
|
||||
* chroma_format_idc 1=4:2:0 / 3=4:4:4; 14/15 are the stage p50s tiling the headline —
|
||||
@@ -263,7 +264,12 @@ object NativeBridge {
|
||||
* `display` stage from the OnFrameRendered render timestamps — when `dispValid` is 1.0 the
|
||||
* headline becomes the directly-measured capture→displayed pair at 24/25, tiled by
|
||||
* `host+network` + `decode` + `display` (23), and when 0.0 the HUD falls back to the
|
||||
* capture→decoded headline at 2/3 without the `display` term).
|
||||
* capture→decoded headline at 2/3 without the `display` term; 26–29 split the `display`
|
||||
* term the timeline presenter owns — `pace` = decoded→release, `latch` = release→displayed,
|
||||
* the window's on-glass confirm count, and whether the presenter is active at all; 30/31
|
||||
* split `decode` (15) the same way — `feed` = received→queued (hand-off + input-slot wait),
|
||||
* `codec` = queued→decoded, the decoder's own time; 32 is the parked-AU overflow subset of
|
||||
* `skipped` (19), i.e. the decoder falling behind rather than benign newest-wins pacing).
|
||||
* Poll ~1 Hz; each call resets the measurement window.
|
||||
*/
|
||||
external fun nativeVideoStats(handle: Long): DoubleArray?
|
||||
@@ -288,15 +294,53 @@ object NativeBridge {
|
||||
external fun nativeStopAudio(handle: Long)
|
||||
|
||||
/**
|
||||
* Start mic uplink: AAudio input → Opus (48 kHz stereo, 20 ms) → host (`send_mic` / 0xCB), all in
|
||||
* Rust. No-op if already running. The caller MUST hold RECORD_AUDIO; otherwise the AAudio input
|
||||
* stream fails to open and the rest of the session keeps streaming.
|
||||
* Start mic uplink: AAudio input → Opus (48 kHz mono, 10 ms) → host (`send_mic` / 0xCB), all in
|
||||
* Rust. [echoCancel] opens the capture under the VoiceCommunication preset (the HAL's own echo
|
||||
* canceller / noise suppressor) and allocates an audio session id; the return value is that id
|
||||
* (`> 0`) so the caller can attach the Java [android.media.audiofx.AcousticEchoCanceler] /
|
||||
* [android.media.audiofx.NoiseSuppressor] as a backstop — `0` when none was allocated
|
||||
* (echoCancel off, the device refused the preset and the open fell back to the plain path, or
|
||||
* the mic failed entirely). No-op if already running (returns the running capture's id). The
|
||||
* caller MUST hold RECORD_AUDIO; otherwise the AAudio input stream fails to open and the rest
|
||||
* of the session keeps streaming.
|
||||
*/
|
||||
external fun nativeStartMic(handle: Long)
|
||||
external fun nativeStartMic(handle: Long, echoCancel: Boolean): Int
|
||||
|
||||
/** Stop + join the mic thread and close the AAudio input stream. No-op on `0`. */
|
||||
/**
|
||||
* Stop + join the mic thread and close the AAudio input stream. No-op on `0`. Leaves the
|
||||
* session's mute state ([nativeSetMicMuted]) alone — a surface recreate stops and restarts the
|
||||
* mic, and a user who muted must stay muted through it.
|
||||
*/
|
||||
external fun nativeStopMic(handle: Long)
|
||||
|
||||
/**
|
||||
* Mute/unmute the mic uplink mid-stream. Muting does NOT stop the capture: the AAudio input
|
||||
* stream, the input preset it settled on and its primed buffers stay as they are, and the
|
||||
* encode loop drops each 10 ms frame instead of encoding + sending it — so room audio is never
|
||||
* encoded and nothing goes on the wire, while a toggle costs an atomic store and takes effect
|
||||
* on the next 10 ms boundary (a stop/start would re-run the preset fallback ladder and re-prime
|
||||
* buffers every time).
|
||||
*
|
||||
* Sticky for the SESSION — the flag lives on the handle, not on the capture — so the mic
|
||||
* restart a surface recreate performs comes back muted, with no window for an unmuted frame to
|
||||
* escape; a fresh session always starts unmuted. Nothing here is persisted. No-op on `0`.
|
||||
* Cheap (one atomic store); UI-safe.
|
||||
*
|
||||
* One honest consequence of keeping the stream open: the platform's own recording indicator
|
||||
* stays lit while muted, because the mic really is still open. What stops is the encode and the
|
||||
* send — no captured audio leaves the process.
|
||||
*/
|
||||
external fun nativeSetMicMuted(handle: Long, muted: Boolean)
|
||||
|
||||
/**
|
||||
* Is a mic capture actually RUNNING — i.e. did [nativeStartMic] open a stream, and has
|
||||
* [nativeStopMic] not been called since? Offer the in-stream mute control on THIS rather than
|
||||
* on the user's setting: a device that refused every AAudio input rung (or a missing
|
||||
* RECORD_AUDIO grant) then shows no control instead of a lie about a mic being heard. `false`
|
||||
* on a `0` handle. Cheap; UI-safe.
|
||||
*/
|
||||
external fun nativeMicActive(handle: Long): Boolean
|
||||
|
||||
// ---- Input: Kotlin captures, Rust forwards to the host (send_input) ----
|
||||
|
||||
/** Relative mouse move; dx/dy are device-pixel deltas (screen +y down). */
|
||||
|
||||
@@ -100,6 +100,34 @@ object VideoDecoders {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One-line per-mime probe readout for the connect log (`adb logcat -s pf.caps`): which
|
||||
* decoder each advertised mime resolves to and whether it declares `FEATURE_PartialFrame` —
|
||||
* the P2 slice-pipeline gate that is otherwise invisible until a stream behaves differently.
|
||||
*/
|
||||
fun capsReport(): String {
|
||||
val mimes = buildList {
|
||||
add("video/avc")
|
||||
add("video/hevc")
|
||||
if (decodableCodecBits() and 4 != 0) add("video/av01")
|
||||
}
|
||||
val infos = runCatching { MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos }
|
||||
.getOrNull() ?: return "codec list unavailable"
|
||||
return mimes.joinToString(" ") { mime ->
|
||||
val pick = pickDecoder(mime)?.name
|
||||
val partial = pick?.let { p ->
|
||||
infos.firstOrNull { it.name == p }?.let { info ->
|
||||
runCatching {
|
||||
info.getCapabilitiesForType(mime)
|
||||
.isFeatureSupported(CodecCapabilities.FEATURE_PartialFrame)
|
||||
}.getOrNull()
|
||||
}
|
||||
}
|
||||
"${mime.removePrefix("video/")}=${pick ?: "platform-default"}" +
|
||||
" partialFrame=${partial ?: "?"}"
|
||||
}
|
||||
}
|
||||
|
||||
fun pickDecoder(mime: String): DecoderChoice? {
|
||||
if (mime.isEmpty()) return null
|
||||
val infos = runCatching { MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos }
|
||||
|
||||
@@ -16,7 +16,7 @@ use std::time::{Duration, Instant};
|
||||
use super::display::{
|
||||
apply_hdr_dataspace, install_render_callback, release_render_callback, DisplayTracker,
|
||||
};
|
||||
use super::latency::{note_decoded_pts, now_realtime_ns, take_flags};
|
||||
use super::latency::{note_decoded_pts, now_realtime_ns, take_flags, take_stamp};
|
||||
use super::presenter::{presenter_disabled_by_sysprop, PresentMeter, PresentPriority, Presenter};
|
||||
use super::setup::{
|
||||
android_hdr_static_info, boost_hot_threads, boost_thread_priority, codec_mime,
|
||||
@@ -280,6 +280,10 @@ pub(super) fn run_async(
|
||||
let mut oversized_dropped: u64 = 0;
|
||||
// Slice-progressive continuity ledger (see `PartFeed`).
|
||||
let mut part_open: Option<PartFeed> = None;
|
||||
// Queued-instant stamps (pts → realtime ns at the AU's LAST piece entering the codec) — the
|
||||
// P3 decode-split ledger: `feed` = received→queued, `codec` = queued→decoded. Always on
|
||||
// (one vDSO clock read per AU); consumed by `present_ready`.
|
||||
let mut queued_stamps: VecDeque<(u64, i128)> = VecDeque::new();
|
||||
// Freeze-until-reanchor gate (see the sync loop for the rationale). Armed on a frame-index gap
|
||||
// (the feeder's Au verdict), a parked-AU overflow drop, a dropped-count climb, or a recoverable
|
||||
// codec error; `recovery_flags` carries each AU's user_flags from `dispatch_event` (feed) to
|
||||
@@ -349,7 +353,7 @@ pub(super) fn run_async(
|
||||
p.on_vsync();
|
||||
}
|
||||
}
|
||||
stats.note_skipped(aus_dropped); // parked-AU overflow drops are client-side skips too
|
||||
stats.note_skipped_overflow(aus_dropped); // parked-AU overflow: skips, flagged as such
|
||||
if fmt_dirty {
|
||||
apply_hdr_dataspace(&codec, &window, &mut applied_ds);
|
||||
}
|
||||
@@ -361,6 +365,7 @@ pub(super) fn run_async(
|
||||
&mut fed,
|
||||
&mut oversized_dropped,
|
||||
&mut part_open,
|
||||
&mut queued_stamps,
|
||||
&mut gate,
|
||||
);
|
||||
let had_output = !ready.is_empty();
|
||||
@@ -372,6 +377,8 @@ pub(super) fn run_async(
|
||||
&mut ready,
|
||||
&stats,
|
||||
&in_flight,
|
||||
&mut queued_stamps,
|
||||
&meter,
|
||||
clock_offset.load(Ordering::Relaxed),
|
||||
&tracker,
|
||||
&mut presenter,
|
||||
@@ -762,6 +769,7 @@ pub(super) struct PartFeed {
|
||||
/// [`BUFFER_FLAG_PARTIAL_FRAME`] except the AU's last, all at the AU's pts. `part_open` is the
|
||||
/// continuity ledger — any break (gap, orphan, oversize) abandons the AU per [`PartFeed::pts_us`]'s
|
||||
/// close contract and re-syncs at the next `first`.
|
||||
#[allow(clippy::too_many_arguments)] // one call site; the split ledger threads through like the gate
|
||||
fn feed_ready(
|
||||
codec: &MediaCodec,
|
||||
client: &NativeClient,
|
||||
@@ -770,6 +778,7 @@ fn feed_ready(
|
||||
fed: &mut u64,
|
||||
oversized_dropped: &mut u64,
|
||||
part_open: &mut Option<PartFeed>,
|
||||
queued_stamps: &mut VecDeque<(u64, i128)>,
|
||||
gate: &mut ReanchorGate,
|
||||
) {
|
||||
while !pending_aus.is_empty() && !free_inputs.is_empty() {
|
||||
@@ -859,9 +868,15 @@ fn feed_ready(
|
||||
}
|
||||
} else {
|
||||
// `fed` counts ACCESS UNITS toward the HUD's fed/decoded balance — the closing
|
||||
// piece (or a whole AU) bumps it.
|
||||
// piece (or a whole AU) bumps it. The queued stamp marks the same instant (the AU
|
||||
// is fully in the codec's hands): the P3 decode split measures `codec` from here,
|
||||
// so a slice-progressive head start shows up as codec-pure shrink.
|
||||
if last {
|
||||
*fed += 1;
|
||||
queued_stamps.push_back((pts_us, now_realtime_ns()));
|
||||
if queued_stamps.len() > IN_FLIGHT_CAP {
|
||||
queued_stamps.pop_front(); // stale — codec never echoed it back
|
||||
}
|
||||
}
|
||||
*part_open = if last {
|
||||
None
|
||||
@@ -876,7 +891,8 @@ fn feed_ready(
|
||||
}
|
||||
}
|
||||
|
||||
/// Route the ready outputs toward glass. With the timeline presenter (default): fold each output
|
||||
/// Route the ready outputs toward glass, recording each one's decode-split + e2e first. With the
|
||||
/// timeline presenter (default): fold each output
|
||||
/// through the re-anchor gate in pts order, hand the approved ones to the presenter's store
|
||||
/// (newest-wins / smoothing FIFO — the actual release happens in `Presenter::pump`, budgeted and
|
||||
/// timeline-timed), and release withheld concealment unrendered. Legacy (`arrival` sysprop):
|
||||
@@ -892,6 +908,8 @@ fn present_ready(
|
||||
ready: &mut Vec<OutputReady>,
|
||||
stats: &crate::stats::VideoStats,
|
||||
in_flight: &Mutex<VecDeque<(u64, i128)>>,
|
||||
queued_stamps: &mut VecDeque<(u64, i128)>,
|
||||
meter: &PresentMeter,
|
||||
clock_offset: i64,
|
||||
tracker: &DisplayTracker,
|
||||
presenter: &mut Option<Presenter>,
|
||||
@@ -903,22 +921,42 @@ fn present_ready(
|
||||
if ready.is_empty() {
|
||||
return;
|
||||
}
|
||||
// Pair each output's decode stage (feeds the ABR decode signal always; the HUD histogram only
|
||||
// while visible) — both consume the receipt map, so enter for either.
|
||||
if stats.enabled() || measure_decode {
|
||||
// Pair each output's decode stage (the ABR decode signal + the HUD histogram consume the
|
||||
// receipt map; the P3 split's codec-pure half needs only the queued stamp, so it records
|
||||
// even with both off — that keeps the 1 Hz pf.present mirror HUD-off readable).
|
||||
{
|
||||
let want_stage = stats.enabled() || measure_decode;
|
||||
let mut g = in_flight
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for o in ready.iter() {
|
||||
note_decoded_pts(
|
||||
client,
|
||||
measure_decode,
|
||||
stats,
|
||||
&mut g,
|
||||
clock_offset,
|
||||
o.pts_us,
|
||||
o.decoded_ns,
|
||||
);
|
||||
let received_ns = if want_stage {
|
||||
note_decoded_pts(
|
||||
client,
|
||||
measure_decode,
|
||||
stats,
|
||||
&mut g,
|
||||
clock_offset,
|
||||
o.pts_us,
|
||||
o.decoded_ns,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let queued = take_stamp(queued_stamps, o.pts_us);
|
||||
let codec_us = queued.map(|q| ((o.decoded_ns - q).max(0) / 1000) as u64);
|
||||
let feed_us = match (queued, received_ns) {
|
||||
(Some(q), Some(r)) => Some(((q - r).max(0) / 1000) as u64),
|
||||
_ => None,
|
||||
};
|
||||
// Always-on e2e for the 1 Hz pf.present mirror (same formula + clamp as the HUD's
|
||||
// capture→decoded headline in `note_decoded_pts`).
|
||||
let e2e_ns = o.decoded_ns + clock_offset as i128 - o.pts_us as i128 * 1000;
|
||||
let e2e_us = (e2e_ns > 0 && e2e_ns < 10_000_000_000).then_some((e2e_ns / 1000) as u64);
|
||||
meter.note_decode(feed_us, codec_us, e2e_us);
|
||||
if let Some(c) = codec_us {
|
||||
stats.note_decode_split(feed_us, c);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fold EVERY output through the gate in pts (== decode) order — even the ones newest-wins discards —
|
||||
|
||||
@@ -20,7 +20,8 @@ pub(super) fn now_realtime_ns() -> i128 {
|
||||
/// entries older than it are evicted (decode order == input order here — low-latency, no
|
||||
/// B-frames — so anything before it was dropped inside the codec or stamped before a flush).
|
||||
/// `decoded_ns` is the availability instant: the dequeue (sync loop) or the output callback's
|
||||
/// stamp (async loop).
|
||||
/// stamp (async loop). Returns the receipt stamp it paired (if any) so the caller can split the
|
||||
/// `decode` stage further (feed wait vs codec-pure) without re-walking the map.
|
||||
pub(super) fn note_decoded_pts(
|
||||
client: &NativeClient,
|
||||
measure_decode: bool,
|
||||
@@ -29,7 +30,7 @@ pub(super) fn note_decoded_pts(
|
||||
clock_offset: i64,
|
||||
pts_us: u64,
|
||||
decoded_ns: i128,
|
||||
) {
|
||||
) -> Option<i128> {
|
||||
// Pair the echoed pts back to its receipt stamp, evicting stale (older) entries as we go.
|
||||
let mut received_ns = None;
|
||||
while let Some(&(p, r)) = in_flight.front() {
|
||||
@@ -61,6 +62,24 @@ pub(super) fn note_decoded_pts(
|
||||
let e2e_us = (e2e_ns > 0 && e2e_ns < 10_000_000_000).then_some((e2e_ns / 1000) as u64);
|
||||
stats.note_decoded(e2e_us, decode_us);
|
||||
}
|
||||
received_ns
|
||||
}
|
||||
|
||||
/// The queued-instant stamp for a decoded output, keyed by the echoed `presentationTimeUs` — the
|
||||
/// same monotonic evict-as-you-go pairing as [`take_flags`], over an `(pts_us, realtime_ns)` map
|
||||
/// (the feed side stamps each AU as its last piece enters the codec). A miss returns `None` —
|
||||
/// the split is simply not recorded for that frame.
|
||||
pub(super) fn take_stamp(map: &mut VecDeque<(u64, i128)>, pts_us: u64) -> Option<i128> {
|
||||
while let Some(&(p, t)) = map.front() {
|
||||
if p > pts_us {
|
||||
break; // future frame — leave it for its own output buffer
|
||||
}
|
||||
map.pop_front();
|
||||
if p == pts_us {
|
||||
return Some(t);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The AU `user_flags` for a decoded output, keyed by the echoed `presentationTimeUs`. Recovery
|
||||
|
||||
@@ -126,6 +126,15 @@ pub(super) struct PresentMeter {
|
||||
struct PresentMeterInner {
|
||||
latch_us: Vec<u64>,
|
||||
displays: u64,
|
||||
/// The `decode` stage's feed split, received→queued µs (P3 science: hand-off + input-slot
|
||||
/// wait). Empty when no receipt stamp matched (HUD off and ABR not measuring decode).
|
||||
feed_us: Vec<u64>,
|
||||
/// The codec-pure half, queued→decoded µs, measured from the AU's LAST piece — always on,
|
||||
/// so a HUD-off logcat A/B still reads the decoder's own time.
|
||||
codec_us: Vec<u64>,
|
||||
/// Capture→decoded end-to-end µs (skew-corrected, clamped) — always on for the same reason:
|
||||
/// the wireless A/B's headline without having to reach the on-screen HUD.
|
||||
e2e_us: Vec<u64>,
|
||||
}
|
||||
|
||||
impl PresentMeter {
|
||||
@@ -134,6 +143,9 @@ impl PresentMeter {
|
||||
inner: Mutex::new(PresentMeterInner {
|
||||
latch_us: Vec::with_capacity(256),
|
||||
displays: 0,
|
||||
feed_us: Vec::with_capacity(256),
|
||||
codec_us: Vec::with_capacity(256),
|
||||
e2e_us: Vec::with_capacity(256),
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -152,14 +164,51 @@ impl PresentMeter {
|
||||
}
|
||||
}
|
||||
|
||||
fn drain(&self) -> (Vec<u64>, u64) {
|
||||
/// One decoded frame's always-on measurements: the `decode`-stage split (feed =
|
||||
/// received→queued when a receipt stamp matched; codec = queued→decoded when the queued
|
||||
/// stamp did) and the capture→decoded end-to-end, µs. Decode thread; poison-proof.
|
||||
pub(super) fn note_decode(
|
||||
&self,
|
||||
feed_us: Option<u64>,
|
||||
codec_us: Option<u64>,
|
||||
e2e_us: Option<u64>,
|
||||
) {
|
||||
let mut g = self
|
||||
.inner
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(f) = feed_us {
|
||||
if g.feed_us.len() < 4096 {
|
||||
g.feed_us.push(f);
|
||||
}
|
||||
}
|
||||
if let Some(c) = codec_us {
|
||||
if g.codec_us.len() < 4096 {
|
||||
g.codec_us.push(c);
|
||||
}
|
||||
}
|
||||
if let Some(e) = e2e_us {
|
||||
if g.e2e_us.len() < 4096 {
|
||||
g.e2e_us.push(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::type_complexity)] // one caller unpacks it in place; a struct would be noise
|
||||
fn drain(&self) -> (Vec<u64>, u64, Vec<u64>, Vec<u64>, Vec<u64>) {
|
||||
let mut g = self
|
||||
.inner
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let displays = g.displays;
|
||||
g.displays = 0;
|
||||
(std::mem::take(&mut g.latch_us), displays)
|
||||
(
|
||||
std::mem::take(&mut g.latch_us),
|
||||
displays,
|
||||
std::mem::take(&mut g.feed_us),
|
||||
std::mem::take(&mut g.codec_us),
|
||||
std::mem::take(&mut g.e2e_us),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -386,7 +435,9 @@ impl Presenter {
|
||||
/// `released` (to glass) / `displays` (OnFrameRendered confirms) / `paced` (policy drops) /
|
||||
/// `noBudget` (waits on the closed budget) / `forced` (stale force-opens — 0 when healthy) /
|
||||
/// `qDry` (FIFO underflows) / `pace` (decoded→release) / `latch` (release→displayed) /
|
||||
/// `vsync` (the measured panel period).
|
||||
/// `feed`+`codec` (the decode stage split: received→queued hand-off/slot wait + the
|
||||
/// codec-pure queued→decoded time) / `e2e` (capture→decoded, skew-corrected — the wireless
|
||||
/// A/B headline) / `vsync` (the measured panel period).
|
||||
///
|
||||
/// Returns this window's CIRCULAR latch statistics `(vector-mean latch ns mod panel period,
|
||||
/// coherence ‰)` when a window actually flushed — the phase-lock reporter's v2 error signal
|
||||
@@ -400,11 +451,14 @@ impl Presenter {
|
||||
return None;
|
||||
}
|
||||
self.last_flush = Instant::now();
|
||||
let (latch, displays) = meter.drain();
|
||||
let (latch, displays, feed, codec, e2e) = meter.drain();
|
||||
if self.released == 0 && displays == 0 {
|
||||
return None; // idle stream — nothing worth a line
|
||||
}
|
||||
let (pace_p50, pace_max) = p50_max_ms(std::mem::take(&mut self.pace_us));
|
||||
let (feed_p50, feed_max) = p50_max_ms(feed);
|
||||
let (codec_p50, codec_max) = p50_max_ms(codec);
|
||||
let (e2e_p50, e2e_max) = p50_max_ms(e2e);
|
||||
let circ = clock.and_then(|c| {
|
||||
punktfunk_core::phase::circular_latch(&latch, c.panel_period_ns().max(c.period_ns()))
|
||||
});
|
||||
@@ -416,7 +470,9 @@ impl Presenter {
|
||||
log::info!(
|
||||
target: "pf.present",
|
||||
"released={} displays={} paced={} noBudget={} forced={} qDry={} \
|
||||
paceMs p50={:.2} max={:.2} latchMs p50={:.2} max={:.2} circ={:.2}ms coh={} \
|
||||
paceMs p50={:.2} max={:.2} latchMs p50={:.2} max={:.2} \
|
||||
feedMs p50={:.2} max={:.2} codecMs p50={:.2} max={:.2} \
|
||||
e2eMs p50={:.2} max={:.2} circ={:.2}ms coh={} \
|
||||
vsyncMs={:.2} panelMs={:.2}",
|
||||
self.released,
|
||||
displays,
|
||||
@@ -428,6 +484,12 @@ impl Presenter {
|
||||
pace_max,
|
||||
latch_p50,
|
||||
latch_max,
|
||||
feed_p50,
|
||||
feed_max,
|
||||
codec_p50,
|
||||
codec_max,
|
||||
e2e_p50,
|
||||
e2e_max,
|
||||
circ.map(|(m, _)| m as f64 / 1e6).unwrap_or(0.0),
|
||||
circ.map(|(_, c)| c).unwrap_or(0),
|
||||
period_ms,
|
||||
|
||||
@@ -1,16 +1,22 @@
|
||||
//! Android microphone uplink (android-only): capture mic PCM via AAudio (LowLatency **input**),
|
||||
//! Opus-encode 20 ms stereo frames, and push them to the host over the connector's mic plane
|
||||
//! Opus-encode 10 ms mono frames, and push them to the host over the connector's mic plane
|
||||
//! (`send_mic` → 0xCB datagram). The mirror of [`crate::audio`] in reverse: AAudio's realtime input
|
||||
//! callback hands captured interleaved f32 to a channel; a worker thread we own does the Opus
|
||||
//! encode + send (encoding is too heavy for the realtime callback, exactly as decode is on the
|
||||
//! playback side). Like the playback path, the realtime callback is allocation-free: captured
|
||||
//! bursts are copied into pre-allocated buffers from a recycle free-list (pool empty = drop the
|
||||
//! chunk, never allocate on the capture thread). Format matches the host decoder + the Linux
|
||||
//! client: 48 kHz **stereo**, 20 ms, Opus VOIP.
|
||||
//! callback hands captured f32 to a channel; a worker thread we own does the Opus encode + send
|
||||
//! (encoding is too heavy for the realtime callback, exactly as decode is on the playback side).
|
||||
//! Like the playback path, the realtime callback is allocation-free: captured bursts are copied
|
||||
//! into pre-allocated buffers from a recycle free-list (pool empty = drop the chunk, never
|
||||
//! allocate on the capture thread). Format: 48 kHz **mono**, 10 ms, Opus VOIP with in-band FEC —
|
||||
//! the host decodes any Opus frame ≤ 120 ms with its stereo decoder (mono packets upmix), so this
|
||||
//! needs no protocol change; speech gains nothing from stereo, and the shorter frame shaves a
|
||||
//! buffering interval off the uplink.
|
||||
//!
|
||||
//! **Mute** is a flag the encode loop reads per 10 ms frame, never a stream teardown: the AAudio
|
||||
//! input stream, the input-preset ladder it settled on and its primed buffers all survive a
|
||||
//! mute/unmute untouched, so toggling costs an atomic load and nothing else.
|
||||
|
||||
use ndk::audio::{
|
||||
AudioCallbackResult, AudioDirection, AudioFormat, AudioPerformanceMode, AudioSharingMode,
|
||||
AudioStream, AudioStreamBuilder,
|
||||
AudioCallbackResult, AudioDirection, AudioFormat, AudioInputPreset, AudioPerformanceMode,
|
||||
AudioSharingMode, AudioStream, AudioStreamBuilder, SessionId,
|
||||
};
|
||||
use punktfunk_core::client::NativeClient;
|
||||
use std::collections::VecDeque;
|
||||
@@ -20,31 +26,57 @@ use std::sync::mpsc::{sync_channel, Receiver, RecvTimeoutError, SyncSender, TryS
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
const CHANNELS: usize = 2;
|
||||
const CHANNELS: usize = 1;
|
||||
const SAMPLE_RATE: i32 = 48_000;
|
||||
/// 20 ms per channel @ 48 kHz — the Linux client's frame; the host accepts ≤ 120 ms.
|
||||
const FRAME_SAMPLES: usize = 960;
|
||||
/// 10 ms per channel @ 48 kHz — half the desktop clients' 20 ms frame, trading a little Opus
|
||||
/// header overhead for one less buffered interval; the host accepts ≤ 120 ms.
|
||||
const FRAME_SAMPLES: usize = 480;
|
||||
/// Captured-chunk hand-off depth (each ~ one burst); drops on overflow (best-effort uplink).
|
||||
/// Bursts are sized in frames, so the wall-time depth is unchanged by the stereo→mono move.
|
||||
const RING_CHUNKS: usize = 64;
|
||||
/// Free-list buffer capacity, in interleaved f32 samples: comfortably above a LowLatency input
|
||||
/// burst (typically ≤ ~480 frames). A device with larger bursts costs each buffer a one-time grow
|
||||
/// on the capture thread, after which the steady state is allocation-free again.
|
||||
const CHUNK_CAP_SAMPLES: usize = 1920; // 20 ms stereo
|
||||
/// Opus VOIP target bitrate (speech; tunable).
|
||||
const MIC_BITRATE: i32 = 64_000;
|
||||
/// burst (typically ≤ ~480 frames — mono, so samples = frames). A device with larger bursts costs
|
||||
/// each buffer a one-time grow on the capture thread, after which the steady state is
|
||||
/// allocation-free again.
|
||||
const CHUNK_CAP_SAMPLES: usize = 960; // 20 ms mono — the same wall-time as the old stereo value
|
||||
/// Opus VOIP target bitrate (mono speech; tunable).
|
||||
const MIC_BITRATE: i32 = 48_000;
|
||||
/// Encode-side self-heal threshold, in queued 10 ms frames (~60 ms): waking to more than this
|
||||
/// means the uplink stalled — and because the capture callback drops the NEWEST chunk when the
|
||||
/// channel is full, a stall otherwise converts to standing mic delay that never drains (real-time
|
||||
/// playback host-side never makes time back up). Skip to the newest few frames instead.
|
||||
const BACKLOG_MAX_FRAMES: usize = 6;
|
||||
/// What a self-heal keeps: ~20 ms of the freshest audio (one audible blip, live again).
|
||||
const BACKLOG_KEEP_FRAMES: usize = 2;
|
||||
|
||||
/// Owned by [`crate::session::SessionHandle`]: the live AAudio input stream + the encode thread.
|
||||
pub struct MicCapture {
|
||||
_stream: AudioStream, // dropping it stops + closes the AAudio input stream
|
||||
/// The audio-session id AAudio allocated (`> 0`) when echo cancellation asked for one — the
|
||||
/// hook Kotlin hangs the Java `AcousticEchoCanceler`/`NoiseSuppressor` on. `0` = none.
|
||||
session_id: i32,
|
||||
shutdown: Arc<AtomicBool>,
|
||||
join: Option<std::thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl MicCapture {
|
||||
/// Open AAudio (LowLatency, 48 kHz/stereo/f32) for **input** with a realtime callback that
|
||||
/// forwards captured PCM to a channel, then spawn the Opus encode + uplink thread. `None` on
|
||||
/// failure (the caller leaves the rest of the session streaming).
|
||||
pub fn start(client: Arc<NativeClient>) -> Option<MicCapture> {
|
||||
/// Open AAudio (LowLatency, 48 kHz/mono/f32) for **input** with a realtime callback that
|
||||
/// forwards captured PCM to a channel, then spawn the Opus encode + uplink thread. With
|
||||
/// `echo_cancel` the stream opens under the `VoiceCommunication` input preset — the HAL's own
|
||||
/// echo canceller / noise suppressor on the capture path (the default `VoiceRecognition`
|
||||
/// preset deliberately bypasses them, which is why the host used to hear its own stream back
|
||||
/// from a speaker-playing phone) — and allocates an audio session id for Kotlin's Java-effect
|
||||
/// backstop. `None` on failure (the caller leaves the rest of the session streaming).
|
||||
///
|
||||
/// `muted` is the SESSION's live mic-mute flag (owned by `SessionHandle`, not by this capture),
|
||||
/// honoured per frame by [`encode_loop`]. Sharing it rather than owning it is what makes mute
|
||||
/// survive the mic stop/start a surface recreate performs — and means a capture started while
|
||||
/// muted never encodes its first frame, so there is no window for one to escape.
|
||||
pub fn start(
|
||||
client: Arc<NativeClient>,
|
||||
echo_cancel: bool,
|
||||
muted: Arc<AtomicBool>,
|
||||
) -> Option<MicCapture> {
|
||||
let captured = Arc::new(AtomicU64::new(0));
|
||||
// Chunks discarded on the capture thread (free-list empty / encoder lagging); logged
|
||||
// throttled from the encode worker.
|
||||
@@ -52,7 +84,9 @@ impl MicCapture {
|
||||
|
||||
// One open attempt at a given sharing mode (same pattern as [`crate::audio`]: `open_stream`
|
||||
// consumes the builder AND the callback, so each try rebuilds the channels it captures).
|
||||
let try_open = |sharing: AudioSharingMode| -> ndk::audio::Result<(
|
||||
let try_open = |sharing: AudioSharingMode,
|
||||
voice: bool|
|
||||
-> ndk::audio::Result<(
|
||||
AudioStream,
|
||||
Receiver<Vec<f32>>,
|
||||
SyncSender<Vec<f32>>,
|
||||
@@ -99,13 +133,25 @@ impl MicCapture {
|
||||
AudioCallbackResult::Continue
|
||||
};
|
||||
|
||||
let stream = AudioStreamBuilder::new()?
|
||||
// NOTE: no `.frames_per_data_callback(...)`: AAudio's own docs call leaving it unset
|
||||
// the lowest-latency path (the callback then runs at the device's optimal burst,
|
||||
// while pinning a size inserts an adaptation buffer), and the encode side re-chunks
|
||||
// to 10 ms frames regardless of how the bursts arrive.
|
||||
let mut builder = AudioStreamBuilder::new()?
|
||||
.direction(AudioDirection::Input)
|
||||
.sample_rate(SAMPLE_RATE)
|
||||
.channel_count(CHANNELS as i32)
|
||||
.format(AudioFormat::PCM_Float)
|
||||
.performance_mode(AudioPerformanceMode::LowLatency)
|
||||
.sharing_mode(sharing)
|
||||
.sharing_mode(sharing);
|
||||
if voice {
|
||||
// VoiceCommunication routes the capture through the HAL's AEC/NS; the allocated
|
||||
// session id (`None` = allocate) is what Kotlin attaches the Java effects to.
|
||||
builder = builder
|
||||
.input_preset(AudioInputPreset::VoiceCommunication)
|
||||
.session_id(None);
|
||||
}
|
||||
let stream = builder
|
||||
.data_callback(Box::new(callback))
|
||||
.error_callback(Box::new(|_s, e| {
|
||||
log::warn!("mic: AAudio error (device reroute/disconnect?): {e:?}");
|
||||
@@ -114,21 +160,52 @@ impl MicCapture {
|
||||
Ok((stream, rx, free_tx))
|
||||
};
|
||||
|
||||
// Exclusive first — MMAP-exclusive is AAudio's lowest-latency path — falling back to Shared
|
||||
// when the device refuses (no MMAP, mic claimed, …). The started-log below prints the mode
|
||||
// the device actually GRANTED (`share=`).
|
||||
let (stream, rx, free_tx) = match try_open(AudioSharingMode::Exclusive) {
|
||||
Ok(opened) => opened,
|
||||
Err(e) => {
|
||||
log::info!("mic: Exclusive open failed ({e}) — retrying Shared");
|
||||
match try_open(AudioSharingMode::Shared) {
|
||||
Ok(opened) => opened,
|
||||
Err(e) => {
|
||||
log::error!("mic: open_stream (RECORD_AUDIO granted?): {e}");
|
||||
return None;
|
||||
}
|
||||
// Exclusive first — MMAP-exclusive is AAudio's lowest-latency path — falling back to
|
||||
// Shared when the device refuses (no MMAP, mic claimed, …); and each sharing mode with
|
||||
// the voice preset before without it, because some HALs reject VoiceCommunication (or a
|
||||
// session id) outright and a mic without echo cancellation still beats no mic. The
|
||||
// ladder's last rungs are exactly the preset-less open this always did. The started-log
|
||||
// below prints what the device actually GRANTED (`share=`/`session=`).
|
||||
let attempts: &[(AudioSharingMode, bool)] = if echo_cancel {
|
||||
&[
|
||||
(AudioSharingMode::Exclusive, true),
|
||||
(AudioSharingMode::Shared, true),
|
||||
(AudioSharingMode::Exclusive, false),
|
||||
(AudioSharingMode::Shared, false),
|
||||
]
|
||||
} else {
|
||||
&[
|
||||
(AudioSharingMode::Exclusive, false),
|
||||
(AudioSharingMode::Shared, false),
|
||||
]
|
||||
};
|
||||
let mut opened = None;
|
||||
for &(sharing, voice) in attempts {
|
||||
match try_open(sharing, voice) {
|
||||
Ok(o) => {
|
||||
opened = Some(o);
|
||||
break;
|
||||
}
|
||||
Err(e) => log::info!(
|
||||
"mic: open {sharing:?}{} failed ({e}) — trying the next fallback",
|
||||
if voice { "+VoiceCommunication" } else { "" },
|
||||
),
|
||||
}
|
||||
}
|
||||
let (stream, rx, free_tx) = match opened {
|
||||
Some(o) => o,
|
||||
None => {
|
||||
log::error!("mic: open_stream (RECORD_AUDIO granted?): every mode refused");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
// The session id AAudio actually allocated (only a voice rung asks for one): `> 0` is the
|
||||
// handle Kotlin hangs the Java AcousticEchoCanceler/NoiseSuppressor off as the HAL
|
||||
// preset's backstop; `0` = none, nothing to attach.
|
||||
let session_id = match stream.session_id() {
|
||||
SessionId::Allocated(id) => id.get(),
|
||||
SessionId::None => 0,
|
||||
};
|
||||
|
||||
if let Err(e) = stream.request_start() {
|
||||
@@ -136,7 +213,7 @@ impl MicCapture {
|
||||
return None;
|
||||
}
|
||||
log::info!(
|
||||
"mic: AAudio input started rate={} ch={} fmt={:?} share={:?}",
|
||||
"mic: AAudio input started rate={} ch={} fmt={:?} share={:?} session={session_id}",
|
||||
stream.sample_rate(),
|
||||
stream.channel_count(),
|
||||
stream.format(),
|
||||
@@ -147,15 +224,21 @@ impl MicCapture {
|
||||
let sd = shutdown.clone();
|
||||
let join = std::thread::Builder::new()
|
||||
.name("pf-mic".into())
|
||||
.spawn(move || encode_loop(client, rx, free_tx, sd, captured, dropped))
|
||||
.spawn(move || encode_loop(client, rx, free_tx, sd, muted, captured, dropped))
|
||||
.ok();
|
||||
|
||||
Some(MicCapture {
|
||||
_stream: stream,
|
||||
session_id,
|
||||
shutdown,
|
||||
join,
|
||||
})
|
||||
}
|
||||
|
||||
/// The audio-session id AAudio allocated (`> 0`; see [`MicCapture::start`]), `0` = none.
|
||||
pub fn session_id(&self) -> i32 {
|
||||
self.session_id
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MicCapture {
|
||||
@@ -168,20 +251,29 @@ impl Drop for MicCapture {
|
||||
}
|
||||
}
|
||||
|
||||
/// Consumer: drain captured f32 → accumulate → Opus `encode_float` 20 ms stereo frames → `send_mic`.
|
||||
/// Consumer: drain captured f32 → accumulate → Opus `encode_float` 10 ms mono frames → `send_mic`.
|
||||
/// Drained chunk buffers go back to the callback's free-list; the encode scratch is reused across
|
||||
/// frames (only the packet Vec handed to `send_mic` is allocated per frame — it's sent away owned).
|
||||
///
|
||||
/// While `muted` is set a formed frame is dropped instead of encoded (see the frame loop) — the
|
||||
/// capture side keeps running exactly as it does unmuted, so nothing about the stream, its ring or
|
||||
/// its backlog behaviour changes across a toggle.
|
||||
fn encode_loop(
|
||||
client: Arc<NativeClient>,
|
||||
rx: Receiver<Vec<f32>>,
|
||||
free_tx: SyncSender<Vec<f32>>,
|
||||
shutdown: Arc<AtomicBool>,
|
||||
muted: Arc<AtomicBool>,
|
||||
captured: Arc<AtomicU64>,
|
||||
dropped: Arc<AtomicU64>,
|
||||
) {
|
||||
// Fold this Opus-encode/uplink thread into the client's hot-thread set so the ADPF session the
|
||||
// decode thread opens keeps mic encode on a fast core too (the playback side's decode_loop
|
||||
// does the same). No-op below API 33.
|
||||
client.register_hot_thread();
|
||||
let mut enc = match opus::Encoder::new(
|
||||
SAMPLE_RATE as u32,
|
||||
opus::Channels::Stereo,
|
||||
opus::Channels::Mono,
|
||||
opus::Application::Voip,
|
||||
) {
|
||||
Ok(e) => e,
|
||||
@@ -191,13 +283,21 @@ fn encode_loop(
|
||||
}
|
||||
};
|
||||
let _ = enc.set_bitrate(opus::Bitrate::Bits(MIC_BITRATE));
|
||||
// Speech tuning: complexity 5 roughly halves encode cost for no audible loss at this rate,
|
||||
// and in-band FEC at an assumed 10% loss lets the host's decoder reconstruct a dropped
|
||||
// datagram from its successor instead of playing a hole (the uplink is fire-and-forget).
|
||||
let _ = enc.set_complexity(5);
|
||||
let _ = enc.set_inband_fec(true);
|
||||
let _ = enc.set_packet_loss_perc(10);
|
||||
|
||||
let frame = FRAME_SAMPLES * CHANNELS;
|
||||
let mut ring: VecDeque<f32> = VecDeque::with_capacity(frame * 4);
|
||||
let mut pcm = vec![0f32; frame]; // reusable encode scratch (one 20 ms frame)
|
||||
let mut out = vec![0u8; 4000]; // max Opus packet for a 20 ms frame fits easily
|
||||
let mut pcm = vec![0f32; frame]; // reusable encode scratch (one 10 ms frame)
|
||||
let mut out = vec![0u8; 4000]; // max Opus packet for a 10 ms frame fits easily
|
||||
let mut seq: u32 = 0;
|
||||
let mut sent: u64 = 0;
|
||||
let mut stale: u64 = 0; // frames shed by the backlog self-heal (see BACKLOG_MAX_FRAMES)
|
||||
let mut muted_frames: u64 = 0; // frames dropped unencoded because the user muted
|
||||
let mut peak = 0f32; // loudest |sample| since the last log — tells speech from silence
|
||||
|
||||
while !shutdown.load(Ordering::Relaxed) {
|
||||
@@ -207,11 +307,41 @@ fn encode_loop(
|
||||
// callback's free-list (dropped only if the pool is momentarily full).
|
||||
ring.extend(chunk.drain(..));
|
||||
let _ = free_tx.try_send(chunk);
|
||||
// Drain whatever else queued while we were away, so a post-stall backlog lands as
|
||||
// ONE lump the self-heal below can size up — chunk-at-a-time it would be encoded
|
||||
// (and inflicted on the host as standing delay) before it ever looked deep.
|
||||
while let Ok(mut chunk) = rx.try_recv() {
|
||||
ring.extend(chunk.drain(..));
|
||||
let _ = free_tx.try_send(chunk);
|
||||
}
|
||||
}
|
||||
Err(RecvTimeoutError::Timeout) => continue, // wake to re-check shutdown
|
||||
Err(RecvTimeoutError::Disconnected) => break,
|
||||
}
|
||||
// Self-heal the latency ratchet: a stall (scheduler hiccup, a slow send) queues stale
|
||||
// audio, and every ms of it would ride the stream as mic delay for the rest of the
|
||||
// session. Jump to the newest ~20 ms (one audible blip), counting the shed.
|
||||
if ring.len() > BACKLOG_MAX_FRAMES * frame {
|
||||
let excess = ring.len() - BACKLOG_KEEP_FRAMES * frame;
|
||||
ring.drain(..excess);
|
||||
stale += (excess / frame) as u64;
|
||||
}
|
||||
while ring.len() >= frame {
|
||||
// Muted: drop the frame at the last point before it would become an Opus packet —
|
||||
// room audio is never encoded and nothing goes on the wire. `seq` does NOT advance:
|
||||
// it numbers the datagrams the host de-jitters, and that side reads a seq jump as
|
||||
// loss (conceal + a counted gap) where a mute is a pause. Freezing it means the
|
||||
// frame after an unmute continues the chain, which is what the host's own
|
||||
// `reset_stream` doc calls for and what the desktop uplink does. (Encoding silence
|
||||
// instead would keep a pointless uplink and a host-side ring alive for the whole
|
||||
// mute.) `peak` is the loudest sample the UPLINK carried since the last log, so a
|
||||
// dropped frame resets rather than raises it.
|
||||
if muted.load(Ordering::Relaxed) {
|
||||
ring.drain(..frame);
|
||||
muted_frames += 1;
|
||||
peak = 0.0;
|
||||
continue;
|
||||
}
|
||||
for (dst, src) in pcm.iter_mut().zip(ring.drain(..frame)) {
|
||||
*dst = src;
|
||||
}
|
||||
@@ -227,9 +357,10 @@ fn encode_loop(
|
||||
let _ = client.send_mic(seq, pts, out[..len].to_vec());
|
||||
seq = seq.wrapping_add(1);
|
||||
sent += 1;
|
||||
if sent % 250 == 0 {
|
||||
if sent % 500 == 0 {
|
||||
log::info!(
|
||||
"mic: sent={sent} captured_frames={} dropped_chunks={} peak={peak:.3}",
|
||||
"mic: sent={sent} captured_frames={} dropped_chunks={} \
|
||||
stale_frames={stale} muted_frames={muted_frames} peak={peak:.3}",
|
||||
captured.load(Ordering::Relaxed),
|
||||
dropped.load(Ordering::Relaxed),
|
||||
);
|
||||
@@ -241,7 +372,8 @@ fn encode_loop(
|
||||
}
|
||||
}
|
||||
log::info!(
|
||||
"mic: stopped (sent={sent} captured_frames={} dropped_chunks={})",
|
||||
"mic: stopped (sent={sent} captured_frames={} dropped_chunks={} stale_frames={stale} \
|
||||
muted_frames={muted_frames})",
|
||||
captured.load(Ordering::Relaxed),
|
||||
dropped.load(Ordering::Relaxed),
|
||||
);
|
||||
|
||||
@@ -83,6 +83,28 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSetLowLaten
|
||||
punktfunk_core::transport::set_dscp_default(enabled != 0);
|
||||
}
|
||||
|
||||
/// `debug.punktfunk.force_parts` = 1: arm slice-progressive parts delivery even when the
|
||||
/// Kotlin `FEATURE_PartialFrame` probe said no — the rebuild-free on-glass experiment for a
|
||||
/// decoder that may accept `BUFFER_FLAG_PARTIAL_FRAME` without declaring the feature (the NP3's
|
||||
/// c2.qti decoders declare nothing). Android-only; everywhere else the probe verdict stands.
|
||||
#[cfg(target_os = "android")]
|
||||
fn force_parts_sysprop() -> bool {
|
||||
let mut buf = [0u8; 92]; // PROP_VALUE_MAX
|
||||
// SAFETY: __system_property_get with a valid name + PROP_VALUE_MAX buffer is always safe.
|
||||
let n = unsafe {
|
||||
libc::__system_property_get(
|
||||
c"debug.punktfunk.force_parts".as_ptr(),
|
||||
buf.as_mut_ptr().cast(),
|
||||
)
|
||||
};
|
||||
n > 0 && std::str::from_utf8(&buf[..n as usize]).unwrap_or("").trim() == "1"
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "android"))]
|
||||
fn force_parts_sysprop() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// `NativeBridge.nativeConnect(host, port, w, h, hz, certPem, keyPem, pinHex, bitrateKbps,
|
||||
/// compositorPref, gamepadPref, hdrEnabled, audioChannels, preferredCodec, timeoutMs, launch,
|
||||
/// deviceName): Long`.
|
||||
@@ -155,6 +177,24 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo
|
||||
} else {
|
||||
Some((cert, key))
|
||||
};
|
||||
// Slice-progressive parts, by decoder truth (Kotlin's FEATURE_PartialFrame probe) — with a
|
||||
// sysprop escape hatch for the on-glass science question the probe can't answer: does the
|
||||
// decoder ACTUALLY choke on BUFFER_FLAG_PARTIAL_FRAME input, or does it merely not declare
|
||||
// the feature? (`adb shell setprop debug.punktfunk.force_parts 1` + stream restart; a codec
|
||||
// that can't take parts errors recoverably and the reanchor gate + keyframe path recovers.)
|
||||
let force_parts = force_parts_sysprop();
|
||||
let frame_parts = frame_parts_ok != 0 || force_parts;
|
||||
// The connect-time capability readout (`adb logcat -s pf.caps`): the P2 slice pipeline is
|
||||
// inert client-side unless BOTH probes pass — this line is the one place that says which.
|
||||
log::info!(
|
||||
target: "pf.caps",
|
||||
"decoder caps: multi_slice={} partial_frame={}{} hdr={} codec_bits={:#x}",
|
||||
multi_slice_ok != 0,
|
||||
frame_parts_ok != 0,
|
||||
if force_parts { " (FORCED by sysprop)" } else { "" },
|
||||
hdr_enabled != 0,
|
||||
video_codecs,
|
||||
);
|
||||
let pin: Option<[u8; 32]> = if pin_hex.is_empty() {
|
||||
None
|
||||
} else {
|
||||
@@ -230,9 +270,10 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo
|
||||
// should say what the client does).
|
||||
punktfunk_core::quic::CLIENT_CAP_PHASE_LOCK,
|
||||
// Slice-progressive delivery, by decoder truth (Kotlin probes FEATURE_PartialFrame on
|
||||
// every decoder this device would use): AU prefixes then arrive as `Frame::part`
|
||||
// pieces and the decode loop feeds them with BUFFER_FLAG_PARTIAL_FRAME.
|
||||
frame_parts_ok != 0,
|
||||
// every decoder this device would use; `debug.punktfunk.force_parts` overrides for the
|
||||
// on-glass experiment): AU prefixes then arrive as `Frame::part` pieces and the decode
|
||||
// loop feeds them with BUFFER_FLAG_PARTIAL_FRAME.
|
||||
frame_parts,
|
||||
launch, // a store-qualified library id to boot into a game, or None for the desktop
|
||||
device_name, // Kotlin's Build.MODEL — the host's approval-list / trust-store label
|
||||
pin, // Some → Crypto on host-fp mismatch
|
||||
@@ -250,6 +291,8 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo
|
||||
audio: Mutex::new(None),
|
||||
#[cfg(target_os = "android")]
|
||||
mic: Mutex::new(None),
|
||||
// A fresh session is never muted (mute is per-session UI state, not a setting).
|
||||
mic_muted: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
};
|
||||
Box::into_raw(Box::new(handle)) as jlong
|
||||
}
|
||||
|
||||
@@ -61,6 +61,13 @@ pub(crate) struct SessionHandle {
|
||||
audio: Mutex<Option<crate::audio::AudioPlayback>>,
|
||||
#[cfg(target_os = "android")]
|
||||
mic: Mutex<Option<crate::mic::MicCapture>>,
|
||||
/// In-stream mic mute, set via `nativeSetMicMuted` and read per 10 ms frame by the mic's
|
||||
/// encode loop ([`crate::mic`]). Session-lifetime rather than per-[`crate::mic::MicCapture`]
|
||||
/// for the same reason the stats gate is: the mic stops and restarts across a surface
|
||||
/// recreate, and a mute the user set must come back with it — with no window in which the
|
||||
/// fresh capture could send an unmuted frame. Per session and never persisted: a new session
|
||||
/// starts unmuted.
|
||||
pub mic_muted: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
struct VideoThread {
|
||||
|
||||
@@ -177,11 +177,12 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopVideo(
|
||||
}
|
||||
|
||||
/// `NativeBridge.nativeVideoStats(handle): DoubleArray?` — drain ~1 s of decode stats for the HUD
|
||||
/// (unified stats spec, `design/stats-unification.md`). Returns 26 doubles
|
||||
/// (unified stats spec, `design/stats-unification.md`). Returns 33 doubles
|
||||
/// `[fps, mbps, e2eP50Ms, e2eP95Ms, latValid, skewCorrected, width, height, refreshHz, framesLost,
|
||||
/// bitDepth, colorPrimaries, colorTransfer, chromaFormatIdc, hostNetP50Ms, decodeP50Ms, hostP50Ms,
|
||||
/// netP50Ms, lostWindow, skippedWindow, fecWindow, framesWindow, dispValid, displayP50Ms,
|
||||
/// e2eDispP50Ms, e2eDispP95Ms, paceP50Ms, latchP50Ms, presentsWindow, presenterActive]`
|
||||
/// e2eDispP50Ms, e2eDispP95Ms, paceP50Ms, latchP50Ms, presentsWindow, presenterActive,
|
||||
/// feedP50Ms, codecP50Ms, skippedOverflowWindow]`
|
||||
/// (the flags are 1.0/0.0; indexes 0–21 match the previous 22-double layout — 0–13 the original
|
||||
/// 14-double one with the latency pair re-based to the end-to-end capture→decoded headline, 14/15
|
||||
/// the stage p50s tiling it: `host+network` = capture→received, `decode` = received→decoded; 16/17
|
||||
@@ -198,7 +199,11 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopVideo(
|
||||
/// term — `pace` = decoded→release (store + glass budget) p50 at 26, `latch` =
|
||||
/// release→displayed (SurfaceFlinger) p50 at 27, the window's on-glass confirm count at 28
|
||||
/// (`presents` vs `fps` is the presenter-health pair), and 29 = 1.0 while the timeline presenter
|
||||
/// is active this session), or `null` when no decode thread is running.
|
||||
/// is active this session; 30/31 are the `decode` stage's split p50s — `feed` =
|
||||
/// received→queued (hand-off + input-slot wait) at 30 and `codec` = queued→decoded (codec-pure,
|
||||
/// from the AU's last piece) at 31, both 0.0 when no sample landed (sync loop); 32 is the
|
||||
/// parked-AU overflow subset of the window's `skipped` at 19 (decoder fell behind, vs benign
|
||||
/// newest-wins pacing)), or `null` when no decode thread is running.
|
||||
/// Poll ~1 Hz from the UI; each call
|
||||
/// resets the measurement window. Not android-gated — pure `jni` + connector reads, so it links on
|
||||
/// the host build too (Kotlin only ever calls it on device).
|
||||
@@ -222,7 +227,7 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats(
|
||||
.drain(h.client.frames_dropped(), h.client.fec_recovered_shards());
|
||||
let mode = h.client.mode();
|
||||
let color = h.client.color;
|
||||
let buf: [f64; 30] = [
|
||||
let buf: [f64; 33] = [
|
||||
snap.fps,
|
||||
snap.mbps,
|
||||
snap.e2e_p50_ms,
|
||||
@@ -270,6 +275,12 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeVideoStats(
|
||||
snap.latch_p50_ms,
|
||||
snap.presents as f64,
|
||||
if h.stats.presenter_active() { 1.0 } else { 0.0 },
|
||||
// The `decode` stage's split (P3 science): feed = received→queued (hand-off +
|
||||
// input-slot wait), codec = queued→decoded (codec-pure) — and the parked-AU
|
||||
// overflow subset of `skipped` (decoder-health vs benign pacing drops).
|
||||
snap.feed_p50_ms,
|
||||
snap.codec_p50_ms,
|
||||
snap.skipped_overflow as f64,
|
||||
];
|
||||
let arr = match env.new_double_array(buf.len() as jsize) {
|
||||
Ok(a) => a,
|
||||
@@ -390,33 +401,49 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopAudio(
|
||||
})
|
||||
}
|
||||
|
||||
/// `NativeBridge.nativeStartMic(handle)` — start mic capture (AAudio input → Opus → host `send_mic`).
|
||||
/// No-op if already running or on a `0` handle. Caller MUST hold RECORD_AUDIO; a failure (e.g. no
|
||||
/// permission) leaves the rest of the session streaming.
|
||||
/// `NativeBridge.nativeStartMic(handle, echoCancel): Int` — start mic capture (AAudio input →
|
||||
/// Opus → host `send_mic`). `echoCancel` opens the capture under the `VoiceCommunication` preset
|
||||
/// (the HAL's echo canceller / noise suppressor) and allocates an audio session id; the return
|
||||
/// value is that id (`> 0`), so Kotlin can attach the Java `AcousticEchoCanceler`/`NoiseSuppressor`
|
||||
/// as a backstop — `0` when none was allocated (echoCancel off, the preset fell back to the plain
|
||||
/// open, a `0` handle, or the mic failed entirely). Already running (a surface recreate) returns
|
||||
/// the running capture's id. Caller MUST hold RECORD_AUDIO; a failure (e.g. no permission) leaves
|
||||
/// the rest of the session streaming.
|
||||
#[cfg(target_os = "android")]
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStartMic(
|
||||
_env: JNIEnv,
|
||||
_this: JObject,
|
||||
handle: jlong,
|
||||
) {
|
||||
echo_cancel: jboolean,
|
||||
) -> jni::sys::jint {
|
||||
if handle == 0 {
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
// SAFETY: live handle per the nativeConnect/nativeClose contract.
|
||||
let h = unsafe { &*(handle as *const SessionHandle) };
|
||||
let mut guard = h.mic.lock().unwrap();
|
||||
if guard.is_some() {
|
||||
return; // already capturing
|
||||
if let Some(m) = guard.as_ref() {
|
||||
return m.session_id(); // already capturing — same stream, same session
|
||||
}
|
||||
match crate::mic::MicCapture::start(h.client.clone()) {
|
||||
Some(m) => *guard = Some(m),
|
||||
None => log::error!("nativeStartMic: mic init failed (RECORD_AUDIO? — session unaffected)"),
|
||||
// The capture SHARES the session's mute flag, so one started while muted stays muted (and
|
||||
// sends nothing) from its very first frame — see `SessionHandle::mic_muted`.
|
||||
match crate::mic::MicCapture::start(h.client.clone(), echo_cancel != 0, h.mic_muted.clone()) {
|
||||
Some(m) => {
|
||||
let session_id = m.session_id();
|
||||
*guard = Some(m);
|
||||
session_id
|
||||
}
|
||||
None => {
|
||||
log::error!("nativeStartMic: mic init failed (RECORD_AUDIO? — session unaffected)");
|
||||
0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `NativeBridge.nativeStopMic(handle)` — stop + join the mic thread and close the AAudio input
|
||||
/// stream (without closing the session). No-op on `0`.
|
||||
/// stream (without closing the session). No-op on `0`. Leaves the session's mute state alone: a
|
||||
/// surface recreate stops and restarts the mic, and a user who muted must stay muted through it.
|
||||
#[cfg(target_os = "android")]
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopMic(
|
||||
@@ -432,3 +459,59 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeStopMic(
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// `NativeBridge.nativeSetMicMuted(handle, muted)` — mute/unmute the mic uplink mid-stream.
|
||||
///
|
||||
/// Muting deliberately does NOT stop the capture: the AAudio input stream, the input-preset rung
|
||||
/// it settled on and its primed buffers all stay exactly as they are, and the encode loop simply
|
||||
/// drops each 10 ms frame instead of encoding + sending it. A stop/start would re-run the preset
|
||||
/// fallback ladder and re-prime buffers on every toggle — hundreds of ms, and possibly a different
|
||||
/// rung (echo cancellation silently lost). This way a toggle costs one atomic store here and one
|
||||
/// relaxed load per frame there, and takes effect on the very next 10 ms boundary.
|
||||
///
|
||||
/// Sticky for the SESSION (the flag lives on the handle, not on the capture), so the mic restart a
|
||||
/// surface recreate performs comes back muted with no window for an unmuted frame to escape; a
|
||||
/// fresh session always starts unmuted. No-op on `0`. Not android-gated — pure `jni` + an atomic
|
||||
/// store, so it links on the host build too.
|
||||
///
|
||||
/// One honest consequence of keeping the stream open: the platform's own recording indicator stays
|
||||
/// lit while muted, because the mic really is still open. What stops is the encode and the send —
|
||||
/// no captured audio leaves the process.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSetMicMuted(
|
||||
_env: JNIEnv,
|
||||
_this: JObject,
|
||||
handle: jlong,
|
||||
muted: jboolean,
|
||||
) {
|
||||
jni_guard((), || {
|
||||
if handle != 0 {
|
||||
// SAFETY: live handle per the nativeConnect/nativeClose contract.
|
||||
let h = unsafe { &*(handle as *const SessionHandle) };
|
||||
h.mic_muted
|
||||
.store(muted != 0, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// `NativeBridge.nativeMicActive(handle): Boolean` — is a mic capture actually RUNNING? `true` only
|
||||
/// between a `nativeStartMic` that opened a stream and the matching `nativeStopMic`. The in-stream
|
||||
/// mute control is offered on this evidence rather than on the user's setting, so a device that
|
||||
/// refused every AAudio input rung (or a missing RECORD_AUDIO grant) shows no control instead of a
|
||||
/// lie about a mic that is being heard. `false` on a `0` handle. Cheap (one uncontended lock).
|
||||
#[cfg(target_os = "android")]
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeMicActive(
|
||||
_env: JNIEnv,
|
||||
_this: JObject,
|
||||
handle: jlong,
|
||||
) -> jboolean {
|
||||
jni_guard(0, || {
|
||||
if handle == 0 {
|
||||
return 0;
|
||||
}
|
||||
// SAFETY: live handle per the nativeConnect/nativeClose contract.
|
||||
let h = unsafe { &*(handle as *const SessionHandle) };
|
||||
jboolean::from(h.mic.lock().unwrap().is_some())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -76,6 +76,12 @@ struct Inner {
|
||||
/// The other half of the split, release→displayed (SurfaceFlinger's latch + scanout), µs —
|
||||
/// from the `OnFrameRendered` render timestamps. `pace + latch ≈ display` per frame.
|
||||
latch_us: Vec<u64>,
|
||||
/// The `decode` stage's feed split, received→queued (hand-off + input-slot wait), µs. Empty
|
||||
/// when no receipt stamp matched (HUD off and ABR not measuring decode).
|
||||
feed_us: Vec<u64>,
|
||||
/// The other half, queued→decoded (codec-pure: the decoder's own time on the AU, measured
|
||||
/// from its LAST piece so a slice-progressive head start shows up as a shrink here), µs.
|
||||
codec_us: Vec<u64>,
|
||||
/// Frames confirmed on glass this window (`OnFrameRendered` callbacks) — the `presents`-vs-
|
||||
/// `fps` health pair: presents ≪ fps means the presenter is dropping/serializing; an fps
|
||||
/// deficit is upstream.
|
||||
@@ -83,6 +89,10 @@ struct Inner {
|
||||
/// Client-side newest-wins/pacing drops this window (decoded frames released without
|
||||
/// rendering, or parked AUs dropped on overflow) — the spec's `skipped` counter.
|
||||
skipped: u64,
|
||||
/// The subset of `skipped` that was parked-AU OVERFLOW (the decoder fell behind and whole
|
||||
/// AUs were dropped before feeding) — a decoder-health signal, vs the benign newest-wins
|
||||
/// pacing majority. Always ≤ `skipped`.
|
||||
skipped_overflow: u64,
|
||||
/// Baselines for windowing the session-cumulative connector counters: the unrecoverable-drop
|
||||
/// and FEC-recovered totals as of the last drain (or the enable that opened the window), so
|
||||
/// each snapshot reports only THIS window's `lost` / `FEC` (spec line 4).
|
||||
@@ -119,6 +129,11 @@ pub struct Snapshot {
|
||||
/// path / no render callbacks).
|
||||
pub pace_p50_ms: f64,
|
||||
pub latch_p50_ms: f64,
|
||||
/// The `decode` stage's split p50s (ms): `feed` = received→queued (hand-off + input-slot
|
||||
/// wait), `codec` = queued→decoded (codec-pure, from the AU's last piece). 0.0 when no
|
||||
/// sample landed (sync loop / no receipt stamps).
|
||||
pub feed_p50_ms: f64,
|
||||
pub codec_p50_ms: f64,
|
||||
/// Frames confirmed on glass this window (`OnFrameRendered` callbacks).
|
||||
pub presents: u64,
|
||||
/// Phase-2 `host` / `network` split p50s (ms) — 0.0 when no 0xCF timing matched this window
|
||||
@@ -135,6 +150,8 @@ pub struct Snapshot {
|
||||
pub lost: u64,
|
||||
/// Client-side newest-wins/pacing drops this window (spec `skipped`).
|
||||
pub skipped: u64,
|
||||
/// The parked-AU overflow subset of `skipped` (decoder fell behind; ≤ `skipped`).
|
||||
pub skipped_overflow: u64,
|
||||
/// FEC shards recovered this window (spec `FEC`, windowed from the cumulative counter).
|
||||
pub fec: u64,
|
||||
}
|
||||
@@ -167,8 +184,11 @@ impl VideoStats {
|
||||
e2e_disp_us: Vec::with_capacity(256),
|
||||
pace_us: Vec::with_capacity(256),
|
||||
latch_us: Vec::with_capacity(256),
|
||||
feed_us: Vec::with_capacity(256),
|
||||
codec_us: Vec::with_capacity(256),
|
||||
presents: 0,
|
||||
skipped: 0,
|
||||
skipped_overflow: 0,
|
||||
last_dropped_total: 0,
|
||||
last_fec_total: 0,
|
||||
skew_corrected: false,
|
||||
@@ -219,8 +239,11 @@ impl VideoStats {
|
||||
g.e2e_disp_us.clear();
|
||||
g.pace_us.clear();
|
||||
g.latch_us.clear();
|
||||
g.feed_us.clear();
|
||||
g.codec_us.clear();
|
||||
g.presents = 0;
|
||||
g.skipped = 0;
|
||||
g.skipped_overflow = 0;
|
||||
g.last_dropped_total = dropped_total;
|
||||
g.last_fec_total = fec_total;
|
||||
}
|
||||
@@ -314,6 +337,45 @@ impl VideoStats {
|
||||
g.skipped += n;
|
||||
}
|
||||
|
||||
/// Record parked-AU OVERFLOW drops (whole AUs dropped before feeding — the decoder fell
|
||||
/// behind). Counts into `skipped` too, plus the overflow-only counter, so the HUD can tell
|
||||
/// benign newest-wins pacing from a decoder that can't keep up.
|
||||
// Driven only by the android-only decode thread; unreferenced on the host build — expected.
|
||||
#[cfg_attr(not(target_os = "android"), allow(dead_code))]
|
||||
pub fn note_skipped_overflow(&self, n: u64) {
|
||||
if n == 0 || !self.enabled.load(Ordering::Relaxed) {
|
||||
return; // HUD hidden — skip the lock
|
||||
}
|
||||
// Poison-proof for the same reason as `note_received`.
|
||||
let mut g = self
|
||||
.inner
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
g.skipped += n;
|
||||
g.skipped_overflow += n;
|
||||
}
|
||||
|
||||
/// Record one decoded frame's `decode`-stage split: `feed` = received→queued (hand-off +
|
||||
/// input-slot wait; absent when no receipt stamp matched) and `codec` = queued→decoded
|
||||
/// (codec-pure, measured from the AU's LAST piece — a slice-progressive head start shows
|
||||
/// as a shrink here), both µs.
|
||||
// Driven only by the android-only decode thread; unreferenced on the host build — expected.
|
||||
#[cfg_attr(not(target_os = "android"), allow(dead_code))]
|
||||
pub fn note_decode_split(&self, feed_us: Option<u64>, codec_us: u64) {
|
||||
if !self.enabled.load(Ordering::Relaxed) {
|
||||
return; // HUD hidden — skip the lock
|
||||
}
|
||||
// Poison-proof for the same reason as `note_received`.
|
||||
let mut g = self
|
||||
.inner
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(f) = feed_us {
|
||||
g.feed_us.push(f);
|
||||
}
|
||||
g.codec_us.push(codec_us);
|
||||
}
|
||||
|
||||
/// Record one decoded output frame: its capture→decoded `end-to-end` sample and its
|
||||
/// received→decoded `decode` stage sample (either may be absent — e.g. the receipt stamp for
|
||||
/// this pts predates the HUD being shown).
|
||||
@@ -408,6 +470,8 @@ impl VideoStats {
|
||||
g.e2e_disp_us.sort_unstable();
|
||||
g.pace_us.sort_unstable();
|
||||
g.latch_us.sort_unstable();
|
||||
g.feed_us.sort_unstable();
|
||||
g.codec_us.sort_unstable();
|
||||
let snap = Snapshot {
|
||||
fps,
|
||||
mbps,
|
||||
@@ -421,6 +485,8 @@ impl VideoStats {
|
||||
disp_valid: !g.e2e_disp_us.is_empty(),
|
||||
pace_p50_ms: pctl_ms(&g.pace_us, 0.50),
|
||||
latch_p50_ms: pctl_ms(&g.latch_us, 0.50),
|
||||
feed_p50_ms: pctl_ms(&g.feed_us, 0.50),
|
||||
codec_p50_ms: pctl_ms(&g.codec_us, 0.50),
|
||||
presents: g.presents,
|
||||
host_p50_ms: pctl_ms(&g.host_us, 0.50),
|
||||
net_p50_ms: pctl_ms(&g.net_us, 0.50),
|
||||
@@ -429,6 +495,7 @@ impl VideoStats {
|
||||
frames: g.frames,
|
||||
lost: dropped_total.saturating_sub(g.last_dropped_total),
|
||||
skipped: g.skipped,
|
||||
skipped_overflow: g.skipped_overflow,
|
||||
fec: fec_total.saturating_sub(g.last_fec_total),
|
||||
};
|
||||
g.window_start = Instant::now();
|
||||
@@ -443,8 +510,11 @@ impl VideoStats {
|
||||
g.e2e_disp_us.clear();
|
||||
g.pace_us.clear();
|
||||
g.latch_us.clear();
|
||||
g.feed_us.clear();
|
||||
g.codec_us.clear();
|
||||
g.presents = 0;
|
||||
g.skipped = 0;
|
||||
g.skipped_overflow = 0;
|
||||
g.last_dropped_total = dropped_total;
|
||||
g.last_fec_total = fec_total;
|
||||
snap
|
||||
|
||||
@@ -315,7 +315,15 @@ struct ContentView: View {
|
||||
clipboardAvailable: model.connection?.hostSupportsClipboard == true,
|
||||
clipboardOn: model.clipboardEnabled,
|
||||
toggleClipboard: { model.toggleClipboardSync() },
|
||||
micAvailable: model.micAvailable,
|
||||
micMuted: model.micMuted,
|
||||
toggleMicMute: { model.toggleMicMute() },
|
||||
disconnect: { model.disconnect() }))
|
||||
// ⌃⌥⇧A fired while input was CAPTURED (InputCapture's chord path posts it — the menu's
|
||||
// identical equivalent can't reach a captured stream). Same toggle either way.
|
||||
.onReceive(NotificationCenter.default.publisher(for: .punktfunkToggleMicMute)) { _ in
|
||||
model.toggleMicMute()
|
||||
}
|
||||
#endif
|
||||
#if os(macOS)
|
||||
// Fullscreen only while a session is up (incl. the trust prompt over the blurred stream),
|
||||
@@ -724,31 +732,48 @@ struct ContentView: View {
|
||||
}
|
||||
.animation(.smooth(duration: 0.28), value: statsVerbosity)
|
||||
}
|
||||
#if os(macOS) || os(tvOS)
|
||||
// The start-of-stream shortcut banner (Windows-client parity): the platform's
|
||||
// reserved controls on a glass pill, bottom-centre, for the first 6 seconds of
|
||||
// every session — independent of the stats HUD, so the keys are discoverable
|
||||
// even with statistics off. The banner's own task drops it (cancelled cleanly
|
||||
// if the session view goes away first). On tvOS it carries the ONLY exits —
|
||||
// Menu/B is swallowed during a session (the `.onExitCommand {}` in the tvOS
|
||||
// session branch), so the hold gestures must be told to the user.
|
||||
// The bottom-centre stack: the muted-microphone badge over the start-of-stream
|
||||
// shortcut banner. ONE overlay for both, so the two can never land on top of each
|
||||
// other in the seconds where they overlap.
|
||||
.overlay(alignment: .bottom) {
|
||||
if captureEnabled && showShortcutHint {
|
||||
Text(Self.shortcutHintText)
|
||||
.font(.geist(Self.shortcutHintFont, relativeTo: .caption))
|
||||
.foregroundStyle(.secondary)
|
||||
.padding(.horizontal, 14)
|
||||
.padding(.vertical, 8)
|
||||
.glassBackground(Capsule())
|
||||
.padding(.bottom, 24)
|
||||
.transition(.opacity)
|
||||
.task {
|
||||
try? await Task.sleep(for: .seconds(6))
|
||||
withAnimation(.easeOut(duration: 0.6)) { showShortcutHint = false }
|
||||
}
|
||||
VStack(spacing: 8) {
|
||||
#if !os(tvOS)
|
||||
// Shown for as long as the mic is muted, at every stats tier and with the
|
||||
// overlay off — see MicMutedBadge. tvOS has no microphone to mute.
|
||||
if captureEnabled && model.micMuted {
|
||||
MicMutedBadge { model.setMicMuted(false) }
|
||||
.transition(.opacity.combined(with: .scale(scale: 0.9)))
|
||||
}
|
||||
#endif
|
||||
#if os(macOS) || os(tvOS)
|
||||
// The start-of-stream shortcut banner (Windows-client parity): the
|
||||
// platform's reserved controls on a glass pill for the first 6 seconds of
|
||||
// every session — independent of the stats HUD, so the keys are
|
||||
// discoverable even with statistics off. The banner's own task drops it
|
||||
// (cancelled cleanly if the session view goes away first). On tvOS it
|
||||
// carries the ONLY exits — Menu/B is swallowed during a session (the
|
||||
// `.onExitCommand {}` in the tvOS session branch), so the hold gestures
|
||||
// must be told to the user.
|
||||
if captureEnabled && showShortcutHint {
|
||||
Text(shortcutHintText)
|
||||
.font(.geist(Self.shortcutHintFont, relativeTo: .caption))
|
||||
.foregroundStyle(.secondary)
|
||||
.padding(.horizontal, 14)
|
||||
.padding(.vertical, 8)
|
||||
.glassBackground(Capsule())
|
||||
.transition(.opacity)
|
||||
.task {
|
||||
try? await Task.sleep(for: .seconds(6))
|
||||
withAnimation(.easeOut(duration: 0.6)) {
|
||||
showShortcutHint = false
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
.padding(.bottom, 24)
|
||||
.animation(.easeOut(duration: 0.2), value: model.micMuted)
|
||||
}
|
||||
#endif
|
||||
#if os(iOS)
|
||||
// Touch users have no menu / ⌘D, so when the HUD's Disconnect button isn't on
|
||||
// screen — the overlay off, or the compact pill (which carries no button) —
|
||||
@@ -766,21 +791,24 @@ struct ContentView: View {
|
||||
.overlay(alignment: .topLeading) {
|
||||
if captureEnabled,
|
||||
statsVerbosity == .compact || (statsVerbosity == .off && showTouchExit) {
|
||||
Button { model.disconnect() } label: {
|
||||
Image(systemName: "xmark")
|
||||
.font(.headline.weight(.semibold))
|
||||
.frame(width: 36, height: 36)
|
||||
// Floating glass disc over the frame (26+, material fallback).
|
||||
// interactive: the disc IS the tap target, so the glass reacts
|
||||
// to press.
|
||||
.glassBackground(Circle(), interactive: true)
|
||||
// Match the hit region to the visible disc so every tap also
|
||||
// triggers the interactive-glass press highlight.
|
||||
.contentShape(Circle())
|
||||
HStack(spacing: 10) {
|
||||
Button { model.disconnect() } label: { touchDisc("xmark") }
|
||||
.buttonStyle(.plain)
|
||||
.accessibilityLabel("Disconnect")
|
||||
// The mic toggle rides the same discs, for the same reason: in these
|
||||
// tiers the HUD carries no buttons (compact is a stat pill, off is
|
||||
// nothing), so this is a touch-only user's ONLY way to mute. Absent —
|
||||
// not greyed — when the session sends no microphone at all.
|
||||
if model.micAvailable {
|
||||
Button { model.toggleMicMute() } label: {
|
||||
touchDisc(model.micMuted ? "mic.slash.fill" : "mic.fill")
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.accessibilityLabel(
|
||||
model.micMuted ? "Unmute microphone" : "Mute microphone")
|
||||
}
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.padding(12)
|
||||
.accessibilityLabel("Disconnect")
|
||||
.transition(.opacity)
|
||||
.task {
|
||||
guard statsVerbosity == .off else { return }
|
||||
@@ -794,14 +822,34 @@ struct ContentView: View {
|
||||
}
|
||||
}
|
||||
|
||||
#if os(iOS)
|
||||
/// One touch-control disc: an SF Symbol on a floating glass disc over the frame (26+,
|
||||
/// material fallback), sized as a comfortable tap target. `interactive`: the disc IS the tap
|
||||
/// target, so the glass reacts to press, and the hit region is matched to the visible disc so
|
||||
/// every tap triggers that press highlight.
|
||||
private func touchDisc(_ symbol: String) -> some View {
|
||||
Image(systemName: symbol)
|
||||
.font(.headline.weight(.semibold))
|
||||
.frame(width: 36, height: 36)
|
||||
.glassBackground(Circle(), interactive: true)
|
||||
.contentShape(Circle())
|
||||
}
|
||||
#endif
|
||||
|
||||
#if os(macOS)
|
||||
private static let shortcutHintText =
|
||||
"Click the stream to capture · ⌃⌥⇧Q releases the mouse · ⌃⌥⇧D disconnects · ⌃⌥⇧S stats"
|
||||
/// The reserved combos, told once per session. The mute segment appears only when the session
|
||||
/// actually sends a microphone — teaching a shortcut for a mic that isn't on would be a lie.
|
||||
private var shortcutHintText: String {
|
||||
let base =
|
||||
"Click the stream to capture · ⌃⌥⇧Q releases the mouse · ⌃⌥⇧D disconnects · ⌃⌥⇧S stats"
|
||||
return model.micAvailable ? base + " · ⌃⌥⇧A mutes the mic" : base
|
||||
}
|
||||
private static let shortcutHintFont: CGFloat = 12
|
||||
#elseif os(tvOS)
|
||||
private static let shortcutHintText =
|
||||
private var shortcutHintText: String {
|
||||
"Hold the remote's Back button — or L1+R1+Start+Select on a controller — to disconnect"
|
||||
+ " · Touch surface moves the pointer · press clicks · Play/Pause right-clicks"
|
||||
}
|
||||
private static let shortcutHintFont: CGFloat = 22 // read from the couch
|
||||
#endif
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
// Session state for the app shell: owns the connection, the input capture, the trust
|
||||
// handshake phase, and the pump-thread → main-actor stats relay.
|
||||
|
||||
// AVFoundation: AVCaptureDevice.authorizationStatus (the mic TCC grant behind `micAvailable`)
|
||||
// and, on tvOS, AVPlayer.eligibleForHDRPlayback (the TV-capability HDR gate).
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
import os
|
||||
import PunktfunkKit
|
||||
@@ -11,9 +14,6 @@ import SwiftUI
|
||||
#elseif canImport(UIKit)
|
||||
import UIKit
|
||||
#endif
|
||||
#if os(tvOS)
|
||||
import AVFoundation // AVPlayer.eligibleForHDRPlayback — the TV-capability HDR gate
|
||||
#endif
|
||||
|
||||
/// 1 Hz latency-stage line mirrored to the unified log so the stages can be read WITHOUT the
|
||||
/// on-screen HUD (Console.app, wirelessly on an iPad/Apple TV). The HUD is not a neutral
|
||||
@@ -137,6 +137,14 @@ final class SessionModel: ObservableObject {
|
||||
/// Mirrors StreamView's capture state (it owns the input capture; this drives the
|
||||
/// HUD's "click to capture" / "⌘⎋ releases" hint).
|
||||
@Published var mouseCaptured = false
|
||||
/// The USER's in-stream mic mute (the HUD button, the Stream menu's ⌃⌥⇧A, the captured-state
|
||||
/// chord, the iOS mic disc) — session state, deliberately NOT persisted: a mute is for the
|
||||
/// people in the room right now, so every new session starts live if the mic is on at all.
|
||||
/// One of the two inputs to the effective mute; `isBackgrounded` is the other, and
|
||||
/// `applyMicMute` composes them — a user mute survives a trip through the background, and the
|
||||
/// background's privacy mute never clears the user's choice. Local and instant: it gates
|
||||
/// capture on this device, nothing is sent to the host.
|
||||
@Published private(set) var micMuted = false
|
||||
/// Resize overlay (design/midstream-resolution-resize.md — client resize UX): true from the
|
||||
/// instant a Match-window resize starts steering toward a new size until a frame at that size
|
||||
/// decodes (or a safety timeout). Drives the blur+spinner so the unavoidable host-rebuild delay
|
||||
@@ -440,7 +448,7 @@ final class SessionModel: ObservableObject {
|
||||
guard phase == .streaming, let conn = connection, !isBackgrounded else { return }
|
||||
isBackgrounded = true
|
||||
conn.setVideoDropped(true)
|
||||
audio?.setMicMuted(true)
|
||||
applyMicMute() // now muted for privacy — on top of the user's own mute, not instead of it
|
||||
// Non-deliberate on fire (keep the host linger) so a user who returns late reconnects fast,
|
||||
// exactly like today's network-drop path. min 1 minute guards a nonsense setting.
|
||||
let minutes = max(1, timeoutMinutes)
|
||||
@@ -465,13 +473,57 @@ final class SessionModel: ObservableObject {
|
||||
backgroundDeadline = nil
|
||||
backgroundTimer?.cancel()
|
||||
backgroundTimer = nil
|
||||
audio?.setMicMuted(false)
|
||||
applyMicMute() // back to the user's own choice — which may well still be "muted"
|
||||
if let conn = connection {
|
||||
conn.setVideoDropped(false)
|
||||
conn.requestKeyframe()
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Microphone mute (in-stream, per session)
|
||||
|
||||
/// Whether this session has a mic uplink there is any point in muting: the mic must be on in
|
||||
/// the session's RESOLVED settings (a profile can turn it on or off), the platform must have
|
||||
/// an app-accessible input at all, and the OS must not have refused us one. Drives whether the
|
||||
/// mute control is offered — a live-looking mute button over a session that sends no
|
||||
/// microphone would be a lie. Same three conditions `SessionAudio` starts an uplink on
|
||||
/// (`.notDetermined` counts: the prompt is pending and a grant starts the uplink mid-session).
|
||||
var micAvailable: Bool {
|
||||
#if os(tvOS)
|
||||
return false // no app-accessible microphone — SessionAudio never opens an uplink either
|
||||
#else
|
||||
guard settings.micEnabled else { return false }
|
||||
switch AVCaptureDevice.authorizationStatus(for: .audio) {
|
||||
case .authorized, .notDetermined: return true
|
||||
default: return false // denied / restricted — there is no uplink to mute
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/// Flip the user's mute. The in-stream surfaces (HUD button, Stream menu, ⌃⌥⇧A while
|
||||
/// captured, the iOS mic disc) all land here.
|
||||
func toggleMicMute() {
|
||||
setMicMuted(!micMuted)
|
||||
}
|
||||
|
||||
/// Set the user's mute directly (the badge's tap-to-unmute). Ignored when the session has no
|
||||
/// microphone, so a stale surface can't leave a phantom "muted" badge over a session that was
|
||||
/// never sending anything.
|
||||
func setMicMuted(_ muted: Bool) {
|
||||
guard micAvailable, micMuted != muted else { return }
|
||||
micMuted = muted
|
||||
applyMicMute()
|
||||
}
|
||||
|
||||
/// Push the EFFECTIVE mute — the user's choice OR the background keep-alive's privacy mute —
|
||||
/// onto the audio engine. The two reasons are composed here and nowhere else: whichever one
|
||||
/// changed, the other still holds, so returning from the background can't un-mute a user who
|
||||
/// muted mid-stream, and a user unmuting while backgrounded (Live Activity, another window)
|
||||
/// doesn't open the mic behind their back.
|
||||
private func applyMicMute() {
|
||||
audio?.setMicMuted(micMuted || isBackgrounded)
|
||||
}
|
||||
|
||||
/// Follow a live stats-overlay cycle (⌃⌥⇧S, the three-finger tap, the Stream menu). Those
|
||||
/// surfaces write the GLOBAL setting as they always have; this moves the session's own tier
|
||||
/// with it, so cycling still works in a session a profile put on a different tier.
|
||||
@@ -509,6 +561,9 @@ final class SessionModel: ObservableObject {
|
||||
backgroundTimer = nil
|
||||
isBackgrounded = false
|
||||
backgroundDeadline = nil
|
||||
// The mic mute is per-session and never persisted: the next stream starts live (if the
|
||||
// mic is enabled), rather than silently carrying a mute nobody remembers making.
|
||||
micMuted = false
|
||||
let audio = self.audio
|
||||
self.audio = nil
|
||||
// Gamepad capture is main-actor (releases held buttons on the wire while the
|
||||
@@ -609,7 +664,8 @@ final class SessionModel: ObservableObject {
|
||||
speakerUID: settings.speakerUID,
|
||||
micUID: settings.micUID,
|
||||
micChannel: settings.micChannel,
|
||||
micEnabled: settings.micEnabled)
|
||||
micEnabled: settings.micEnabled,
|
||||
echoCancel: settings.echoCancel)
|
||||
self.audio = audio
|
||||
// Gamepads: forward every controller GamepadManager selected — each on its own wire pad
|
||||
// index (a pin forwards only one, Automatic forwards all) — and render the host's feedback
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
// The app's "Stream" menu (macOS menu bar + iPad hardware-keyboard shortcuts). These live at
|
||||
// the Scene level so they keep working when the HUD overlay is hidden. The shortcuts are the
|
||||
// CROSS-CLIENT set every punktfunk client reserves — Ctrl+Alt+Shift+Q (release the captured
|
||||
// mouse) / +D (disconnect) / +S (stats) — and the menu is their discoverable surface on macOS
|
||||
// mouse) / +D (disconnect) / +S (stats), plus +A (mute the microphone), the Apple clients'
|
||||
// addition to it — and the menu is their discoverable surface on macOS
|
||||
// (the Linux client has its GTK Shortcuts window, Windows its start-of-stream banner). While
|
||||
// input is CAPTURED these key equivalents never reach the menu (the stream view swallows
|
||||
// keys); InputCapture's monitor detects the same combos there and performs the same actions —
|
||||
@@ -27,6 +28,12 @@ struct SessionFocus {
|
||||
/// Clipboard sync is live (host-acked) — drives the item's Stop/Share title.
|
||||
var clipboardOn: Bool
|
||||
var toggleClipboard: () -> Void
|
||||
/// The session has a mic uplink at all (its resolved `micEnabled`) — gates the mute item, so
|
||||
/// it is never an enabled control over a session that sends no microphone.
|
||||
var micAvailable: Bool
|
||||
/// The user's mic mute is engaged — drives the item's Mute/Unmute title.
|
||||
var micMuted: Bool
|
||||
var toggleMicMute: () -> Void
|
||||
var disconnect: () -> Void
|
||||
}
|
||||
|
||||
@@ -60,6 +67,17 @@ struct StreamCommands: Commands {
|
||||
}
|
||||
.keyboardShortcut("q", modifiers: [.control, .option, .shift])
|
||||
.disabled(session?.isStreaming != true)
|
||||
// Mic mute, local and instant (it gates capture on this device — the host is never
|
||||
// asked). Per SESSION: it starts off every time, so this item is a live toggle, not a
|
||||
// setting. Greyed when the session sends no microphone at all (Settings → mic off, or
|
||||
// a profile that turns it off) rather than pretending there is something to mute.
|
||||
// Captured, the combo is handled by InputCapture's chord path before menus see it;
|
||||
// this item is the released-state path and the shortcut's documentation.
|
||||
Button(session?.micMuted == true ? "Unmute Microphone" : "Mute Microphone") {
|
||||
session?.toggleMicMute()
|
||||
}
|
||||
.keyboardShortcut("a", modifiers: [.control, .option, .shift])
|
||||
.disabled(session?.isStreaming != true || session?.micAvailable != true)
|
||||
#if os(macOS)
|
||||
// Mid-session clipboard flip (design/clipboard-and-file-transfer.md §5.3). Greyed
|
||||
// when the host doesn't advertise the cap (older host / operator policy off).
|
||||
|
||||
@@ -179,6 +179,18 @@ struct StreamHUDView: View {
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
#endif
|
||||
// Mic mute — the in-stream toggle, on the same card as the other in-overlay action.
|
||||
// Absent (not greyed) when the session sends no microphone: the HUD is a status card,
|
||||
// and a dead control on it would read as "there is a mic, and it is on". The muted
|
||||
// STATE is not this button's job — the badge over the stream says that at every tier
|
||||
// and with the overlay off entirely. tvOS gets no control: no microphone, and a
|
||||
// focusable one would steal the controller's A press from the host.
|
||||
#if !os(tvOS)
|
||||
if model.micAvailable {
|
||||
Button(micButtonTitle) { model.toggleMicMute() }
|
||||
.font(.geist(12, relativeTo: .caption))
|
||||
}
|
||||
#endif
|
||||
// ⌃⌥⇧D lives on the app's Stream menu (so it still works when the HUD is hidden)
|
||||
// and in InputCapture's monitor while captured; this button is the in-overlay,
|
||||
// click-to-disconnect affordance. tvOS deliberately gets NEITHER a button (a
|
||||
@@ -195,6 +207,19 @@ struct StreamHUDView: View {
|
||||
}
|
||||
}
|
||||
|
||||
#if !os(tvOS)
|
||||
/// The mute button's wording. macOS names the chord, exactly as its Disconnect button does;
|
||||
/// iOS/iPadOS spells the action out (the HUD's buttons there carry no shortcuts, even where a
|
||||
/// hardware keyboard could fire one — the Stream menu is that keyboard's surface).
|
||||
private var micButtonTitle: String {
|
||||
#if os(macOS)
|
||||
return model.micMuted ? "Unmute Mic (⌃⌥⇧A)" : "Mute Mic (⌃⌥⇧A)"
|
||||
#else
|
||||
return model.micMuted ? "Unmute Microphone" : "Mute Microphone"
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
// MARK: - Card metrics
|
||||
|
||||
/// The card's inner content padding. Roomier on tvOS — the stat text auto-scales for the
|
||||
@@ -242,6 +267,42 @@ struct StreamHUDView: View {
|
||||
}
|
||||
}
|
||||
|
||||
#if !os(tvOS)
|
||||
/// The muted-microphone badge — the mute STATE, as opposed to the buttons that flip it. It rides
|
||||
/// over the stream whenever the mic is muted, INDEPENDENT of the stats overlay (which the user
|
||||
/// may have cycled off, and which the compact tier reduces to a stat line): "am I muted?" is not a
|
||||
/// statistic, and a mute you can't see is how people talk to nobody for a minute. Same glass
|
||||
/// language as the HUD, sized like the start-of-stream banner it shares the bottom edge with.
|
||||
///
|
||||
/// It is also a control: tapping it unmutes. That is the guaranteed way back for a touch user who
|
||||
/// muted with the overlay off, and it costs the badge nothing (it is on screen either way).
|
||||
struct MicMutedBadge: View {
|
||||
let onUnmute: () -> Void
|
||||
|
||||
var body: some View {
|
||||
Button(action: onUnmute) {
|
||||
HStack(spacing: 7) {
|
||||
Image(systemName: "mic.slash.fill")
|
||||
.font(.system(size: 13, weight: .semibold))
|
||||
.foregroundStyle(.red)
|
||||
Text("Microphone muted")
|
||||
.font(.geist(12, .medium, relativeTo: .caption))
|
||||
.foregroundStyle(.white.opacity(0.9))
|
||||
}
|
||||
.padding(.horizontal, 14)
|
||||
.padding(.vertical, 8)
|
||||
// interactive: the badge IS the tap target, so the glass reacts to press.
|
||||
.glassBackground(Capsule(), interactive: true)
|
||||
.contentShape(Capsule())
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.environment(\.colorScheme, .dark) // reads over any frame, like the resize overlay
|
||||
.accessibilityLabel("Microphone muted")
|
||||
.accessibilityHint("Unmutes the microphone")
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if os(iOS)
|
||||
/// Device display geometry the overlay needs but UIKit doesn't expose publicly.
|
||||
enum DeviceMetrics {
|
||||
|
||||
@@ -32,6 +32,7 @@ struct GamepadSettingsView: View {
|
||||
@AppStorage(DefaultsKey.enable444) private var enable444 = false
|
||||
@AppStorage(DefaultsKey.codec) private var codec = "auto"
|
||||
@AppStorage(DefaultsKey.micEnabled) private var micEnabled = true
|
||||
@AppStorage(DefaultsKey.echoCancel) private var echoCancel = true
|
||||
// The overlay tier's raw string (rows tag by rawValue); the absent-key default runs the
|
||||
// legacy-hudEnabled migration (same pattern as ContentView/SettingsView).
|
||||
@AppStorage(DefaultsKey.statsVerbosity) private var statsVerbosityRaw
|
||||
@@ -316,6 +317,11 @@ struct GamepadSettingsView: View {
|
||||
id: "mic", icon: "mic", label: "Microphone",
|
||||
detail: "Send this device's microphone to the host's virtual mic.",
|
||||
value: $micEnabled),
|
||||
toggleRow(
|
||||
id: "echoCancel", icon: "waveform", label: "Echo cancellation",
|
||||
detail: "Cancel the audio this device plays out of the mic signal — stops "
|
||||
+ "speaker setups feeding the game back to the host.",
|
||||
value: $echoCancel),
|
||||
|
||||
choiceRow(
|
||||
id: "pad", header: "Controller", icon: "gamecontroller", label: "Use controller",
|
||||
|
||||
@@ -98,6 +98,10 @@ enum SettingsFields {
|
||||
.init(name: "mic_enabled", key: DefaultsKey.micEnabled,
|
||||
overlay: \.micEnabled, effective: \.micEnabled)
|
||||
}
|
||||
static var echoCancel: SettingsField<Bool> {
|
||||
.init(name: "echo_cancel", key: DefaultsKey.echoCancel,
|
||||
overlay: \.echoCancel, effective: \.echoCancel)
|
||||
}
|
||||
static var touchMode: SettingsField<String> {
|
||||
.init(name: "touch_mode", key: DefaultsKey.touchMode,
|
||||
overlay: \.touchMode, effective: \.touchMode)
|
||||
@@ -175,6 +179,7 @@ extension SettingsView {
|
||||
base.compositor = compositor
|
||||
base.audioChannels = audioChannels
|
||||
base.micEnabled = micEnabled
|
||||
base.echoCancel = echoCancel
|
||||
base.gamepadType = gamepadType
|
||||
base.statsVerbosity = statsVerbosityRaw
|
||||
base.fullscreenWhileStreaming = fullscreenWhileStreaming
|
||||
|
||||
@@ -581,6 +581,10 @@ extension SettingsView {
|
||||
field: "mic_enabled") {
|
||||
Toggle("Send microphone to the host", isOn: scoped(SettingsFields.micEnabled))
|
||||
}
|
||||
described(echoCancelCaption, field: "echo_cancel") {
|
||||
Toggle("Echo cancellation", isOn: scoped(SettingsFields.echoCancel))
|
||||
.disabled(!effective.micEnabled)
|
||||
}
|
||||
#if os(macOS)
|
||||
if !inProfileScope {
|
||||
Picker("Microphone", selection: $micUID) {
|
||||
@@ -619,6 +623,20 @@ extension SettingsView {
|
||||
}
|
||||
}
|
||||
|
||||
/// Honest about the macOS escape hatch: the voice processor only follows the system
|
||||
/// default devices, so hand-picked endpoints silently keep the raw path (see
|
||||
/// SessionAudio's topology note) — better said here than discovered mid-call.
|
||||
private var echoCancelCaption: String {
|
||||
let base = "Voice processing cancels the audio this device plays out of the mic "
|
||||
+ "signal, so a speaker setup doesn't feed the game back to the host."
|
||||
#if os(macOS)
|
||||
return base + " Follows the system default devices — a hand-picked speaker, "
|
||||
+ "microphone or input channel streams the raw mic instead."
|
||||
#else
|
||||
return base
|
||||
#endif
|
||||
}
|
||||
|
||||
// MARK: - Controllers
|
||||
|
||||
@ViewBuilder var controllersSection: some View {
|
||||
|
||||
@@ -65,6 +65,7 @@ struct SettingsView: View {
|
||||
@AppStorage(DefaultsKey.libraryEnabled) var libraryEnabled = true
|
||||
@AppStorage(DefaultsKey.fullscreenWhileStreaming) var fullscreenWhileStreaming = true
|
||||
@AppStorage(DefaultsKey.micEnabled) var micEnabled = true
|
||||
@AppStorage(DefaultsKey.echoCancel) var echoCancel = true
|
||||
@AppStorage(DefaultsKey.audioChannels) var audioChannels = 2
|
||||
@AppStorage(DefaultsKey.codec) var codec = "auto"
|
||||
// The overlay tier's raw string (the pickers tag by rawValue); the absent-key default runs
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
// Opus ⇄ PCM through CoreAudio's built-in codec (kAudioFormatOpus, macOS 10.13+ / iOS
|
||||
// 11+) — no bundled libopus. The host's audio plane is raw Opus packets (48 kHz stereo,
|
||||
// one frame per packet); AVAudioConverter handles them as single-packet
|
||||
// AVAudioCompressedBuffers with explicit packet descriptions.
|
||||
// one frame per packet); the mic uplink is 48 kHz MONO packets (one microphone bus —
|
||||
// the host's decoder upmixes, so duplicating it into a second channel only cost bits).
|
||||
// AVAudioConverter handles both as single-packet AVAudioCompressedBuffers with explicit
|
||||
// packet descriptions.
|
||||
//
|
||||
// Both classes are single-threaded by contract (one per direction, owned by their
|
||||
// drain/capture pipelines).
|
||||
@@ -14,16 +16,16 @@ enum OpusCodecError: Error {
|
||||
case convertFailed(String)
|
||||
}
|
||||
|
||||
/// 48 kHz stereo float32 interleaved — the PCM side of both converters and the layout
|
||||
/// of the playback ring buffer.
|
||||
/// 48 kHz stereo float32 interleaved — the decoder's PCM side (the host plane's shape).
|
||||
func opusPCMFormat() -> AVAudioFormat? {
|
||||
AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32, sampleRate: 48_000, channels: 2, interleaved: true)
|
||||
}
|
||||
|
||||
/// The compressed side: raw Opus, `framesPerPacket` nominal samples per packet at 48 kHz
|
||||
/// (240 = the host's 5 ms audio plane; 960 = the 20 ms packets the encoder emits).
|
||||
private func opusFormat(framesPerPacket: UInt32) -> AVAudioFormat? {
|
||||
/// (240 = the host's 5 ms audio plane; 480 = the 10 ms packets the encoder emits) and
|
||||
/// `channels` (2 = the host plane, 1 = the mic uplink).
|
||||
private func opusFormat(framesPerPacket: UInt32, channels: UInt32) -> AVAudioFormat? {
|
||||
var desc = AudioStreamBasicDescription(
|
||||
mSampleRate: 48_000,
|
||||
mFormatID: kAudioFormatOpus,
|
||||
@@ -31,7 +33,7 @@ private func opusFormat(framesPerPacket: UInt32) -> AVAudioFormat? {
|
||||
mBytesPerPacket: 0,
|
||||
mFramesPerPacket: framesPerPacket,
|
||||
mBytesPerFrame: 0,
|
||||
mChannelsPerFrame: 2,
|
||||
mChannelsPerFrame: channels,
|
||||
mBitsPerChannel: 0,
|
||||
mReserved: 0)
|
||||
return AVAudioFormat(streamDescription: &desc)
|
||||
@@ -45,7 +47,8 @@ final class OpusDecoder {
|
||||
|
||||
/// `framesPerPacket`: the sender's packet duration in samples (host audio = 240).
|
||||
init(framesPerPacket: UInt32) throws {
|
||||
guard let pcm = opusPCMFormat(), let opus = opusFormat(framesPerPacket: framesPerPacket),
|
||||
guard let pcm = opusPCMFormat(),
|
||||
let opus = opusFormat(framesPerPacket: framesPerPacket, channels: 2),
|
||||
let converter = AVAudioConverter(from: opus, to: pcm)
|
||||
else { throw OpusCodecError.unavailable }
|
||||
self.converter = converter
|
||||
@@ -90,24 +93,43 @@ final class OpusDecoder {
|
||||
}
|
||||
|
||||
final class OpusEncoder {
|
||||
/// The encoder's packet duration: 960 samples = 20 ms, CoreAudio's default Opus
|
||||
/// framing. The host's mic service decodes any Opus frame size up to 120 ms.
|
||||
static let framesPerPacket: AVAudioFrameCount = 960
|
||||
/// The encoder's packet duration in samples: 480 = 10 ms, halving the packetization
|
||||
/// latency of the old 20 ms framing. CoreAudio honors it — mFramesPerPacket 480/mono
|
||||
/// creates a converter that truly emits 10 ms CELT packets (TOC config 30), one per
|
||||
/// 480-frame chunk, verified by inspection of the emitted TOC bytes and per-packet
|
||||
/// frame accounting. 960 stays as the fallback should an older codec refuse 480.
|
||||
/// The host's mic service decodes any Opus frame size up to 120 ms, so either is
|
||||
/// wire-compatible.
|
||||
let framesPerPacket: AVAudioFrameCount
|
||||
|
||||
/// 48 kHz MONO float32 interleaved — the uplink carries one microphone bus.
|
||||
let pcmFormat: AVAudioFormat
|
||||
|
||||
private let converter: AVAudioConverter
|
||||
private let outBuf: AVAudioCompressedBuffer
|
||||
let pcmFormat: AVAudioFormat
|
||||
|
||||
init() throws {
|
||||
guard let pcm = opusPCMFormat(),
|
||||
let opus = opusFormat(framesPerPacket: UInt32(Self.framesPerPacket)),
|
||||
let converter = AVAudioConverter(from: pcm, to: opus)
|
||||
guard let pcm = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32, sampleRate: 48_000, channels: 1,
|
||||
interleaved: true)
|
||||
else { throw OpusCodecError.unavailable }
|
||||
converter.bitRate = 96_000
|
||||
self.converter = converter
|
||||
self.pcmFormat = pcm
|
||||
var made: (converter: AVAudioConverter, fpp: AVAudioFrameCount)?
|
||||
for fpp: AVAudioFrameCount in [480, 960] {
|
||||
if let opus = opusFormat(framesPerPacket: UInt32(fpp), channels: 1),
|
||||
let converter = AVAudioConverter(from: pcm, to: opus) {
|
||||
made = (converter, fpp)
|
||||
break
|
||||
}
|
||||
}
|
||||
guard let made else { throw OpusCodecError.unavailable }
|
||||
// 48 kbps: transparent for mono voice — the old 96 kbps budget was sized for
|
||||
// the duplicated-stereo framing this encoder no longer emits.
|
||||
made.converter.bitRate = 48_000
|
||||
converter = made.converter
|
||||
framesPerPacket = made.fpp
|
||||
pcmFormat = pcm
|
||||
outBuf = AVAudioCompressedBuffer(
|
||||
format: opus, packetCapacity: 4, maximumPacketSize: 1500)
|
||||
format: made.converter.outputFormat, packetCapacity: 4, maximumPacketSize: 1500)
|
||||
}
|
||||
|
||||
/// Encode exactly `framesPerPacket` frames of `pcmFormat` audio; returns the encoded
|
||||
|
||||
@@ -5,15 +5,22 @@
|
||||
// AVAudioSourceNode pulls from the ring (silence on underrun with re-priming, so a
|
||||
// network gap costs one dip, not permanent crackle).
|
||||
//
|
||||
// mic → host: a second AVAudioEngine taps the input device, folds it to one mono bus (the
|
||||
// chosen channel of a multi-channel interface, or a sum of all channels), resamples to 48 kHz
|
||||
// stereo, slices 20 ms chunks, Opus-encodes, and sendMic()s each packet — the host feeds them
|
||||
// into a virtual PipeWire source.
|
||||
// mic → host: a tap on the input node folds the capture to one mono bus (the chosen channel
|
||||
// of a multi-channel interface, or a sum of all channels), resamples to 48 kHz mono, slices
|
||||
// 10 ms chunks, Opus-encodes, and sendMic()s each packet — the host feeds them into a
|
||||
// virtual PipeWire source.
|
||||
//
|
||||
// Engine topology. With the mic enabled and echo cancellation on (both defaults), BOTH
|
||||
// directions run on ONE AVAudioEngine with the system voice processor engaged
|
||||
// (`setVoiceProcessingEnabled`) — AEC needs render and capture on the same unit so it can
|
||||
// subtract what the speaker is playing from what the mic hears; without it, a loudspeaker
|
||||
// client feeds the host's own game audio straight back to it (the primary reported echo
|
||||
// source). The voice processor can only follow the system DEFAULT devices, so explicit
|
||||
// endpoint choices fall back to the old two-engine topology — see `wantsCombined` for the
|
||||
// exact decision, and `startCapture` for why two engines handle arbitrary device pairs.
|
||||
//
|
||||
// Devices are chosen by UID ("" = system default: the engine is then never pinned to a
|
||||
// concrete device and follows default-device changes). Two engines, not one — a single
|
||||
// AVAudioEngine ties input+output to one aggregate clock, separate engines keep
|
||||
// arbitrary mic/speaker combinations trivial.
|
||||
// concrete device and follows default-device changes).
|
||||
|
||||
import AVFoundation
|
||||
import os
|
||||
@@ -40,7 +47,21 @@ public final class SessionAudio {
|
||||
private let stateLock = NSLock()
|
||||
private var playbackEngine: AVAudioEngine?
|
||||
private var captureEngine: AVAudioEngine?
|
||||
/// The one engine running BOTH directions when the voice processor is engaged;
|
||||
/// `playbackEngine`/`captureEngine` stay nil while this is set.
|
||||
private var combinedEngine: AVAudioEngine?
|
||||
private var drainStarted = false
|
||||
/// The mute LATCH: the effective mute the owner last asked for (see `setMicMuted`). Held
|
||||
/// because the uplink engine can appear LATER than the request — the mic permission prompt
|
||||
/// is answered at the user's leisure, and the engine is built on the grant — so a mute set
|
||||
/// in the meantime must be waiting for it. Applied by whichever start path wins the race.
|
||||
/// Guarded by `stateLock`, like the engines it applies to.
|
||||
private var micMuted = false
|
||||
/// The playback jitter ring — created by whichever engine starts playback first and KEPT
|
||||
/// across an engine rebuild (the permission-grant upgrade in `startEngines` swaps engines,
|
||||
/// not the ring, so the drain thread never has to be re-pointed). Main-thread confined,
|
||||
/// like every start path.
|
||||
private var ring: AudioRing?
|
||||
#if !os(macOS)
|
||||
/// AVAudioSession `setCategory`/`setActive` are synchronous and block on the audio server, so
|
||||
/// they must not run on the main thread (UI stall — AVFoundation warns about it). PROCESS-WIDE
|
||||
@@ -69,11 +90,15 @@ public final class SessionAudio {
|
||||
/// ASYNCHRONOUS: it activates the AVAudioSession off the main thread, then starts the engines on
|
||||
/// a later main-queue hop (gated by `!flag.isStopped`) — so playback is live shortly after, not
|
||||
/// on return. The mic may start later still if the permission prompt is pending.
|
||||
public func start(speakerUID: String, micUID: String, micChannel: Int, micEnabled: Bool) {
|
||||
/// `echoCancel` picks the engine topology — see the header note and `wantsCombined`.
|
||||
public func start(
|
||||
speakerUID: String, micUID: String, micChannel: Int, micEnabled: Bool, echoCancel: Bool
|
||||
) {
|
||||
#if os(macOS)
|
||||
// No AVAudioSession on macOS — start the engines directly (caller's thread, as before).
|
||||
startEngines(
|
||||
speakerUID: speakerUID, micUID: micUID, micChannel: micChannel, micEnabled: micEnabled)
|
||||
speakerUID: speakerUID, micUID: micUID, micChannel: micChannel,
|
||||
micEnabled: micEnabled, echoCancel: echoCancel)
|
||||
#else
|
||||
// Configure + activate the session OFF the main thread (it blocks on the audio server),
|
||||
// then start the engines back on the main thread once it's active — engine routing/format
|
||||
@@ -85,7 +110,7 @@ public final class SessionAudio {
|
||||
guard let self, !self.flag.isStopped else { return }
|
||||
self.startEngines(
|
||||
speakerUID: speakerUID, micUID: micUID, micChannel: micChannel,
|
||||
micEnabled: micEnabled)
|
||||
micEnabled: micEnabled, echoCancel: echoCancel)
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -104,6 +129,12 @@ public final class SessionAudio {
|
||||
try session.setCategory(
|
||||
.playAndRecord, mode: .default,
|
||||
options: [.allowBluetoothA2DP, .defaultToSpeaker])
|
||||
// Uplink latency: ask for 5 ms IO quanta at the wire rate (the default ~10-23 ms
|
||||
// quantum is most of the mic path's burst latency). Best-effort — the hardware
|
||||
// has the final word (a Bluetooth route will ignore both), and whatever quantum
|
||||
// is actually granted, the capture tap handles the buffers it gets.
|
||||
try? session.setPreferredIOBufferDuration(0.005)
|
||||
try? session.setPreferredSampleRate(48_000)
|
||||
} else {
|
||||
try session.setCategory(.playback, mode: .default)
|
||||
}
|
||||
@@ -117,32 +148,80 @@ public final class SessionAudio {
|
||||
}
|
||||
#endif
|
||||
|
||||
/// Build + start the playback engine (and the mic uplink when enabled + authorized). Main
|
||||
/// thread (engine setup); on iOS/tvOS the session is already active by the time this runs.
|
||||
/// Build + start the engines — combined (voice-processed) or split, per `wantsCombined` —
|
||||
/// with the mic uplink only when enabled + authorized. Main thread (engine setup); on
|
||||
/// iOS/tvOS the session is already active by the time this runs.
|
||||
private func startEngines(
|
||||
speakerUID: String, micUID: String, micChannel: Int, micEnabled: Bool
|
||||
speakerUID: String, micUID: String, micChannel: Int, micEnabled: Bool, echoCancel: Bool
|
||||
) {
|
||||
startPlayback(speakerUID: speakerUID)
|
||||
#if os(tvOS)
|
||||
// No app-accessible microphone input on tvOS — playback only.
|
||||
startPlayback(speakerUID: speakerUID)
|
||||
#else
|
||||
guard micEnabled else { return }
|
||||
guard micEnabled else {
|
||||
startPlayback(speakerUID: speakerUID)
|
||||
return
|
||||
}
|
||||
let combined = wantsCombined(
|
||||
speakerUID: speakerUID, micUID: micUID, micChannel: micChannel,
|
||||
echoCancel: echoCancel)
|
||||
switch AVCaptureDevice.authorizationStatus(for: .audio) {
|
||||
case .authorized:
|
||||
startCapture(micUID: micUID, micChannel: micChannel)
|
||||
if combined {
|
||||
startCombined(speakerUID: speakerUID, micUID: micUID, micChannel: micChannel)
|
||||
} else {
|
||||
startPlayback(speakerUID: speakerUID)
|
||||
startCapture(micUID: micUID, micChannel: micChannel)
|
||||
}
|
||||
case .notDetermined:
|
||||
// Playback must not wait out the permission prompt (the user answers at their
|
||||
// leisure) — start it now, and on a grant either bolt the capture engine on
|
||||
// (split) or swap the playback engine for the combined one (the ring and its
|
||||
// drain thread carry over — see `makePlaybackChain`).
|
||||
startPlayback(speakerUID: speakerUID)
|
||||
AVCaptureDevice.requestAccess(for: .audio) { [weak self] granted in
|
||||
DispatchQueue.main.async {
|
||||
guard let self, granted, !self.flag.isStopped else { return }
|
||||
self.startCapture(micUID: micUID, micChannel: micChannel)
|
||||
if combined {
|
||||
self.stateLock.lock()
|
||||
let playback = self.playbackEngine
|
||||
self.playbackEngine = nil
|
||||
self.stateLock.unlock()
|
||||
playback?.stop()
|
||||
self.startCombined(
|
||||
speakerUID: speakerUID, micUID: micUID, micChannel: micChannel)
|
||||
} else {
|
||||
self.startCapture(micUID: micUID, micChannel: micChannel)
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
startPlayback(speakerUID: speakerUID)
|
||||
log.warning("microphone access denied — mic uplink disabled (System Settings → Privacy)")
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
#if !os(tvOS)
|
||||
/// One engine or two: the voice processor requires render + capture on one unit, and that
|
||||
/// unit can only follow the system DEFAULT devices — so echo cancellation gets the combined
|
||||
/// engine only while nothing is explicitly pinned. On macOS a chosen speaker/mic UID or a
|
||||
/// picked input channel (the voice processor's capture side is its own mono mix — a
|
||||
/// per-channel pick can't survive it) keeps today's two-engine path, AEC-less but honoring
|
||||
/// the exact endpoints the user named. On iOS routes are session-managed and the UIDs are
|
||||
/// ignored, so the toggle alone decides.
|
||||
private func wantsCombined(
|
||||
speakerUID: String, micUID: String, micChannel: Int, echoCancel: Bool
|
||||
) -> Bool {
|
||||
guard echoCancel else { return false }
|
||||
#if os(macOS)
|
||||
return speakerUID.isEmpty && micUID.isEmpty && micChannel == 0
|
||||
#else
|
||||
return true
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
/// Stop both directions. Safe from any thread; waits the drain thread out (≤ its
|
||||
/// poll timeout) so the caller can close the connection right after.
|
||||
public func stop() {
|
||||
@@ -152,6 +231,8 @@ public final class SessionAudio {
|
||||
captureEngine = nil
|
||||
let playback = playbackEngine
|
||||
playbackEngine = nil
|
||||
let combined = combinedEngine
|
||||
combinedEngine = nil
|
||||
let wasDraining = drainStarted
|
||||
drainStarted = false
|
||||
stateLock.unlock()
|
||||
@@ -160,6 +241,10 @@ public final class SessionAudio {
|
||||
capture.stop()
|
||||
}
|
||||
playback?.stop()
|
||||
if let combined {
|
||||
combined.inputNode.removeTap(onBus: 0)
|
||||
combined.stop()
|
||||
}
|
||||
#if !os(macOS)
|
||||
// Release the session so audio we interrupted (Music, podcasts) gets its resume cue. Like
|
||||
// activation, setActive is synchronous/blocking — run it on the shared serial session queue
|
||||
@@ -180,15 +265,38 @@ public final class SessionAudio {
|
||||
}
|
||||
}
|
||||
|
||||
/// Background keep-alive: silence the mic uplink while backgrounded (privacy — no room audio
|
||||
/// leaves the device) and restore it on return. Pauses/resumes the capture engine; a no-op when
|
||||
/// there's no uplink (playback-only / tvOS / mic disabled). The audio SESSION stays active for
|
||||
/// background playback, so iOS may keep showing the recording indicator until a full reconfigure
|
||||
/// — this stops the actual capture, which is the privacy-relevant part. Main thread.
|
||||
/// Silence the mic uplink (no room audio leaves the device) or restore it. THE one muting
|
||||
/// mechanism: the owner composes its reasons — the user's in-stream mute and the background
|
||||
/// keep-alive's privacy mute — into one effective state and passes that here, so neither can
|
||||
/// clear the other (see `SessionModel.applyMicMute`).
|
||||
///
|
||||
/// Two-engine sessions pause/resume the capture engine; a combined session instead mutes the
|
||||
/// voice processor's input (playback shares that engine and must keep running, so the engine
|
||||
/// itself never pauses — the mute zeroes the mic at the IO unit, and the tap encodes silence).
|
||||
/// Local and instant either way: nothing is negotiated with the host, and the packets that do
|
||||
/// leave carry silence. A no-op when there's no uplink (playback-only / tvOS / mic disabled),
|
||||
/// except that the state is LATCHED for an uplink that starts later. The audio SESSION stays
|
||||
/// active for background playback, so iOS may keep showing the recording indicator until a
|
||||
/// full reconfigure — either path stops room audio leaving the device, which is the
|
||||
/// privacy-relevant part. Main thread.
|
||||
public func setMicMuted(_ muted: Bool) {
|
||||
stateLock.lock()
|
||||
micMuted = muted
|
||||
let capture = captureEngine
|
||||
let combined = combinedEngine
|
||||
stateLock.unlock()
|
||||
apply(micMuted: muted, capture: capture, combined: combined)
|
||||
}
|
||||
|
||||
/// Push the latched mute onto whichever engine carries the uplink. Split out from
|
||||
/// `setMicMuted` because the start paths call it too, with the engine they just started —
|
||||
/// that's how a mute requested before the permission grant lands on the engine the grant
|
||||
/// creates. Never resumes a stopped session's engine.
|
||||
private func apply(micMuted muted: Bool, capture: AVAudioEngine?, combined: AVAudioEngine?) {
|
||||
if let combined {
|
||||
combined.inputNode.isVoiceProcessingInputMuted = muted
|
||||
return
|
||||
}
|
||||
guard let capture else { return }
|
||||
if muted {
|
||||
capture.pause()
|
||||
@@ -199,28 +307,21 @@ public final class SessionAudio {
|
||||
|
||||
// MARK: - Playback (host → speaker)
|
||||
|
||||
private func startPlayback(speakerUID: String) {
|
||||
/// The playback jitter ring + the source node draining it — shared by the plain playback
|
||||
/// engine and the combined voice-processing engine, and REUSED across an engine rebuild
|
||||
/// (same session, same ring: the drain thread keeps writing right through the swap). nil
|
||||
/// when the host's channel layout can't be expressed (already logged). Main thread.
|
||||
private func makePlaybackChain()
|
||||
-> (ring: AudioRing, source: AVAudioSourceNode, format: AVAudioFormat)?
|
||||
{
|
||||
// Build the playback layout from the host-RESOLVED channel count (never the request):
|
||||
// 2 = stereo / 6 = 5.1 / 8 = 7.1, canonical wire order FL FR FC LFE RL RR SL SR.
|
||||
let channels = Int(connection.resolvedAudioChannels)
|
||||
// 1 s interleaved capacity, ~20 ms prefill (four 5 ms host packets of jitter absorption
|
||||
// before the first sample plays), both scaled by the channel count.
|
||||
let ring = AudioRing(
|
||||
let ring = self.ring ?? AudioRing(
|
||||
capacity: 48_000 * channels, prefill: 960 * channels, channels: channels)
|
||||
|
||||
let engine = AVAudioEngine()
|
||||
#if os(macOS)
|
||||
if !speakerUID.isEmpty {
|
||||
if let dev = AudioDevices.deviceID(forUID: speakerUID),
|
||||
let unit = engine.outputNode.audioUnit {
|
||||
if !Self.setDevice(dev, on: unit) {
|
||||
log.error("could not select speaker \(speakerUID) — using default")
|
||||
}
|
||||
} else {
|
||||
log.warning("speaker \(speakerUID) not present — using default")
|
||||
}
|
||||
}
|
||||
#endif
|
||||
self.ring = ring
|
||||
|
||||
// Engine-native deinterleaved float; the render block deinterleaves from the ring. Surround
|
||||
// uses an explicit wire-order channel layout; the mixer downmixes to the output device when
|
||||
@@ -234,7 +335,7 @@ public final class SessionAudio {
|
||||
}
|
||||
guard let format else {
|
||||
log.error("could not build \(channels)-channel audio format — audio disabled")
|
||||
return
|
||||
return nil
|
||||
}
|
||||
let scratch = ScratchBuffer() // block-owned; freed with the closure
|
||||
let source = AVAudioSourceNode(format: format) { _, _, frameCount, abl -> OSStatus in
|
||||
@@ -252,6 +353,24 @@ public final class SessionAudio {
|
||||
}
|
||||
return noErr
|
||||
}
|
||||
return (ring, source, format)
|
||||
}
|
||||
|
||||
private func startPlayback(speakerUID: String) {
|
||||
guard let (ring, source, format) = makePlaybackChain() else { return }
|
||||
let engine = AVAudioEngine()
|
||||
#if os(macOS)
|
||||
if !speakerUID.isEmpty {
|
||||
if let dev = AudioDevices.deviceID(forUID: speakerUID),
|
||||
let unit = engine.outputNode.audioUnit {
|
||||
if !Self.setDevice(dev, on: unit) {
|
||||
log.error("could not select speaker \(speakerUID) — using default")
|
||||
}
|
||||
} else {
|
||||
log.warning("speaker \(speakerUID) not present — using default")
|
||||
}
|
||||
}
|
||||
#endif
|
||||
engine.attach(source)
|
||||
engine.connect(source, to: engine.mainMixerNode, format: format)
|
||||
engine.prepare()
|
||||
@@ -272,8 +391,14 @@ public final class SessionAudio {
|
||||
startDrain(into: ring)
|
||||
}
|
||||
|
||||
/// Idempotent — the permission-grant engine swap reaches here a second time with the
|
||||
/// drain thread already feeding the (carried-over) ring.
|
||||
private func startDrain(into ring: AudioRing) {
|
||||
stateLock.lock()
|
||||
if drainStarted {
|
||||
stateLock.unlock()
|
||||
return
|
||||
}
|
||||
drainStarted = true
|
||||
stateLock.unlock()
|
||||
let thread = Thread { [connection, flag, drainDone] in
|
||||
@@ -308,6 +433,80 @@ public final class SessionAudio {
|
||||
// MARK: - Mic (mic → host)
|
||||
|
||||
#if !os(tvOS)
|
||||
/// One engine, both directions: engage the system voice processor on the shared IO unit
|
||||
/// (AEC + noise suppression + AGC), hang the playback source off its render side and the
|
||||
/// mic tap off its capture side. Every failure falls back to a WORKING configuration —
|
||||
/// the split path (no AEC) when the voice processor won't engage, plain playback when the
|
||||
/// mic chain can't be built — a session never loses audio to the echo-cancel feature.
|
||||
private func startCombined(speakerUID: String, micUID: String, micChannel: Int) {
|
||||
let engine = AVAudioEngine()
|
||||
let input = engine.inputNode
|
||||
do {
|
||||
// Before anything reads the input's format: the voice processor changes it (often
|
||||
// to its own mono mix, sometimes at a lower rate) — installMicTap reads the format
|
||||
// AFTER this, so the converter chain adapts to whatever the processor emits.
|
||||
try input.setVoiceProcessingEnabled(true)
|
||||
} catch {
|
||||
log.warning("""
|
||||
voice processing unavailable (\(error.localizedDescription)) — separate \
|
||||
engines, no echo cancellation
|
||||
""")
|
||||
startPlayback(speakerUID: speakerUID)
|
||||
startCapture(micUID: micUID, micChannel: micChannel)
|
||||
return
|
||||
}
|
||||
// Symmetric enable for the render side; with both directions on one engine the
|
||||
// input-node enable already covers it, so a refusal here is not a failure.
|
||||
try? engine.outputNode.setVoiceProcessingEnabled(true)
|
||||
// This is a game stream, not a call: never duck the host's audio under the outgoing
|
||||
// voice. .min is the closest to "off" the API offers, and advanced (selective)
|
||||
// ducking stays off with it.
|
||||
input.voiceProcessingOtherAudioDuckingConfiguration = .init(
|
||||
enableAdvancedDucking: false, duckingLevel: .min)
|
||||
|
||||
guard let (ring, source, format) = makePlaybackChain() else {
|
||||
// Playback impossible (logged) — keep the uplink alive, as the split path would.
|
||||
startCapture(micUID: micUID, micChannel: micChannel)
|
||||
return
|
||||
}
|
||||
engine.attach(source)
|
||||
engine.connect(source, to: engine.mainMixerNode, format: format)
|
||||
guard installMicTap(on: input, micUID: micUID, micChannel: micChannel) else {
|
||||
// Mic chain unavailable (logged) — keep the session audible on the plain playback
|
||||
// engine rather than playing through an idle voice processor.
|
||||
startPlayback(speakerUID: speakerUID)
|
||||
return
|
||||
}
|
||||
engine.prepare()
|
||||
do {
|
||||
try engine.start()
|
||||
} catch {
|
||||
log.error("combined engine failed to start: \(error.localizedDescription)")
|
||||
input.removeTap(onBus: 0)
|
||||
startPlayback(speakerUID: speakerUID) // no echo cancellation beats no audio
|
||||
return
|
||||
}
|
||||
stateLock.lock()
|
||||
if flag.isStopped {
|
||||
stateLock.unlock()
|
||||
input.removeTap(onBus: 0)
|
||||
engine.stop() // stop() already ran — don't strand a started engine (or a hot mic)
|
||||
return
|
||||
}
|
||||
combinedEngine = engine
|
||||
let muted = micMuted // latched before this engine existed (a mute during the prompt)
|
||||
stateLock.unlock()
|
||||
apply(micMuted: muted, capture: nil, combined: engine)
|
||||
startDrain(into: ring)
|
||||
log.info("audio engines joined — voice processing (echo cancellation) active")
|
||||
}
|
||||
|
||||
/// The split path: capture on its OWN engine, playback on another — the pre-echo-cancel
|
||||
/// topology, kept verbatim. Two engines, not one — a single AVAudioEngine ties
|
||||
/// input+output to one aggregate clock, separate engines keep arbitrary mic/speaker
|
||||
/// combinations trivial. That freedom is exactly why the voice processor can't ride this
|
||||
/// path (AEC needs both directions on one unit) and why explicitly pinned endpoints land
|
||||
/// here — see `wantsCombined`.
|
||||
private func startCapture(micUID: String, micChannel: Int) {
|
||||
let engine = AVAudioEngine()
|
||||
let input = engine.inputNode
|
||||
@@ -322,11 +521,44 @@ public final class SessionAudio {
|
||||
}
|
||||
}
|
||||
#endif
|
||||
guard installMicTap(on: input, micUID: micUID, micChannel: micChannel) else { return }
|
||||
engine.prepare()
|
||||
do {
|
||||
try engine.start()
|
||||
} catch {
|
||||
log.error("capture engine failed to start: \(error.localizedDescription)")
|
||||
input.removeTap(onBus: 0)
|
||||
return
|
||||
}
|
||||
stateLock.lock()
|
||||
if flag.isStopped {
|
||||
// stop() ran while we were starting (the permission prompt resolves at the
|
||||
// user's leisure) — tear the engine down ourselves, nobody else owns it now.
|
||||
stateLock.unlock()
|
||||
input.removeTap(onBus: 0)
|
||||
engine.stop()
|
||||
return
|
||||
}
|
||||
captureEngine = engine
|
||||
let muted = micMuted // latched before this engine existed (a mute during the prompt)
|
||||
stateLock.unlock()
|
||||
apply(micMuted: muted, capture: engine, combined: nil)
|
||||
log.info("mic uplink started (\(micUID.isEmpty ? "default input" : micUID))")
|
||||
}
|
||||
|
||||
/// Resolve the input's live format + fold plan, build the mono→Opus chain, and install the
|
||||
/// capture tap on `input` — everything mic except engine ownership, shared verbatim by the
|
||||
/// combined and split topologies. Reads `input.outputFormat(forBus:)` at call time, so the
|
||||
/// chain follows whatever the node emits: the raw device format, or the voice processor's
|
||||
/// own mix when that's enabled. False (logged) when no input is usable or the encoder
|
||||
/// can't be built; the tap is installed on true.
|
||||
private func installMicTap(
|
||||
on input: AVAudioInputNode, micUID: String, micChannel: Int
|
||||
) -> Bool {
|
||||
let inFormat = input.outputFormat(forBus: 0)
|
||||
guard inFormat.sampleRate > 0, inFormat.channelCount > 0 else {
|
||||
log.error("no usable input device — mic uplink disabled")
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
// Multi-channel-interface handling. A pro interface exposes N discrete inputs with the mic
|
||||
@@ -378,22 +610,38 @@ public final class SessionAudio {
|
||||
#endif
|
||||
|
||||
// Encode a single mono bus (folded from `inFormat` in the tap): the resampler goes
|
||||
// mono@inputSR → the encoder's 48 kHz stereo, so it handles both the rate change and the
|
||||
// mono→stereo duplication, and the wrong-channel downmix never happens.
|
||||
// mono@inputSR → the encoder's 48 kHz mono, so it handles the rate change and the
|
||||
// wrong-channel downmix never happens. Mono end to end — the host's decoder upmixes,
|
||||
// so the old duplicate-into-stereo step only cost bits and cycles.
|
||||
//
|
||||
// `mono`/`staging` are the per-callback scratch buffers, preallocated HERE (grown only
|
||||
// if a larger-than-expected device quantum ever arrives) — the steady-state tap path
|
||||
// allocates nothing.
|
||||
let scratchFrames: AVAudioFrameCount = 8192
|
||||
let stagingCapacity = { (frames: AVAudioFrameCount) -> AVAudioFrameCount in
|
||||
AVAudioFrameCount(
|
||||
(Double(frames) * 48_000 / inFormat.sampleRate).rounded(.up)) + 64
|
||||
}
|
||||
guard let monoFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32, sampleRate: inFormat.sampleRate,
|
||||
channels: 1, interleaved: false),
|
||||
let encoder = try? OpusEncoder(),
|
||||
let resampler = AVAudioConverter(from: monoFormat, to: encoder.pcmFormat),
|
||||
let chunk = AVAudioPCMBuffer(
|
||||
pcmFormat: encoder.pcmFormat, frameCapacity: OpusEncoder.framesPerPacket)
|
||||
pcmFormat: encoder.pcmFormat, frameCapacity: encoder.framesPerPacket),
|
||||
let monoScratch = AVAudioPCMBuffer(
|
||||
pcmFormat: monoFormat, frameCapacity: scratchFrames),
|
||||
let stagingScratch = AVAudioPCMBuffer(
|
||||
pcmFormat: encoder.pcmFormat, frameCapacity: stagingCapacity(scratchFrames))
|
||||
else {
|
||||
log.error("Opus encoder unavailable — mic uplink disabled")
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
// Tap-thread-confined state: resample into `staging`, accumulate in `fifo`,
|
||||
// slice 960-frame chunks for the encoder.
|
||||
// Tap-thread-confined state: fold into `mono`, resample into `staging`, accumulate in
|
||||
// `fifo`, slice `framesPerPacket` (10 ms) chunks for the encoder.
|
||||
var mono = monoScratch
|
||||
var staging = stagingScratch
|
||||
var fifo: [Float] = []
|
||||
fifo.reserveCapacity(48_000)
|
||||
var seq: UInt32 = 0
|
||||
@@ -412,14 +660,26 @@ public final class SessionAudio {
|
||||
var inputPeak: Float = 0
|
||||
var levelReported = false
|
||||
|
||||
input.installTap(onBus: 0, bufferSize: 2048, format: inFormat) { buffer, _ in
|
||||
// 480 frames = 10 ms, matching the packet duration. Advisory — CoreAudio delivers the
|
||||
// device quantum whatever we ask (the old 2048 request came back as 42.7 ms bursts, most
|
||||
// of the uplink's latency) — but where the system honors it, the tap fires per-packet.
|
||||
input.installTap(onBus: 0, bufferSize: 480, format: inFormat) { buffer, _ in
|
||||
if flag.isStopped { return }
|
||||
let frames = Int(buffer.frameLength)
|
||||
guard frames > 0, let src = buffer.floatChannelData,
|
||||
let mono = AVAudioPCMBuffer(
|
||||
pcmFormat: monoFormat, frameCapacity: buffer.frameLength),
|
||||
let dst = mono.floatChannelData?[0]
|
||||
else { return }
|
||||
guard frames > 0, let src = buffer.floatChannelData else { return }
|
||||
if frames > Int(mono.frameCapacity) {
|
||||
// A quantum larger than the scratch (bufferSize is advisory both ways) — regrow
|
||||
// once to the new high-water mark; the steady state stays allocation-free.
|
||||
guard let biggerMono = AVAudioPCMBuffer(
|
||||
pcmFormat: monoFormat, frameCapacity: buffer.frameLength),
|
||||
let biggerStaging = AVAudioPCMBuffer(
|
||||
pcmFormat: encoder.pcmFormat,
|
||||
frameCapacity: stagingCapacity(buffer.frameLength))
|
||||
else { return }
|
||||
mono = biggerMono
|
||||
staging = biggerStaging
|
||||
}
|
||||
guard let dst = mono.floatChannelData?[0] else { return }
|
||||
mono.frameLength = buffer.frameLength
|
||||
|
||||
// Fold the multi-channel input down to the one mono bus we encode.
|
||||
@@ -451,11 +711,6 @@ public final class SessionAudio {
|
||||
}
|
||||
}
|
||||
|
||||
let ratio = 48_000 / inFormat.sampleRate
|
||||
let outCapacity = AVAudioFrameCount((Double(frames) * ratio).rounded(.up) + 64)
|
||||
guard let staging = AVAudioPCMBuffer(
|
||||
pcmFormat: encoder.pcmFormat, frameCapacity: outCapacity)
|
||||
else { return }
|
||||
var fed = false
|
||||
var convError: NSError?
|
||||
let status = resampler.convert(to: staging, error: &convError) { _, outStatus in
|
||||
@@ -469,16 +724,20 @@ public final class SessionAudio {
|
||||
}
|
||||
guard status != .error, let p = staging.floatChannelData?[0] else { return }
|
||||
fifo.append(contentsOf: UnsafeBufferPointer(
|
||||
start: p, count: Int(staging.frameLength) * 2))
|
||||
start: p, count: Int(staging.frameLength)))
|
||||
|
||||
let samplesPerChunk = Int(OpusEncoder.framesPerPacket) * 2
|
||||
while fifo.count >= samplesPerChunk {
|
||||
chunk.frameLength = OpusEncoder.framesPerPacket
|
||||
// Consume whole chunks through a head index, then drop the eaten prefix in ONE
|
||||
// move of the sub-chunk remainder. The old per-chunk removeFirst memmoved the
|
||||
// entire backlog for every packet — O(n) on the render-adjacent tap thread.
|
||||
let samplesPerChunk = Int(encoder.framesPerPacket)
|
||||
var head = 0
|
||||
while fifo.count - head >= samplesPerChunk {
|
||||
chunk.frameLength = encoder.framesPerPacket
|
||||
fifo.withUnsafeBufferPointer { src in
|
||||
chunk.floatChannelData![0].update(
|
||||
from: src.baseAddress!, count: samplesPerChunk)
|
||||
from: src.baseAddress! + head, count: samplesPerChunk)
|
||||
}
|
||||
fifo.removeFirst(samplesPerChunk)
|
||||
head += samplesPerChunk
|
||||
guard let packets = try? encoder.encode(chunk) else { continue }
|
||||
for packet in packets {
|
||||
connection.sendMic(
|
||||
@@ -486,28 +745,9 @@ public final class SessionAudio {
|
||||
seq &+= 1
|
||||
}
|
||||
}
|
||||
if head > 0 { fifo.removeFirst(head) } // keeps capacity — no realloc
|
||||
}
|
||||
|
||||
engine.prepare()
|
||||
do {
|
||||
try engine.start()
|
||||
} catch {
|
||||
log.error("capture engine failed to start: \(error.localizedDescription)")
|
||||
input.removeTap(onBus: 0)
|
||||
return
|
||||
}
|
||||
stateLock.lock()
|
||||
if flag.isStopped {
|
||||
// stop() ran while we were starting (the permission prompt resolves at the
|
||||
// user's leisure) — tear the engine down ourselves, nobody else owns it now.
|
||||
stateLock.unlock()
|
||||
input.removeTap(onBus: 0)
|
||||
engine.stop()
|
||||
return
|
||||
}
|
||||
captureEngine = engine
|
||||
stateLock.unlock()
|
||||
log.info("mic uplink started (\(micUID.isEmpty ? "default input" : micUID))")
|
||||
return true
|
||||
}
|
||||
|
||||
/// Fold `channels` of input (`floatChannelData` layout: `interleaved` → one buffer strided by
|
||||
|
||||
@@ -128,11 +128,19 @@ public final class InputCapture {
|
||||
/// carries the same key equivalents for discoverability) can't see them, so the monitor is the
|
||||
/// captured-state delivery path; released, the events pass through and the menu handles them.
|
||||
/// ⌃⌥⇧Q releases the captured mouse/keyboard; ⌃⌥⇧D disconnects; ⌃⌥⇧S cycles the stats
|
||||
/// overlay tier (off → compact → normal → detailed). Main queue.
|
||||
/// overlay tier (off → compact → normal → detailed). ⌃⌥⇧A (`onToggleMicMute`, below) rides
|
||||
/// the same path. Main queue.
|
||||
public var onReleaseCapture: (() -> Void)?
|
||||
public var onDisconnect: (() -> Void)?
|
||||
public var onCycleStats: (() -> Void)?
|
||||
|
||||
/// Fired on ⌃⌥⇧A — mute/unmute the microphone uplink, the one in-stream control a captured
|
||||
/// session can't otherwise reach (the HUD's button is behind a grabbed cursor). Same delivery
|
||||
/// rule as the combos above: only WHILE FORWARDING, because that's when the menu's identical
|
||||
/// key equivalent can't fire. ⌃⌥⇧M — the obvious letter — is long since the mouse-model flip
|
||||
/// (cross-client), so A ("audio in") is the mic's. Main queue.
|
||||
public var onToggleMicMute: (() -> Void)?
|
||||
|
||||
/// Fired on ⌃⌘F (macOS) — toggle the streaming window in/out of fullscreen. Detected in the
|
||||
/// monitor only WHILE FORWARDING, for the same reason as the ⌃⌥⇧ combos: a captured stream view
|
||||
/// swallows keys, so the Stream menu's identical ⌃⌘F equivalent never reaches it; released, the
|
||||
@@ -140,13 +148,13 @@ public final class InputCapture {
|
||||
public var onToggleFullscreen: (() -> Void)?
|
||||
|
||||
#if os(iOS)
|
||||
/// Windows VKs of the three modifier classes in the ⌃⌥⇧Q release chord, both L/R sides:
|
||||
/// Windows VKs of the three modifier classes in the ⌃⌥⇧ chords, both L/R sides:
|
||||
/// control (0xA2/0xA3), option (0xA4/0xA5), shift (0xA0/0xA1). Used to sift the HID key stream.
|
||||
private static let chordModifierVKs: Set<UInt32> = [0xA2, 0xA3, 0xA4, 0xA5, 0xA0, 0xA1]
|
||||
|
||||
/// Whether Control AND Option AND Shift are all currently held (either side of each counts) —
|
||||
/// the modifier precondition for the iPad ⌃⌥⇧Q release chord.
|
||||
private var hasReleaseChordModifiers: Bool {
|
||||
/// the modifier precondition for the iPad ⌃⌥⇧ chords (Q releases capture, A mutes the mic).
|
||||
private var hasChordModifiers: Bool {
|
||||
let m = chordModifiersDown
|
||||
return (m.contains(0xA2) || m.contains(0xA3)) // control
|
||||
&& (m.contains(0xA4) || m.contains(0xA5)) // option
|
||||
@@ -284,6 +292,10 @@ public final class InputCapture {
|
||||
self.suppressedVK = 0x53
|
||||
self.onCycleStats?()
|
||||
return nil
|
||||
case 0 /* A */:
|
||||
self.suppressedVK = 0x41
|
||||
self.onToggleMicMute?()
|
||||
return nil
|
||||
default:
|
||||
break
|
||||
}
|
||||
@@ -704,7 +716,7 @@ public final class InputCapture {
|
||||
}
|
||||
}
|
||||
#if os(iOS)
|
||||
// Track Control/Option/Shift for the ⌃⌥⇧Q release chord below — in both forwarding
|
||||
// Track Control/Option/Shift for the ⌃⌥⇧ chords below — in both forwarding
|
||||
// states (like `cmdKeysDown`) so a modifier held before capture engaged still counts.
|
||||
if Self.chordModifierVKs.contains(vk) {
|
||||
if pressed { self.chordModifiersDown.insert(vk) } else { self.chordModifiersDown.remove(vk) }
|
||||
@@ -732,11 +744,19 @@ public final class InputCapture {
|
||||
// otherwise). The Q is latched (`suppressedVK`) so its keyUp can't type into the host;
|
||||
// the ⌃⌥⇧ modifiers were forwarded as they went down and are flushed by the release
|
||||
// path (setCaptured(false) → releaseAll). VK 0x51 is layout-independent (physical Q).
|
||||
if pressed, vk == 0x51, self.hasReleaseChordModifiers {
|
||||
if pressed, vk == 0x51, self.hasChordModifiers {
|
||||
self.suppressedVK = 0x51
|
||||
self.onReleaseCapture?()
|
||||
return
|
||||
}
|
||||
// ⌃⌥⇧A mutes/unmutes the mic uplink — same detection, same latching, and needed here
|
||||
// for the same reason as on macOS: a captured iPad swallows the Stream menu's
|
||||
// identical key equivalent. VK 0x41 is layout-independent (physical A).
|
||||
if pressed, vk == 0x41, self.hasChordModifiers {
|
||||
self.suppressedVK = 0x41
|
||||
self.onToggleMicMute?()
|
||||
return
|
||||
}
|
||||
#endif
|
||||
// Release direction of the toggle: GC's Esc-down can beat the NSEvent
|
||||
// monitor — never type Esc into the host while ⌘ is held (⌘⎋ is reserved).
|
||||
|
||||
@@ -881,6 +881,12 @@ public final class StreamLayerView: NSView {
|
||||
guard self?.window?.isKeyWindow == true else { return }
|
||||
NotificationCenter.default.post(name: .punktfunkToggleFullscreen, object: nil)
|
||||
}
|
||||
capture.onToggleMicMute = { [weak self] in
|
||||
// Session-level state the view doesn't own — post to the app (same routing as the
|
||||
// fullscreen chord), so the captured and released paths end at one toggle.
|
||||
guard self?.window?.isKeyWindow == true else { return }
|
||||
NotificationCenter.default.post(name: .punktfunkToggleMicMute, object: nil)
|
||||
}
|
||||
capture.onCycleStats = { [weak self] in
|
||||
guard self?.window?.isKeyWindow == true else { return }
|
||||
// Advance the shared tier setting directly — every @AppStorage reader (the HUD's
|
||||
|
||||
@@ -424,6 +424,12 @@ public final class StreamViewController: StreamViewControllerBase {
|
||||
capture.onReleaseCapture = { [weak self] in
|
||||
self?.setCaptured(false)
|
||||
}
|
||||
// ⌃⌥⇧A mutes/unmutes the mic uplink. Session state this controller doesn't own, so it
|
||||
// posts to the app exactly as the macOS chord does — the Stream menu's identical
|
||||
// equivalent (which a captured scene swallows) ends at the same toggle.
|
||||
capture.onToggleMicMute = {
|
||||
NotificationCenter.default.post(name: .punktfunkToggleMicMute, object: nil)
|
||||
}
|
||||
capture.onPreempted = { [weak self] in
|
||||
self?.setCaptured(false)
|
||||
}
|
||||
|
||||
@@ -42,6 +42,13 @@ public enum DefaultsKey {
|
||||
/// falls back. Drives the decoder via `Welcome.codec`.
|
||||
public static let codec = "punktfunk.codec"
|
||||
public static let micEnabled = "punktfunk.micEnabled"
|
||||
/// Echo cancellation for the mic uplink (on by default): playback + capture share ONE
|
||||
/// audio engine so the system voice processor can subtract what this device is playing
|
||||
/// from what its mic hears — without it a loudspeaker client feeds the game audio straight
|
||||
/// back to the host. Off = the raw two-engine capture path. macOS: an explicitly pinned
|
||||
/// speaker/mic or mic channel also bypasses it (the voice processor only follows the
|
||||
/// system default devices) — see SessionAudio's topology note.
|
||||
public static let echoCancel = "punktfunk.echoCancel"
|
||||
public static let speakerUID = "punktfunk.speakerUID"
|
||||
public static let micUID = "punktfunk.micUID"
|
||||
/// macOS: which input channel of the chosen mic device feeds the host. 0 = "Auto" (sum every
|
||||
@@ -193,6 +200,12 @@ extension Notification.Name {
|
||||
/// state. macOS only.
|
||||
public static let punktfunkToggleFullscreen = Notification.Name("io.unom.punktfunk.toggle-fullscreen")
|
||||
|
||||
/// Posted by InputCapture's chord path (⌃⌥⇧A) when the combo fires while input is CAPTURED —
|
||||
/// the state in which the Stream menu's identical key equivalent never reaches the app. The
|
||||
/// live session's owner (ContentView) flips the session's mic mute. Released, the menu item
|
||||
/// handles the same combo directly; both end at `SessionModel.toggleMicMute`.
|
||||
public static let punktfunkToggleMicMute = Notification.Name("io.unom.punktfunk.toggle-mic-mute")
|
||||
|
||||
/// Posted by the Live Activity's / Shortcuts' End-stream intent (`EndStreamIntent.perform`,
|
||||
/// which runs in the app's process): the app tears the active session down deliberately
|
||||
/// (quit-close the host). Same cross-process-signal pattern as `punktfunkReleaseCapture` —
|
||||
|
||||
@@ -29,6 +29,7 @@ public struct EffectiveSettings: Equatable, Sendable {
|
||||
public var compositor = 0
|
||||
public var audioChannels = 2
|
||||
public var micEnabled = true
|
||||
public var echoCancel = true
|
||||
public var touchMode = "trackpad"
|
||||
public var mouseMode = "capture"
|
||||
public var invertScroll = false
|
||||
@@ -87,6 +88,7 @@ public struct EffectiveSettings: Equatable, Sendable {
|
||||
compositor = int(DefaultsKey.compositor, compositor)
|
||||
audioChannels = int(DefaultsKey.audioChannels, audioChannels)
|
||||
micEnabled = bool(DefaultsKey.micEnabled, micEnabled)
|
||||
echoCancel = bool(DefaultsKey.echoCancel, echoCancel)
|
||||
touchMode = str(DefaultsKey.touchMode, touchMode)
|
||||
mouseMode = str(DefaultsKey.mouseMode, mouseMode)
|
||||
invertScroll = bool(DefaultsKey.invertScroll, invertScroll)
|
||||
@@ -133,6 +135,7 @@ public struct EffectiveSettings: Equatable, Sendable {
|
||||
if let v = overlay.compositor { s.compositor = v }
|
||||
if let v = overlay.audioChannels { s.audioChannels = v }
|
||||
if let v = overlay.micEnabled { s.micEnabled = v }
|
||||
if let v = overlay.echoCancel { s.echoCancel = v }
|
||||
if let v = overlay.touchMode { s.touchMode = v }
|
||||
if let v = overlay.mouseMode { s.mouseMode = v }
|
||||
if let v = overlay.invertScroll { s.invertScroll = v }
|
||||
|
||||
@@ -105,6 +105,7 @@ public struct SettingsOverlay: Codable, Equatable, Sendable {
|
||||
public var compositor: Int?
|
||||
public var audioChannels: Int?
|
||||
public var micEnabled: Bool?
|
||||
public var echoCancel: Bool?
|
||||
public var touchMode: String?
|
||||
public var mouseMode: String?
|
||||
public var invertScroll: Bool?
|
||||
@@ -145,6 +146,7 @@ public struct SettingsOverlay: Codable, Equatable, Sendable {
|
||||
case compositor
|
||||
case audioChannels = "audio_channels"
|
||||
case micEnabled = "mic_enabled"
|
||||
case echoCancel = "echo_cancel"
|
||||
case touchMode = "touch_mode"
|
||||
case mouseMode = "mouse_mode"
|
||||
case invertScroll = "invert_scroll"
|
||||
@@ -177,6 +179,7 @@ public struct SettingsOverlay: Codable, Equatable, Sendable {
|
||||
compositor = int(.compositor)
|
||||
audioChannels = int(.audioChannels)
|
||||
micEnabled = bool(.micEnabled)
|
||||
echoCancel = bool(.echoCancel)
|
||||
touchMode = str(.touchMode)
|
||||
mouseMode = str(.mouseMode)
|
||||
invertScroll = bool(.invertScroll)
|
||||
@@ -211,6 +214,7 @@ public struct SettingsOverlay: Codable, Equatable, Sendable {
|
||||
try c.encodeIfPresent(compositor, forKey: AnyKey(Key.compositor.rawValue))
|
||||
try c.encodeIfPresent(audioChannels, forKey: AnyKey(Key.audioChannels.rawValue))
|
||||
try c.encodeIfPresent(micEnabled, forKey: AnyKey(Key.micEnabled.rawValue))
|
||||
try c.encodeIfPresent(echoCancel, forKey: AnyKey(Key.echoCancel.rawValue))
|
||||
try c.encodeIfPresent(touchMode, forKey: AnyKey(Key.touchMode.rawValue))
|
||||
try c.encodeIfPresent(mouseMode, forKey: AnyKey(Key.mouseMode.rawValue))
|
||||
try c.encodeIfPresent(invertScroll, forKey: AnyKey(Key.invertScroll.rawValue))
|
||||
@@ -262,6 +266,7 @@ public enum OverlayField {
|
||||
case "compositor": overlay.compositor = nil
|
||||
case "audio_channels": overlay.audioChannels = nil
|
||||
case "mic_enabled": overlay.micEnabled = nil
|
||||
case "echo_cancel": overlay.echoCancel = nil
|
||||
case "touch_mode": overlay.touchMode = nil
|
||||
case "mouse_mode": overlay.mouseMode = nil
|
||||
case "invert_scroll": overlay.invertScroll = nil
|
||||
@@ -296,6 +301,7 @@ public enum OverlayField {
|
||||
case "compositor": return o.compositor != nil
|
||||
case "audio_channels": return o.audioChannels != nil
|
||||
case "mic_enabled": return o.micEnabled != nil
|
||||
case "echo_cancel": return o.echoCancel != nil
|
||||
case "touch_mode": return o.touchMode != nil
|
||||
case "mouse_mode": return o.mouseMode != nil
|
||||
case "invert_scroll": return o.invertScroll != nil
|
||||
|
||||
@@ -9,32 +9,34 @@ import XCTest
|
||||
@testable import PunktfunkKit
|
||||
|
||||
final class OpusCodecTests: XCTestCase {
|
||||
/// Encode a 440 Hz stereo tone, decode it back, and require the result to be
|
||||
/// recognizably the same signal (Opus is lossy — check correlation, not bytes).
|
||||
/// Encode a 440 Hz mono tone (the uplink's shape), decode it back through the
|
||||
/// STEREO-configured decoder (the host-plane shape — Opus upmixes mono packets), and
|
||||
/// require the result to be recognizably the same signal (Opus is lossy — check
|
||||
/// correlation, not bytes).
|
||||
func testEncodeDecodeRoundTripPreservesTone() throws {
|
||||
let encoder = try OpusEncoder()
|
||||
let decoder = try OpusDecoder(framesPerPacket: UInt32(OpusEncoder.framesPerPacket))
|
||||
let decoder = try OpusDecoder(framesPerPacket: UInt32(encoder.framesPerPacket))
|
||||
let pcmFormat = encoder.pcmFormat
|
||||
|
||||
let frames = OpusEncoder.framesPerPacket
|
||||
let frames = encoder.framesPerPacket
|
||||
var packets: [Data] = []
|
||||
var phase: Float = 0
|
||||
let step = 2 * Float.pi * 440 / 48_000
|
||||
|
||||
// 50 packets = 1 s of tone.
|
||||
for _ in 0..<50 {
|
||||
// 1 s of tone, whatever packet duration the encoder chose (10 ms → 100 chunks).
|
||||
let chunks = Int(48_000 / frames)
|
||||
for _ in 0..<chunks {
|
||||
let buf = AVAudioPCMBuffer(pcmFormat: pcmFormat, frameCapacity: frames)!
|
||||
buf.frameLength = frames
|
||||
let p = buf.floatChannelData![0] // interleaved: one plane, L R L R …
|
||||
let p = buf.floatChannelData![0] // mono: one plane
|
||||
for f in 0..<Int(frames) {
|
||||
let s = sin(phase) * 0.5
|
||||
p[f] = sin(phase) * 0.5
|
||||
phase += step
|
||||
p[f * 2] = s
|
||||
p[f * 2 + 1] = s
|
||||
}
|
||||
packets.append(contentsOf: try encoder.encode(buf))
|
||||
}
|
||||
XCTAssertGreaterThanOrEqual(packets.count, 45, "encoder must emit ~one packet per buffer")
|
||||
XCTAssertGreaterThanOrEqual(
|
||||
packets.count, chunks - 5, "encoder must emit ~one packet per buffer")
|
||||
XCTAssertTrue(packets.allSatisfy { !$0.isEmpty })
|
||||
|
||||
var decoded: [Float] = []
|
||||
|
||||
@@ -62,29 +62,30 @@ final class RemoteFirstLightTests: XCTestCase {
|
||||
host: host, port: port, width: 1280, height: 720, refreshHz: 60)
|
||||
defer { conn.close() }
|
||||
|
||||
// Mic uplink: 2 s of 440 Hz tone (the host's mic service opens its virtual
|
||||
// Mic uplink: 2 s of 440 Hz mono tone (the host's mic service opens its virtual
|
||||
// source on the first frame — check its log).
|
||||
let encoder = try OpusEncoder()
|
||||
let chunk = AVAudioPCMBuffer(
|
||||
pcmFormat: encoder.pcmFormat, frameCapacity: OpusEncoder.framesPerPacket)!
|
||||
pcmFormat: encoder.pcmFormat, frameCapacity: encoder.framesPerPacket)!
|
||||
var phase: Float = 0
|
||||
let step = 2 * Float.pi * 440 / 48_000
|
||||
var seq: UInt32 = 0
|
||||
for _ in 0..<100 {
|
||||
chunk.frameLength = OpusEncoder.framesPerPacket
|
||||
let p = chunk.floatChannelData![0]
|
||||
for f in 0..<Int(OpusEncoder.framesPerPacket) {
|
||||
let s = sin(phase) * 0.25
|
||||
let chunks = 2 * 48_000 / Int(encoder.framesPerPacket)
|
||||
let packetNs = UInt64(encoder.framesPerPacket) * 1_000_000_000 / 48_000
|
||||
for _ in 0..<chunks {
|
||||
chunk.frameLength = encoder.framesPerPacket
|
||||
let p = chunk.floatChannelData![0] // mono: one plane
|
||||
for f in 0..<Int(encoder.framesPerPacket) {
|
||||
p[f] = sin(phase) * 0.25
|
||||
phase += step
|
||||
p[f * 2] = s
|
||||
p[f * 2 + 1] = s
|
||||
}
|
||||
for packet in try encoder.encode(chunk) {
|
||||
conn.sendMic(packet, seq: seq, ptsNs: UInt64(seq) * 20_000_000)
|
||||
conn.sendMic(packet, seq: seq, ptsNs: UInt64(seq) * packetNs)
|
||||
seq &+= 1
|
||||
}
|
||||
}
|
||||
XCTAssertGreaterThanOrEqual(seq, 95, "mic encoder must emit ~one packet per chunk")
|
||||
XCTAssertGreaterThanOrEqual(
|
||||
seq, UInt32(chunks - 5), "mic encoder must emit ~one packet per chunk")
|
||||
|
||||
// Downlink: pull host audio packets and decode them (the host streams its sink
|
||||
// monitor — silence still produces packets).
|
||||
|
||||
@@ -1019,6 +1019,12 @@ pub fn shortcuts_window(parent: &adw::ApplicationWindow) -> gtk::ShortcutsWindow
|
||||
<property name="accelerator"><Control><Alt><Shift>s</property>
|
||||
</object>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkShortcutsShortcut">
|
||||
<property name="title">Mute or unmute your microphone (only while the stream sends one)</property>
|
||||
<property name="accelerator"><Control><Alt><Shift>v</property>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
|
||||
@@ -480,8 +480,23 @@ fn headless_check_update() -> glib::ExitCode {
|
||||
"installed {} ({}, {})",
|
||||
status.current, status.kind, status.channel
|
||||
);
|
||||
println!("available {}", status.latest);
|
||||
if let Some(err) = &status.error {
|
||||
// `latest` falls back to `current` when the check couldn't run — printing that as
|
||||
// "available" would read as a confirmed answer we don't have.
|
||||
if status.error.is_some() {
|
||||
println!("available unknown");
|
||||
} else {
|
||||
println!("available {}", status.latest);
|
||||
}
|
||||
if status.not_published {
|
||||
// Says what it is, in words, instead of a raw HTTP status. The exit code still
|
||||
// reports "could not tell" (see the doc comment above): an empty channel is the
|
||||
// absence of evidence that this build is current, and a mistyped
|
||||
// PUNKTFUNK_UPDATE_FEED is indistinguishable from one out here.
|
||||
println!(
|
||||
"update nothing published on the {} channel yet",
|
||||
status.channel
|
||||
);
|
||||
} else if let Some(err) = &status.error {
|
||||
eprintln!("check-update: {err}");
|
||||
} else if status.update_available {
|
||||
println!("update yes");
|
||||
|
||||
@@ -607,6 +607,9 @@ fn commit_profile(active: &StreamProfile, touched: &Touched, values: &Settings)
|
||||
if touched.has("mic_enabled") {
|
||||
o.mic_enabled = Some(values.mic_enabled);
|
||||
}
|
||||
if touched.has("echo_cancel") {
|
||||
o.echo_cancel = Some(values.echo_cancel);
|
||||
}
|
||||
if touched.has("touch_mode") {
|
||||
o.touch_mode = Some(values.touch_mode.clone());
|
||||
}
|
||||
@@ -1301,7 +1304,11 @@ pub fn show_scoped(
|
||||
);
|
||||
let mic_row = adw::SwitchRow::builder()
|
||||
.title("Stream microphone")
|
||||
.subtitle("Sends your microphone to the host's virtual mic")
|
||||
.subtitle("Sends your microphone to the host's virtual mic — Ctrl+Alt+Shift+V mutes it mid-stream")
|
||||
.build();
|
||||
let echo_row = adw::SwitchRow::builder()
|
||||
.title("Echo cancellation")
|
||||
.subtitle("Keeps the host's audio, playing from this machine's speakers, out of the uplink")
|
||||
.build();
|
||||
// Endpoint pickers (from the PipeWire probe): visible labels are descriptions, the
|
||||
// stored value is the node name. Hidden when the probe found nothing; a saved
|
||||
@@ -1345,12 +1352,23 @@ pub fn show_scoped(
|
||||
"Microphone",
|
||||
"The input that feeds the host's virtual mic",
|
||||
);
|
||||
// The device pick only matters while the mic streams at all.
|
||||
// The device pick and the echo canceller only matter while the mic streams at all — both
|
||||
// follow it. One handler each (the pickers are optional, the echo row never is), and the
|
||||
// initial state is set here because the seed block further down fires these too.
|
||||
//
|
||||
// Insensitivity covers the whole row, including the per-row Reset a profile scope adds:
|
||||
// an echo_cancel override can only be reset while the mic row is on. Turn it on, reset,
|
||||
// turn it back off — the alternative is a control that looks live and isn't.
|
||||
if let Some(r) = &micdev_row {
|
||||
let w = r.widget().clone();
|
||||
w.set_sensitive(mic_row.is_active());
|
||||
mic_row.connect_active_notify(move |m| w.set_sensitive(m.is_active()));
|
||||
}
|
||||
{
|
||||
let w = echo_row.clone();
|
||||
w.set_sensitive(mic_row.is_active());
|
||||
mic_row.connect_active_notify(move |m| w.set_sensitive(m.is_active()));
|
||||
}
|
||||
|
||||
// ---- Controllers ----
|
||||
// Controller forwarding: Automatic forwards EVERY real controller, each as its own pad
|
||||
@@ -1453,6 +1471,7 @@ pub fn show_scoped(
|
||||
inhibit_row.set_active(s.inhibit_shortcuts);
|
||||
invert_row.set_active(s.invert_scroll);
|
||||
mic_row.set_active(s.mic_enabled);
|
||||
echo_row.set_active(s.echo_cancel);
|
||||
hdr_row.set_active(s.hdr_enabled);
|
||||
chroma_row.set_active(s.enable_444);
|
||||
library_row.set_active(s.library_enabled);
|
||||
@@ -1673,6 +1692,12 @@ pub fn show_scoped(
|
||||
invert_scroll
|
||||
);
|
||||
toggle!(mic_row, "mic_enabled", o.mic_enabled.is_some(), mic_enabled);
|
||||
toggle!(
|
||||
echo_row,
|
||||
"echo_cancel",
|
||||
o.echo_cancel.is_some(),
|
||||
echo_cancel
|
||||
);
|
||||
{
|
||||
let revert = {
|
||||
let (row, globals, touched) =
|
||||
@@ -1781,6 +1806,7 @@ pub fn show_scoped(
|
||||
audio_group.add(r.widget());
|
||||
}
|
||||
audio_group.add(&mic_row);
|
||||
audio_group.add(&echo_row);
|
||||
if let (Some(r), false) = (&micdev_row, profile_mode) {
|
||||
audio_group.add(r.widget());
|
||||
}
|
||||
@@ -1890,6 +1916,7 @@ pub fn show_scoped(
|
||||
s.inhibit_shortcuts = inhibit_row.is_active();
|
||||
s.invert_scroll = invert_row.is_active();
|
||||
s.mic_enabled = mic_row.is_active();
|
||||
s.echo_cancel = echo_row.is_active();
|
||||
s.hdr_enabled = hdr_row.is_active();
|
||||
s.enable_444 = chroma_row.is_active();
|
||||
s.audio_channels = match surround_row.selected() {
|
||||
|
||||
@@ -290,6 +290,7 @@ mod session_main {
|
||||
// compositors only).
|
||||
cursor_forward: settings.mouse_mode() == trust::MouseMode::Desktop,
|
||||
mic_enabled: settings.mic_enabled,
|
||||
echo_cancel: settings.echo_cancel,
|
||||
clipboard,
|
||||
// The Settings preference (auto → VAAPI where it exists; the presenter
|
||||
// demotes to software on boxes whose Vulkan can't import the dmabufs).
|
||||
|
||||
@@ -21,6 +21,10 @@ const STREAM_SHORTCUTS: &[(&str, &str)] = &[
|
||||
"Ctrl+Alt+Shift+S",
|
||||
"Cycle the statistics overlay (off \u{00B7} compact \u{00B7} normal \u{00B7} detailed)",
|
||||
),
|
||||
(
|
||||
"Ctrl+Alt+Shift+V",
|
||||
"Mute or unmute your microphone (only while the stream sends one)",
|
||||
),
|
||||
(
|
||||
"LB+RB+Start+Back",
|
||||
"Controller: release input / leave fullscreen \u{2014} hold to disconnect",
|
||||
|
||||
@@ -439,6 +439,7 @@ struct OverrideFlags {
|
||||
compositor: bool,
|
||||
audio_channels: bool,
|
||||
mic_enabled: bool,
|
||||
echo_cancel: bool,
|
||||
touch_mode: bool,
|
||||
mouse_mode: bool,
|
||||
invert_scroll: bool,
|
||||
@@ -466,6 +467,7 @@ impl OverrideFlags {
|
||||
compositor: o.compositor.is_some(),
|
||||
audio_channels: o.audio_channels.is_some(),
|
||||
mic_enabled: o.mic_enabled.is_some(),
|
||||
echo_cancel: o.echo_cancel.is_some(),
|
||||
touch_mode: o.touch_mode.is_some(),
|
||||
mouse_mode: o.mouse_mode.is_some(),
|
||||
invert_scroll: o.invert_scroll.is_some(),
|
||||
@@ -954,6 +956,12 @@ pub(crate) fn settings_page(
|
||||
};
|
||||
let speaker_combo = dev_combo(&s.speaker_device, &speakers, |s, v| s.speaker_device = v);
|
||||
let mic_dev_combo = dev_combo(&s.mic_device, &mics, |s, v| s.mic_device = v);
|
||||
// Echo cancellation is meaningless without an uplink, so it greys out with the mic above
|
||||
// it. Every commit bumps `rev` and re-renders this screen, so the two stay in step live.
|
||||
let echo_toggle = setting_toggle(ctx, scope, (rev, set_rev), s.echo_cancel, |s, on| {
|
||||
s.echo_cancel = on
|
||||
})
|
||||
.enabled(s.mic_enabled);
|
||||
|
||||
let (hud_names, hud_i) = presets(STATS_TIERS, |v| *v == s.stats_verbosity());
|
||||
let hud_combo = setting_combo(ctx, scope, (rev, set_rev), hud_names, hud_i, |s, i| {
|
||||
@@ -1281,7 +1289,8 @@ pub(crate) fn settings_page(
|
||||
"Stream microphone to the host",
|
||||
over.mic_enabled,
|
||||
mic_toggle,
|
||||
"This device\u{2019}s microphone feeds the host\u{2019}s virtual mic.",
|
||||
"This device\u{2019}s microphone feeds the host\u{2019}s virtual mic. \
|
||||
Ctrl+Alt+Shift+V mutes and unmutes it during a stream.",
|
||||
)),
|
||||
(!profile_mode)
|
||||
.then(|| {
|
||||
@@ -1294,6 +1303,17 @@ pub(crate) fn settings_page(
|
||||
})
|
||||
})
|
||||
.flatten(),
|
||||
Some(described_overridable(
|
||||
(rev, set_rev),
|
||||
scope,
|
||||
"echo_cancel",
|
||||
"Echo cancellation",
|
||||
over.echo_cancel,
|
||||
echo_toggle,
|
||||
"Keeps the host\u{2019}s audio, playing from this machine\u{2019}s \
|
||||
speakers, from being picked up and sent straight back. Turn it off if \
|
||||
your microphone already does its own processing.",
|
||||
)),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
@@ -1696,5 +1716,16 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
assert!(OverrideFlags::of(Some(&p2)).resolution);
|
||||
|
||||
// The audio pair: the mic and its echo canceller are separate overrides, so a profile
|
||||
// can pin one without claiming the other.
|
||||
let mut p3 = StreamProfile::new("t3".to_string());
|
||||
p3.overrides = SettingsOverlay {
|
||||
echo_cancel: Some(false),
|
||||
..Default::default()
|
||||
};
|
||||
let f3 = OverrideFlags::of(Some(&p3));
|
||||
assert!(f3.echo_cancel);
|
||||
assert!(!f3.mic_enabled);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,9 +14,12 @@ use std::sync::mpsc::{Receiver, SyncSender, TrySendError};
|
||||
use std::sync::Arc;
|
||||
|
||||
const SAMPLE_RATE: u32 = 48_000;
|
||||
const CHANNELS: usize = 2;
|
||||
/// Mic frames are 20 ms (960 samples/channel) — any size ≤ 120 ms is fine host-side.
|
||||
const MIC_FRAME: usize = 960;
|
||||
/// Mic capture is MONO: voice is mono at the source, the host accepts any Opus channel
|
||||
/// layout (its stereo decoder upmixes), and half the samples halve the encode + wire cost.
|
||||
const MIC_CHANNELS: usize = 1;
|
||||
/// Mic frames are 10 ms (480 mono samples) — any size ≤ 120 ms is fine host-side; 10 ms
|
||||
/// halves the frame-fill share of mouth-to-ear latency vs the old 20 ms.
|
||||
const MIC_FRAME: usize = 480;
|
||||
|
||||
struct Terminate;
|
||||
|
||||
@@ -327,20 +330,31 @@ fn pw_thread(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The microphone uplink: capture the default input device, Opus-encode 20 ms chunks,
|
||||
/// ship them as 0xCB datagrams into the host's virtual PipeWire source.
|
||||
/// The microphone uplink: capture the default input device (or the picked / echo-cancelled
|
||||
/// source), Opus-encode 10 ms mono chunks, ship them as 0xCB datagrams into the host's
|
||||
/// virtual PipeWire source.
|
||||
pub struct MicStreamer {
|
||||
quit_tx: pipewire::channel::Sender<Terminate>,
|
||||
thread: Option<std::thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl MicStreamer {
|
||||
pub fn spawn(connector: Arc<NativeClient>) -> Result<MicStreamer> {
|
||||
/// `muted` is the in-stream mute (B4), shared live with the capture callback: set, the
|
||||
/// callback keeps pulling and discarding whole frames but sends nothing. Muting by
|
||||
/// STOPPING the stream was rejected — it re-primes the device buffers and re-runs the
|
||||
/// source selection below on every unmute, so the first second back is glitchy.
|
||||
///
|
||||
/// `echo_cancel` is the Settings toggle; `PUNKTFUNK_NO_AEC=1` overrides it off.
|
||||
pub fn spawn(
|
||||
connector: Arc<NativeClient>,
|
||||
muted: Arc<std::sync::atomic::AtomicBool>,
|
||||
echo_cancel: bool,
|
||||
) -> Result<MicStreamer> {
|
||||
let (quit_tx, quit_rx) = pipewire::channel::channel::<Terminate>();
|
||||
let thread = std::thread::Builder::new()
|
||||
.name("punktfunk-mic".into())
|
||||
.spawn(move || {
|
||||
if let Err(e) = mic_thread(&connector, quit_rx) {
|
||||
if let Err(e) = mic_thread(&connector, quit_rx, muted, echo_cancel) {
|
||||
tracing::warn!(error = %e, "mic uplink thread ended");
|
||||
}
|
||||
})
|
||||
@@ -361,19 +375,76 @@ impl Drop for MicStreamer {
|
||||
}
|
||||
}
|
||||
|
||||
/// Capture-side state: accumulated PCM and the Opus encoder (encoding a 20 ms frame is
|
||||
/// ~100 µs — fine inside the process callback).
|
||||
/// Capture-side state: accumulated PCM and the Opus encoder (encoding a 10 ms frame is
|
||||
/// well under 100 µs — fine inside the process callback).
|
||||
struct MicData {
|
||||
connector: Arc<NativeClient>,
|
||||
ring: VecDeque<f32>,
|
||||
encoder: opus::Encoder,
|
||||
seq: u32,
|
||||
out: Vec<u8>,
|
||||
/// The in-stream mute (B4), flipped by the session's chord. Read per callback.
|
||||
muted: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
/// Whether the mic echo-cancellation hooks run this session: the `echo_cancel` setting, with
|
||||
/// `PUNKTFUNK_NO_AEC=1` as a one-way override OFF. The env var wins — it is the escape hatch
|
||||
/// for a box whose canceller misbehaves, and it predates the setting; nothing turns AEC back
|
||||
/// on once it is set. Here the hook is the echo-cancelled-source preference below; the WASAPI
|
||||
/// twin gates its Communications stream category the same way.
|
||||
fn aec_enabled(echo_cancel: bool) -> bool {
|
||||
echo_cancel && !std::env::var("PUNKTFUNK_NO_AEC").is_ok_and(|v| !v.is_empty() && v != "0")
|
||||
}
|
||||
|
||||
/// The capture stream's `target.object`, in preference order: the Settings microphone pick
|
||||
/// (`Settings::mic_device` via session main's `PUNKTFUNK_AUDIO_SOURCE`) verbatim, else — so a
|
||||
/// desktop that already runs `module-echo-cancel` stops feeding its own downlink audio back
|
||||
/// into the host's virtual mic — the first echo-cancelled source in the graph. `None` = the
|
||||
/// user picked nothing and no such source exists: PipeWire's default routing, as before.
|
||||
///
|
||||
/// Preference-only by design: loading `libpipewire-module-echo-cancel` ourselves needs
|
||||
/// `pw_context_load_module`, which the pipewire crate (0.9) doesn't expose safely — until it
|
||||
/// does, we only ever target processing the user (or their session) already set up.
|
||||
fn mic_capture_target(echo_cancel: bool) -> Option<String> {
|
||||
if let Ok(target) = std::env::var("PUNKTFUNK_AUDIO_SOURCE") {
|
||||
if !target.is_empty() {
|
||||
return Some(target);
|
||||
}
|
||||
}
|
||||
if !aec_enabled(echo_cancel) {
|
||||
return None;
|
||||
}
|
||||
let name = echo_cancel_source()?;
|
||||
tracing::info!(
|
||||
source = %name,
|
||||
"mic capture targets the echo-cancelled source (Echo cancellation off, or \
|
||||
PUNKTFUNK_NO_AEC=1, disables this)"
|
||||
);
|
||||
Some(name)
|
||||
}
|
||||
|
||||
/// Find an existing echo-cancelled capture node: the first `Audio/Source` whose `node.name`
|
||||
/// or description says echo-cancel (`module-echo-cancel`'s convention — `echo-cancel-*`
|
||||
/// nodes, "Echo-Cancel …" descriptions; PulseAudio-compat setups match too). One registry
|
||||
/// roundtrip via [`devices`]; any failure reads as "none".
|
||||
fn echo_cancel_source() -> Option<String> {
|
||||
let (_, sources) = devices().ok()?;
|
||||
sources.into_iter().find_map(|d| {
|
||||
let name = d.name.to_ascii_lowercase();
|
||||
let desc = d.description.to_ascii_lowercase();
|
||||
(name.contains("echo-cancel")
|
||||
|| name.contains("echo_cancel")
|
||||
|| desc.contains("echo-cancel")
|
||||
|| desc.contains("echo cancel"))
|
||||
.then_some(d.name)
|
||||
})
|
||||
}
|
||||
|
||||
fn mic_thread(
|
||||
connector: &Arc<NativeClient>,
|
||||
quit_rx: pipewire::channel::Receiver<Terminate>,
|
||||
muted: Arc<std::sync::atomic::AtomicBool>,
|
||||
echo_cancel: bool,
|
||||
) -> Result<()> {
|
||||
use pipewire as pw;
|
||||
use pw::{properties::properties, spa};
|
||||
@@ -384,9 +455,14 @@ fn mic_thread(
|
||||
PW_INIT.call_once(pw::init);
|
||||
|
||||
let mut encoder =
|
||||
opus::Encoder::new(SAMPLE_RATE, opus::Channels::Stereo, opus::Application::Voip)
|
||||
opus::Encoder::new(SAMPLE_RATE, opus::Channels::Mono, opus::Application::Voip)
|
||||
.map_err(|e| anyhow::anyhow!("opus encoder: {e}"))?;
|
||||
let _ = encoder.set_bitrate(opus::Bitrate::Bits(64_000));
|
||||
// Voice tuning: 48 kbps mono is transparent for speech; in-band FEC + an assumed 10 %
|
||||
// loss let the host's decoder rebuild a lost 0xCB datagram from its successor instead
|
||||
// of concealing (datagrams are fire-and-forget — this FEC is the only redundancy).
|
||||
let _ = encoder.set_bitrate(opus::Bitrate::Bits(48_000));
|
||||
let _ = encoder.set_inband_fec(true);
|
||||
let _ = encoder.set_packet_loss_perc(10);
|
||||
|
||||
let mainloop = pw::main_loop::MainLoopRc::new(None).context("pw mic MainLoop")?;
|
||||
let context = pw::context::ContextRc::new(&mainloop, None).context("pw mic Context")?;
|
||||
@@ -405,14 +481,15 @@ fn mic_thread(
|
||||
*pw::keys::MEDIA_ROLE => "Communication",
|
||||
*pw::keys::NODE_NAME => "punktfunk-mic-capture",
|
||||
*pw::keys::NODE_DESCRIPTION => "Punktfunk Microphone",
|
||||
// ~10 ms quantum (one mic frame). Without it the capture stream inherits the graph
|
||||
// quantum — commonly 1024–2048 samples, so the mic arrived in 21–43 ms bursts that
|
||||
// sat ahead of the encoder as latency (the playback stream always asked for 5 ms).
|
||||
*pw::keys::NODE_LATENCY => "480/48000",
|
||||
};
|
||||
// The Settings microphone pick (`Settings::mic_device` via session main).
|
||||
if let Ok(target) = std::env::var("PUNKTFUNK_AUDIO_SOURCE") {
|
||||
if !target.is_empty() {
|
||||
// Raw key: the `keys::TARGET_OBJECT` constant is feature-gated on a newer
|
||||
// libpipewire than we require; the wire name is stable.
|
||||
props.insert("target.object", target);
|
||||
}
|
||||
if let Some(target) = mic_capture_target(echo_cancel) {
|
||||
// Raw key: the `keys::TARGET_OBJECT` constant is feature-gated on a newer
|
||||
// libpipewire than we require; the wire name is stable.
|
||||
props.insert("target.object", target);
|
||||
}
|
||||
let stream = pw::stream::StreamBox::new(&core, "punktfunk-mic-capture", props)
|
||||
.context("pw mic Stream")?;
|
||||
@@ -423,6 +500,7 @@ fn mic_thread(
|
||||
encoder,
|
||||
seq: 0,
|
||||
out: vec![0u8; 4000],
|
||||
muted,
|
||||
};
|
||||
|
||||
let _listener = stream
|
||||
@@ -447,9 +525,20 @@ fn mic_thread(
|
||||
.push_back(f32::from_le_bytes([s[0], s[1], s[2], s[3]]));
|
||||
}
|
||||
}
|
||||
// Ship every complete 20 ms stereo frame.
|
||||
while ud.ring.len() >= MIC_FRAME * CHANNELS {
|
||||
let pcm: Vec<f32> = ud.ring.drain(..MIC_FRAME * CHANNELS).collect();
|
||||
// Muted (B4): the stream stays open and the device keeps its primed buffers —
|
||||
// only the sending stops. Whole frames are discarded so the ring can't grow,
|
||||
// and `seq` deliberately does NOT advance: the host sees one continuous
|
||||
// sequence with a silent pause in the middle rather than a gap the size of the
|
||||
// mute, which its de-jitter would try to conceal frame by frame.
|
||||
if ud.muted.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
let whole =
|
||||
(ud.ring.len() / (MIC_FRAME * MIC_CHANNELS)) * (MIC_FRAME * MIC_CHANNELS);
|
||||
ud.ring.drain(..whole);
|
||||
return;
|
||||
}
|
||||
// Ship every complete 10 ms mono frame.
|
||||
while ud.ring.len() >= MIC_FRAME * MIC_CHANNELS {
|
||||
let pcm: Vec<f32> = ud.ring.drain(..MIC_FRAME * MIC_CHANNELS).collect();
|
||||
match ud.encoder.encode_float(&pcm, &mut ud.out) {
|
||||
Ok(len) => {
|
||||
let pts = std::time::SystemTime::now()
|
||||
@@ -473,7 +562,8 @@ fn mic_thread(
|
||||
let mut info = AudioInfoRaw::new();
|
||||
info.set_format(AudioFormat::F32LE);
|
||||
info.set_rate(SAMPLE_RATE);
|
||||
info.set_channels(CHANNELS as u32);
|
||||
// Mono: the stream's adapter downmixes whatever layout the source really has.
|
||||
info.set_channels(MIC_CHANNELS as u32);
|
||||
let obj = pw::spa::pod::Object {
|
||||
type_: pw::spa::utils::SpaTypes::ObjectParamFormat.as_raw(),
|
||||
id: pw::spa::param::ParamType::EnumFormat.as_raw(),
|
||||
|
||||
@@ -23,14 +23,22 @@ use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{Receiver, SyncSender, TrySendError};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use wasapi::{DeviceEnumerator, Direction, SampleType, StreamMode, WaveFormat};
|
||||
use wasapi::{
|
||||
AudioClientProperties, DeviceEnumerator, Direction, SampleType, StreamCategory, StreamMode,
|
||||
WaveFormat,
|
||||
};
|
||||
|
||||
const SAMPLE_RATE: usize = 48_000;
|
||||
/// The microphone uplink stays stereo (the host's virtual mic is stereo). The render path is
|
||||
/// multichannel — its channel count + block align are runtime, driven by the host-resolved layout.
|
||||
const CHANNELS: usize = 2;
|
||||
/// Mic frames are 20 ms (960 samples/channel) — any size ≤ 120 ms is fine host-side.
|
||||
const MIC_FRAME: usize = 960;
|
||||
/// Mic capture requests STEREO from WASAPI (autoconvert matrixes any endpoint layout down to
|
||||
/// it — the proven path; `read_from_device_to_deque` then delivers our requested format) and
|
||||
/// downmixes to MONO in code before the encoder: voice is mono at the source, the host accepts
|
||||
/// any Opus channel layout (its stereo decoder upmixes), and half the samples halve the
|
||||
/// encode + wire cost. The render path is multichannel — its channel count + block align are
|
||||
/// runtime, driven by the host-resolved layout.
|
||||
const CAPT_CHANNELS: usize = 2;
|
||||
/// Mic frames are 10 ms (480 mono samples) — any size ≤ 120 ms is fine host-side; 10 ms
|
||||
/// halves the frame-fill share of mouth-to-ear latency vs the old 20 ms.
|
||||
const MIC_FRAME: usize = 480;
|
||||
|
||||
/// A selectable WASAPI endpoint for the settings pickers.
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -302,21 +310,31 @@ fn render_thread(
|
||||
res
|
||||
}
|
||||
|
||||
/// The microphone uplink: capture the default input device, Opus-encode 20 ms chunks, ship
|
||||
/// them as 0xCB datagrams into the host's virtual mic source.
|
||||
/// The microphone uplink: capture the default input device, Opus-encode 10 ms mono chunks,
|
||||
/// ship them as 0xCB datagrams into the host's virtual mic source.
|
||||
pub struct MicStreamer {
|
||||
stop: Arc<AtomicBool>,
|
||||
thread: Option<std::thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl MicStreamer {
|
||||
pub fn spawn(connector: Arc<NativeClient>) -> Result<MicStreamer> {
|
||||
/// `muted` is the in-stream mute (B4), shared live with the capture loop: set, the loop
|
||||
/// keeps reading the endpoint and discarding whole frames but sends nothing. Muting by
|
||||
/// STOPPING the client was rejected — an `IAudioClient` stop/start re-primes the endpoint
|
||||
/// buffers and re-runs the category negotiation below on every unmute.
|
||||
///
|
||||
/// `echo_cancel` is the Settings toggle; `PUNKTFUNK_NO_AEC=1` overrides it off.
|
||||
pub fn spawn(
|
||||
connector: Arc<NativeClient>,
|
||||
muted: Arc<AtomicBool>,
|
||||
echo_cancel: bool,
|
||||
) -> Result<MicStreamer> {
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let stop_t = stop.clone();
|
||||
let thread = std::thread::Builder::new()
|
||||
.name("punktfunk-mic".into())
|
||||
.spawn(move || {
|
||||
if let Err(e) = mic_thread(&connector, stop_t) {
|
||||
if let Err(e) = mic_thread(&connector, stop_t, muted, echo_cancel) {
|
||||
tracing::warn!(error = %format!("{e:#}"), "mic uplink thread ended");
|
||||
}
|
||||
})
|
||||
@@ -337,24 +355,58 @@ impl Drop for MicStreamer {
|
||||
}
|
||||
}
|
||||
|
||||
fn mic_thread(connector: &Arc<NativeClient>, stop: Arc<AtomicBool>) -> Result<()> {
|
||||
/// Whether the mic echo-cancellation hooks run this session: the `echo_cancel` setting, with
|
||||
/// `PUNKTFUNK_NO_AEC=1` as a one-way override OFF. The env var wins — it is the escape hatch
|
||||
/// for a box whose canceller misbehaves, and it predates the setting; nothing turns AEC back
|
||||
/// on once it is set. Here the hook is the Communications stream category below; the PipeWire
|
||||
/// twin gates its echo-cancelled-source preference the same way.
|
||||
fn aec_enabled(echo_cancel: bool) -> bool {
|
||||
echo_cancel && !std::env::var("PUNKTFUNK_NO_AEC").is_ok_and(|v| !v.is_empty() && v != "0")
|
||||
}
|
||||
|
||||
fn mic_thread(
|
||||
connector: &Arc<NativeClient>,
|
||||
stop: Arc<AtomicBool>,
|
||||
muted: Arc<AtomicBool>,
|
||||
echo_cancel: bool,
|
||||
) -> Result<()> {
|
||||
wasapi::initialize_mta()
|
||||
.ok()
|
||||
.context("CoInitializeEx (MTA)")?;
|
||||
|
||||
let mut encoder = opus::Encoder::new(
|
||||
SAMPLE_RATE as u32,
|
||||
opus::Channels::Stereo,
|
||||
opus::Channels::Mono,
|
||||
opus::Application::Voip,
|
||||
)
|
||||
.map_err(|e| anyhow!("opus encoder: {e}"))?;
|
||||
let _ = encoder.set_bitrate(opus::Bitrate::Bits(64_000));
|
||||
// Voice tuning: 48 kbps mono is transparent for speech; in-band FEC + an assumed 10 %
|
||||
// loss let the host's decoder rebuild a lost 0xCB datagram from its successor instead
|
||||
// of concealing (datagrams are fire-and-forget — this FEC is the only redundancy).
|
||||
let _ = encoder.set_bitrate(opus::Bitrate::Bits(48_000));
|
||||
let _ = encoder.set_inband_fec(true);
|
||||
let _ = encoder.set_packet_loss_perc(10);
|
||||
|
||||
let enumerator = DeviceEnumerator::new().context("DeviceEnumerator")?;
|
||||
let device = pick_device(&enumerator, &Direction::Capture, "PUNKTFUNK_AUDIO_SOURCE")
|
||||
.context("capture endpoint (no microphone?)")?;
|
||||
let mut audio_client = device.get_iaudioclient().context("IAudioClient")?;
|
||||
let desired = WaveFormat::new(32, 32, &SampleType::Float, SAMPLE_RATE, CHANNELS, None);
|
||||
// Communications category → the endpoint's communications signal-processing chain. A
|
||||
// driver/APO stack with an echo canceller only engages it for communications-category
|
||||
// streams; the default (Other) category never did, so the downlink audio playing on
|
||||
// this box fed straight back into the host's virtual mic. Must precede Initialize
|
||||
// (SetClientProperties is a pre-init call; the wasapi crate QIs IAudioClient2 inside).
|
||||
// Best-effort: an endpoint without IAudioClient2 just keeps the default category.
|
||||
// The "Echo cancellation" setting opts out, and PUNKTFUNK_NO_AEC=1 overrides that off
|
||||
// (same lever as the Linux echo-cancel-source preference) — see `aec_enabled`.
|
||||
if aec_enabled(echo_cancel) {
|
||||
if let Err(e) = audio_client.set_properties(
|
||||
AudioClientProperties::new().set_category(StreamCategory::Communications),
|
||||
) {
|
||||
tracing::debug!(error = %e, "mic capture: Communications category not set");
|
||||
}
|
||||
}
|
||||
let desired = WaveFormat::new(32, 32, &SampleType::Float, SAMPLE_RATE, CAPT_CHANNELS, None);
|
||||
let (default_period, _min_period) =
|
||||
audio_client.get_device_period().context("device period")?;
|
||||
let mode = StreamMode::EventsShared {
|
||||
@@ -392,13 +444,33 @@ fn mic_thread(connector: &Arc<NativeClient>, stop: Arc<AtomicBool>) -> Result<()
|
||||
Err(e) => return Err(anyhow!("get_next_packet_size: {e}")),
|
||||
}
|
||||
}
|
||||
let whole = (bytes.len() / 4) * 4;
|
||||
for c in bytes.drain(..whole).collect::<Vec<u8>>().chunks_exact(4) {
|
||||
ring.push_back(f32::from_le_bytes([c[0], c[1], c[2], c[3]]));
|
||||
// One stereo capture frame (8 bytes) → one mono sample: average L/R. Autoconvert
|
||||
// already matrixed the endpoint's real layout (mono/stereo/array mic) into the
|
||||
// stereo stream we initialized, so this is the only downmix left to do.
|
||||
let stereo_frame = 4 * CAPT_CHANNELS;
|
||||
let whole = (bytes.len() / stereo_frame) * stereo_frame;
|
||||
for c in bytes
|
||||
.drain(..whole)
|
||||
.collect::<Vec<u8>>()
|
||||
.chunks_exact(stereo_frame)
|
||||
{
|
||||
let l = f32::from_le_bytes([c[0], c[1], c[2], c[3]]);
|
||||
let r = f32::from_le_bytes([c[4], c[5], c[6], c[7]]);
|
||||
ring.push_back((l + r) * 0.5);
|
||||
}
|
||||
// Ship every complete 20 ms stereo frame.
|
||||
while ring.len() >= MIC_FRAME * CHANNELS {
|
||||
let pcm: Vec<f32> = ring.drain(..MIC_FRAME * CHANNELS).collect();
|
||||
// Muted (B4): the capture client stays started and keeps its primed buffers — only
|
||||
// the sending stops. Whole frames are discarded so the ring can't grow, and `seq`
|
||||
// deliberately does NOT advance: the host sees one continuous sequence with a silent
|
||||
// pause in the middle rather than a gap the size of the mute, which its de-jitter
|
||||
// would try to conceal frame by frame.
|
||||
if muted.load(Ordering::Relaxed) {
|
||||
let drop_n = (ring.len() / MIC_FRAME) * MIC_FRAME;
|
||||
ring.drain(..drop_n);
|
||||
continue;
|
||||
}
|
||||
// Ship every complete 10 ms mono frame.
|
||||
while ring.len() >= MIC_FRAME {
|
||||
let pcm: Vec<f32> = ring.drain(..MIC_FRAME).collect();
|
||||
match encoder.encode_float(&pcm, &mut out) {
|
||||
Ok(len) => {
|
||||
let pts = std::time::SystemTime::now()
|
||||
|
||||
@@ -62,6 +62,8 @@ pub struct SettingsOverlay {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mic_enabled: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub echo_cancel: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub touch_mode: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mouse_mode: Option<String>,
|
||||
@@ -122,6 +124,9 @@ impl SettingsOverlay {
|
||||
if let Some(v) = self.mic_enabled {
|
||||
s.mic_enabled = v;
|
||||
}
|
||||
if let Some(v) = self.echo_cancel {
|
||||
s.echo_cancel = v;
|
||||
}
|
||||
if let Some(v) = &self.touch_mode {
|
||||
s.touch_mode = v.clone();
|
||||
}
|
||||
@@ -197,6 +202,9 @@ impl SettingsOverlay {
|
||||
if after.mic_enabled != before.mic_enabled {
|
||||
self.mic_enabled = Some(after.mic_enabled);
|
||||
}
|
||||
if after.echo_cancel != before.echo_cancel {
|
||||
self.echo_cancel = Some(after.echo_cancel);
|
||||
}
|
||||
if after.touch_mode != before.touch_mode {
|
||||
self.touch_mode = Some(after.touch_mode.clone());
|
||||
}
|
||||
@@ -243,6 +251,7 @@ impl SettingsOverlay {
|
||||
"compositor" => self.compositor = None,
|
||||
"audio_channels" => self.audio_channels = None,
|
||||
"mic_enabled" => self.mic_enabled = None,
|
||||
"echo_cancel" => self.echo_cancel = None,
|
||||
"touch_mode" => self.touch_mode = None,
|
||||
"mouse_mode" => self.mouse_mode = None,
|
||||
"invert_scroll" => self.invert_scroll = None,
|
||||
@@ -437,6 +446,7 @@ mod tests {
|
||||
compositor: Some("gamescope".into()),
|
||||
audio_channels: Some(6),
|
||||
mic_enabled: Some(true),
|
||||
echo_cancel: Some(false),
|
||||
touch_mode: Some("pointer".into()),
|
||||
mouse_mode: Some("desktop".into()),
|
||||
invert_scroll: Some(true),
|
||||
@@ -457,6 +467,7 @@ mod tests {
|
||||
assert_eq!(out.compositor, "gamescope");
|
||||
assert_eq!(out.audio_channels, 6);
|
||||
assert!(out.mic_enabled);
|
||||
assert!(!out.echo_cancel);
|
||||
assert_eq!(out.touch_mode, "pointer");
|
||||
assert_eq!(out.mouse_mode, "desktop");
|
||||
assert!(out.invert_scroll);
|
||||
@@ -529,6 +540,39 @@ mod tests {
|
||||
assert_eq!(o2, o);
|
||||
}
|
||||
|
||||
/// `echo_cancel` is a first-class overlay field, not an `extra` passenger: it applies,
|
||||
/// absorbs, clears, and serialises under the `echo_cancel` key the Apple and Android
|
||||
/// clients write — one catalog has to round-trip through all three.
|
||||
#[test]
|
||||
fn echo_cancel_is_a_first_class_override() {
|
||||
let base = Settings::default();
|
||||
assert!(base.echo_cancel, "the setting ships on");
|
||||
|
||||
let mut o = SettingsOverlay::default();
|
||||
let before = o.apply(&base);
|
||||
let mut after = before.clone();
|
||||
after.echo_cancel = false;
|
||||
o.absorb(&before, &after);
|
||||
assert_eq!(o.echo_cancel, Some(false));
|
||||
assert!(!o.apply(&base).echo_cancel);
|
||||
assert!(
|
||||
o.extra.is_empty(),
|
||||
"modelled fields must never land in the passthrough"
|
||||
);
|
||||
|
||||
// Serialised under the shared key, and read back from a foreign client's file.
|
||||
let text = serde_json::to_string(&o).unwrap();
|
||||
assert!(text.contains("\"echo_cancel\":false"), "{text}");
|
||||
let from_apple: SettingsOverlay =
|
||||
serde_json::from_str(r#"{"mic_enabled":true,"echo_cancel":false}"#).unwrap();
|
||||
assert_eq!(from_apple.echo_cancel, Some(false));
|
||||
assert!(from_apple.extra.is_empty());
|
||||
|
||||
assert!(o.clear("echo_cancel"));
|
||||
assert_eq!(o.echo_cancel, None);
|
||||
assert!(o.is_empty());
|
||||
}
|
||||
|
||||
/// `clear` is the explicit way back to inheriting, including the resolution tri-state.
|
||||
#[test]
|
||||
fn clear_drops_one_override() {
|
||||
|
||||
@@ -41,6 +41,9 @@ pub struct SessionParams {
|
||||
pub display_hdr: Option<punktfunk_core::quic::HdrMeta>,
|
||||
/// Stream the default microphone to the host's virtual mic source.
|
||||
pub mic_enabled: bool,
|
||||
/// Run the uplink through the platform's echo cancellation ([`Settings::echo_cancel`]).
|
||||
/// Ignored when `mic_enabled` is false; `PUNKTFUNK_NO_AEC=1` overrides it off.
|
||||
pub echo_cancel: bool,
|
||||
/// Share the clipboard with this host (the per-host `KnownHost::clipboard_sync`). The
|
||||
/// bridge additionally needs the host to advertise `HOST_CAP_CLIPBOARD`.
|
||||
pub clipboard: bool,
|
||||
@@ -145,6 +148,13 @@ pub struct Stats {
|
||||
/// received+lost (%). The OSD renders the counter line only when nonzero.
|
||||
pub lost: u32,
|
||||
pub lost_pct: f32,
|
||||
/// Mic uplink frames this window: handed to the QUIC datagram send, and shed anywhere
|
||||
/// client-side (queue-full at the producer + the pump's stale-oldest backlog governor —
|
||||
/// see [`NativeClient::mic_stats`]). Both stay 0 while the mic is off OR muted (a mute
|
||||
/// stops the sending, not the capture), so the OSD renders the mic line only while voice
|
||||
/// is actually going out — the muted case has its own badge, which does not need stats on.
|
||||
pub mic_sent: u32,
|
||||
pub mic_dropped: u32,
|
||||
/// The decode path frames actually took this window (`"vaapi"`/`"software"`, empty
|
||||
/// until the first frame) — the OSD's trailing tag; tracks a mid-session fallback.
|
||||
pub decoder: &'static str,
|
||||
@@ -199,10 +209,61 @@ pub enum SessionEvent {
|
||||
Stats(Stats),
|
||||
}
|
||||
|
||||
/// The in-stream microphone mute (B4), shared between the embedder's toggle (a keyboard chord
|
||||
/// in the presenter) and the capture callback that reads it every quantum.
|
||||
///
|
||||
/// Two flags, not one, so the indicator can never lie: `live` is raised by the pump only once
|
||||
/// the uplink is actually running, so a session whose mic is off in Settings — or whose capture
|
||||
/// device failed to open — reports "no mic here" and the chord is a documented no-op instead of
|
||||
/// silently latching a mute nothing implements. Per session by design: the mute is a moment
|
||||
/// ("don't send the doorbell"), not a preference, so it is never persisted and every new
|
||||
/// session starts unmuted.
|
||||
#[derive(Clone, Default)]
|
||||
pub struct MicControl {
|
||||
muted: Arc<AtomicBool>,
|
||||
live: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl MicControl {
|
||||
/// True when this session has a running uplink to mute at all.
|
||||
pub fn live(&self) -> bool {
|
||||
self.live.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// True when the user has muted a uplink that exists — what the OSD indicator draws.
|
||||
pub fn muted(&self) -> bool {
|
||||
self.live() && self.muted.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Flip the mute. `Some(now_muted)` when it applied, `None` when this session has no
|
||||
/// uplink (the caller says so rather than pretending something happened).
|
||||
pub fn toggle(&self) -> Option<bool> {
|
||||
if !self.live() {
|
||||
return None;
|
||||
}
|
||||
let next = !self.muted.load(Ordering::Relaxed);
|
||||
self.muted.store(next, Ordering::Relaxed);
|
||||
Some(next)
|
||||
}
|
||||
|
||||
/// The capture side's handle on the flag (the streamer reads it per quantum).
|
||||
fn flag(&self) -> Arc<AtomicBool> {
|
||||
self.muted.clone()
|
||||
}
|
||||
|
||||
/// The pump's report that the uplink came up (or went away).
|
||||
fn set_live(&self, live: bool) {
|
||||
self.live.store(live, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SessionHandle {
|
||||
pub events: async_channel::Receiver<SessionEvent>,
|
||||
pub frames: async_channel::Receiver<DecodedFrame>,
|
||||
pub stop: Arc<AtomicBool>,
|
||||
/// The in-stream mic mute. Inert (`live()` false) until the pump has the uplink running,
|
||||
/// and for the whole session when the mic is off in Settings.
|
||||
pub mic: MicControl,
|
||||
/// The pump thread. A Vulkan-Video pump SUBMITS to the shared device's decode
|
||||
/// queue — the presenter must join this before any `vkDeviceWaitIdle`/teardown
|
||||
/// (external-sync rule over every device queue).
|
||||
@@ -215,14 +276,17 @@ pub fn start(params: SessionParams) -> SessionHandle {
|
||||
let (frame_tx, frame_rx) = async_channel::bounded(2);
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let stop_w = stop.clone();
|
||||
let mic = MicControl::default();
|
||||
let mic_w = mic.clone();
|
||||
let thread = std::thread::Builder::new()
|
||||
.name("punktfunk-session".into())
|
||||
.spawn(move || pump(params, ev_tx, frame_tx, stop_w))
|
||||
.spawn(move || pump(params, ev_tx, frame_tx, stop_w, mic_w))
|
||||
.expect("spawn session thread");
|
||||
SessionHandle {
|
||||
events: ev_rx,
|
||||
frames: frame_rx,
|
||||
stop,
|
||||
mic,
|
||||
thread: Some(thread),
|
||||
}
|
||||
}
|
||||
@@ -275,6 +339,7 @@ fn pump(
|
||||
ev_tx: async_channel::Sender<SessionEvent>,
|
||||
frame_tx: async_channel::Sender<DecodedFrame>,
|
||||
stop: Arc<AtomicBool>,
|
||||
mic: MicControl,
|
||||
) {
|
||||
// PUNKTFUNK_PREFER_PYROWAVE=1 — the Phase-2 lab opt-in for the wired-LAN wavelet codec
|
||||
// (a Settings toggle is the Phase-3 productization). Riding `preferred_codec` is exactly
|
||||
@@ -430,14 +495,18 @@ fn pump(
|
||||
.ok()
|
||||
})
|
||||
.flatten();
|
||||
// The uplink, and with it the mute the embedder's chord drives. `set_live` is what makes
|
||||
// the chord (and its indicator) real: a mic turned off in Settings, or a capture device
|
||||
// that wouldn't open, leaves it false and the chord stays an honest no-op.
|
||||
let _mic = params
|
||||
.mic_enabled
|
||||
.then(|| {
|
||||
audio::MicStreamer::spawn(connector.clone())
|
||||
audio::MicStreamer::spawn(connector.clone(), mic.flag(), params.echo_cancel)
|
||||
.map_err(|e| tracing::warn!(error = %e, "mic uplink disabled"))
|
||||
.ok()
|
||||
})
|
||||
.flatten();
|
||||
mic.set_live(_mic.is_some());
|
||||
|
||||
// Live host↔client clock offset: loaded per frame (Relaxed) so mid-stream re-syncs (an NTP
|
||||
// step, drift) keep the capture-clock latency stats honest — never cached at session start.
|
||||
@@ -492,6 +561,9 @@ fn pump(
|
||||
let mut dec_path: &'static str = "";
|
||||
// The stats window keeps its own drop cursor — the OSD shows the per-window delta.
|
||||
let mut window_dropped = connector.frames_dropped();
|
||||
// Mic uplink cursor (same per-window diffing): a healthy 10 ms-frame mic reads ~100
|
||||
// sent/s; a nonzero drop delta is the queue shedding backlog (see NativeClient::mic_stats).
|
||||
let mut window_mic = connector.mic_stats();
|
||||
let mut last_kf_req: Option<Instant> = None;
|
||||
// Freeze-until-reanchor: the shared post-loss gate ([`punktfunk_core::reanchor::ReanchorGate`]).
|
||||
// Armed on any loss signal (frame-index gap, dropped-count climb, decoder wedge/demotion), it
|
||||
@@ -898,6 +970,12 @@ fn pump(
|
||||
let (pace_p50, _) = window_percentiles(&mut pace_us_win);
|
||||
let lost = dropped.saturating_sub(window_dropped) as u32;
|
||||
window_dropped = dropped;
|
||||
let mic_now = connector.mic_stats();
|
||||
let mic_sent = mic_now.sent.saturating_sub(window_mic.sent) as u32;
|
||||
let mic_dropped = (mic_now.dropped_full + mic_now.dropped_stale)
|
||||
.saturating_sub(window_mic.dropped_full + window_mic.dropped_stale)
|
||||
as u32;
|
||||
window_mic = mic_now;
|
||||
tracing::debug!(
|
||||
fps = frames_n,
|
||||
hostnet_p50_us = hn_p50,
|
||||
@@ -909,6 +987,8 @@ fn pump(
|
||||
pace_p50_us = pace_p50,
|
||||
decode_p50_us = dec_p50,
|
||||
lost,
|
||||
mic_sent,
|
||||
mic_dropped,
|
||||
total_frames,
|
||||
"stream window"
|
||||
);
|
||||
@@ -931,6 +1011,8 @@ fn pump(
|
||||
} else {
|
||||
0.0
|
||||
},
|
||||
mic_sent,
|
||||
mic_dropped,
|
||||
decoder: dec_path,
|
||||
target_kbps: connector.current_bitrate_kbps(),
|
||||
auto_rate,
|
||||
@@ -957,6 +1039,10 @@ fn pump(
|
||||
"session ended"
|
||||
);
|
||||
stop.store(true, Ordering::SeqCst);
|
||||
// The uplink is about to be dropped with the rest of this frame — stop claiming a mute
|
||||
// surface, so an embedder still holding the handle through its end path (browse mode
|
||||
// returns to the console with it) can't draw a muted mic that no longer exists.
|
||||
mic.set_live(false);
|
||||
if let Some(t) = audio_thread {
|
||||
let _ = t.join(); // exits within its 100 ms pull timeout once `stop` is set
|
||||
}
|
||||
@@ -1067,4 +1153,28 @@ mod tests {
|
||||
assert!(parse_debug_reconfigure(bad).is_none(), "{bad:?} parsed");
|
||||
}
|
||||
}
|
||||
|
||||
/// The mute is inert until the pump reports a live uplink — a session without a mic must
|
||||
/// answer "nothing to mute" rather than latching a mute and drawing the indicator.
|
||||
#[test]
|
||||
fn mic_mute_is_a_no_op_without_an_uplink() {
|
||||
let mic = MicControl::default();
|
||||
assert!(!mic.live());
|
||||
assert_eq!(mic.toggle(), None, "no uplink, nothing to toggle");
|
||||
assert!(!mic.muted(), "and nothing to show");
|
||||
|
||||
mic.set_live(true);
|
||||
assert_eq!(mic.toggle(), Some(true));
|
||||
assert!(mic.muted());
|
||||
// The capture side reads the same flag the toggle writes.
|
||||
assert!(mic.flag().load(Ordering::Relaxed));
|
||||
assert_eq!(mic.toggle(), Some(false));
|
||||
assert!(!mic.muted());
|
||||
|
||||
// A mute that outlives its uplink stops being shown (session end clears `live`).
|
||||
assert_eq!(mic.toggle(), Some(true));
|
||||
mic.set_live(false);
|
||||
assert!(!mic.muted());
|
||||
assert_eq!(mic.toggle(), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -835,6 +835,15 @@ pub struct Settings {
|
||||
pub inhibit_shortcuts: bool,
|
||||
/// Stream the default microphone to the host's virtual mic source.
|
||||
pub mic_enabled: bool,
|
||||
/// Run the mic uplink through the platform's echo cancellation (the Apple/Android clients'
|
||||
/// "Echo cancellation" toggle, same `echo_cancel` key). On Linux that means preferring an
|
||||
/// echo-cancelled PipeWire source; on Windows, asking WASAPI for the Communications stream
|
||||
/// category so the endpoint's own canceller engages. Default ON — without it, a laptop
|
||||
/// speaker playing the host's audio is heard by this device's mic and sent straight back.
|
||||
/// Only meaningful while `mic_enabled`. `PUNKTFUNK_NO_AEC=1` overrides it off (see
|
||||
/// `audio::aec_enabled`). `default` so pre-existing stores load with it on.
|
||||
#[serde(default = "default_true")]
|
||||
pub echo_cancel: bool,
|
||||
/// Requested audio channel count: 2 (stereo), 6 (5.1) or 8 (7.1). The host clamps to what it
|
||||
/// can capture; the resolved count drives the decoder + playback layout.
|
||||
pub audio_channels: u8,
|
||||
@@ -991,6 +1000,7 @@ impl Default for Settings {
|
||||
mouse_mode: "capture".into(),
|
||||
inhibit_shortcuts: true,
|
||||
mic_enabled: false,
|
||||
echo_cancel: true,
|
||||
audio_channels: 2,
|
||||
codec: "auto".into(),
|
||||
decoder: "auto".into(),
|
||||
@@ -1177,6 +1187,9 @@ mod tests {
|
||||
assert_eq!(s.forward_pad, "");
|
||||
assert!(s.fullscreen_on_stream);
|
||||
assert!(!s.library_enabled);
|
||||
// Echo cancellation post-dates every stored file: it must load ON, or an upgrade
|
||||
// would silently turn a user's echo protection off.
|
||||
assert!(s.echo_cancel);
|
||||
}
|
||||
|
||||
/// Stats-tier resolution: a pre-tier store falls back to `show_stats` (off → Off,
|
||||
|
||||
@@ -99,6 +99,18 @@ pub struct Status {
|
||||
/// Why the check couldn't complete. `update_available` is always false when set.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
/// The feed answered, but this channel has **no release published yet** — an expected
|
||||
/// state rather than a malfunction, so a caller can say so plainly instead of showing a
|
||||
/// raw "HTTP 404".
|
||||
///
|
||||
/// Deliberately NOT symmetric with the host's `UpdateStatus`, which clears `last_error`
|
||||
/// for this case: there the consumer is a human reading a console, and a red "last check
|
||||
/// failed" on an empty feed is the bug being fixed. Here the consumer is a shell script
|
||||
/// reading an exit code, so `error` stays set and `--check-update` keeps returning 1.
|
||||
/// An empty channel is not evidence that this build is current, and a mistyped
|
||||
/// `PUNKTFUNK_UPDATE_FEED` is indistinguishable from one out here.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub not_published: bool,
|
||||
}
|
||||
|
||||
/// The Ed25519 keys trusted for update manifests — pinned once in [`pf_update_check`] so the
|
||||
@@ -302,6 +314,7 @@ pub fn check(current: &str) -> Status {
|
||||
opt_in_hint: opt_in_would_help(kind, caps).then(opt_in_hint),
|
||||
notes_url: String::new(),
|
||||
error: None,
|
||||
not_published: false,
|
||||
};
|
||||
let (apply, applier) = apply_route(kind, caps);
|
||||
status.apply = apply;
|
||||
@@ -320,7 +333,8 @@ pub fn check(current: &str) -> Status {
|
||||
) {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
status.error = Some(e);
|
||||
status.not_published = e.is_not_published();
|
||||
status.error = Some(e.to_string());
|
||||
return status;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -28,6 +28,7 @@ enum RowId {
|
||||
Chroma444,
|
||||
Audio,
|
||||
Mic,
|
||||
EchoCancel,
|
||||
Pad,
|
||||
PadType,
|
||||
Touch,
|
||||
@@ -42,10 +43,10 @@ enum RowId {
|
||||
|
||||
// The couch-relevant subset grew 2026-07-31: this screen is the ONLY settings editor in
|
||||
// Gaming Mode, so a field it omits is simply unreachable there (render scale, 4:4:4,
|
||||
// scroll/shortcut behavior, fullscreen-on-stream, auto-wake, the library toggle all
|
||||
// were). Still deliberately smaller than the desktop dialogs — device pickers
|
||||
// (GPU/speaker/mic) and the profile catalog stay desktop-only.
|
||||
const ROWS: [RowId; 21] = [
|
||||
// scroll/shortcut behavior, fullscreen-on-stream, auto-wake, the library toggle and echo
|
||||
// cancellation all were). Still deliberately smaller than the desktop dialogs — device
|
||||
// pickers (GPU/speaker/mic) and the profile catalog stay desktop-only.
|
||||
const ROWS: [RowId; 22] = [
|
||||
RowId::Resolution,
|
||||
RowId::Refresh,
|
||||
RowId::RenderScale,
|
||||
@@ -57,6 +58,7 @@ const ROWS: [RowId; 21] = [
|
||||
RowId::Chroma444,
|
||||
RowId::Audio,
|
||||
RowId::Mic,
|
||||
RowId::EchoCancel,
|
||||
RowId::Pad,
|
||||
RowId::PadType,
|
||||
RowId::Touch,
|
||||
@@ -220,6 +222,9 @@ impl SettingsScreen {
|
||||
|
||||
fn row_spec(id: RowId, ctx: &Ctx) -> RowSpec {
|
||||
let s = &ctx.settings;
|
||||
// Echo cancellation only means anything while the mic streams — dimmed and inert while it
|
||||
// doesn't, the same relationship the desktop shells draw with a greyed-out row.
|
||||
let enabled = !matches!(id, RowId::EchoCancel) || s.mic_enabled;
|
||||
let (header, label, value): (Option<&'static str>, &str, String) = match id {
|
||||
RowId::Resolution => (
|
||||
Some("Stream"),
|
||||
@@ -284,6 +289,7 @@ fn row_spec(id: RowId, ctx: &Ctx) -> RowSpec {
|
||||
.into(),
|
||||
),
|
||||
RowId::Mic => (None, "Microphone", on_off(s.mic_enabled).into()),
|
||||
RowId::EchoCancel => (None, "Echo cancellation", on_off(s.echo_cancel).into()),
|
||||
RowId::Pad => (
|
||||
Some("Controller"),
|
||||
"Use controller",
|
||||
@@ -330,10 +336,10 @@ fn row_spec(id: RowId, ctx: &Ctx) -> RowSpec {
|
||||
header,
|
||||
label: label.into(),
|
||||
value: Some(value),
|
||||
value_dim: false,
|
||||
value_dim: !enabled,
|
||||
caret: false,
|
||||
adjustable: true,
|
||||
enabled: true,
|
||||
adjustable: enabled,
|
||||
enabled,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -362,7 +368,14 @@ fn detail(id: RowId) -> &'static str {
|
||||
HEVC only, and only where the host can encode it."
|
||||
}
|
||||
RowId::Audio => "The speaker layout requested from the host.",
|
||||
RowId::Mic => "Send this device's microphone to the host's virtual mic.",
|
||||
RowId::Mic => {
|
||||
"Send this device's microphone to the host's virtual mic. \
|
||||
Ctrl+Alt+Shift+V mutes and unmutes it while streaming."
|
||||
}
|
||||
RowId::EchoCancel => {
|
||||
"Stops the host's audio, playing from this device's speakers, being picked up \
|
||||
and sent back. Turn it off if your microphone already runs its own processing."
|
||||
}
|
||||
RowId::Pad => "Which pad is forwarded to the host, as player 1.",
|
||||
RowId::PadType => "The virtual pad the host creates — Automatic matches this controller.",
|
||||
RowId::Touch => {
|
||||
@@ -454,6 +467,14 @@ fn adjust(id: RowId, delta: i32, wrap: bool, ctx: &mut Ctx) -> bool {
|
||||
step_option(cur, AUDIO.len(), delta, wrap).map(|i| s.audio_channels = AUDIO[i].0)
|
||||
}
|
||||
RowId::Mic => toggle(&mut s.mic_enabled, delta, wrap),
|
||||
// Inert while the mic is off — a boundary thud, matching what the dimmed row shows.
|
||||
RowId::EchoCancel => {
|
||||
if s.mic_enabled {
|
||||
toggle(&mut s.echo_cancel, delta, wrap)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
RowId::Pad => {
|
||||
// Automatic first, then every connected pad by stable key.
|
||||
let keys: Vec<String> = std::iter::once(String::new())
|
||||
@@ -592,6 +613,40 @@ mod tests {
|
||||
assert!(!ctx.settings.mic_enabled);
|
||||
}
|
||||
|
||||
/// Echo cancellation follows the microphone: inert and dimmed while the mic is off, live
|
||||
/// the moment it goes on. A row that silently accepted a change nobody could act on would
|
||||
/// be the same lie as an enabled-looking control.
|
||||
#[test]
|
||||
fn echo_cancellation_follows_the_microphone() {
|
||||
let (mut settings, pads) = ctx_parts();
|
||||
settings.mic_enabled = false;
|
||||
assert!(settings.echo_cancel, "it ships on");
|
||||
let library = crate::library::LibraryShared::default();
|
||||
let mut ctx = Ctx {
|
||||
hosts: &[],
|
||||
library: &library,
|
||||
settings: &mut settings,
|
||||
pads: &pads,
|
||||
deck: false,
|
||||
device_name: "t",
|
||||
t: 0.0,
|
||||
};
|
||||
assert!(!row_spec(RowId::EchoCancel, &ctx).enabled);
|
||||
assert!(
|
||||
!adjust(RowId::EchoCancel, -1, false, &mut ctx),
|
||||
"mic off = thud"
|
||||
);
|
||||
assert!(!adjust(RowId::EchoCancel, 1, true, &mut ctx), "A too");
|
||||
assert!(ctx.settings.echo_cancel, "and nothing was written");
|
||||
|
||||
ctx.settings.mic_enabled = true;
|
||||
assert!(row_spec(RowId::EchoCancel, &ctx).enabled);
|
||||
assert!(adjust(RowId::EchoCancel, -1, false, &mut ctx));
|
||||
assert!(!ctx.settings.echo_cancel);
|
||||
assert!(adjust(RowId::EchoCancel, 1, true, &mut ctx));
|
||||
assert!(ctx.settings.echo_cancel);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn touch_mode_steps_and_wraps() {
|
||||
let (mut settings, pads) = ctx_parts();
|
||||
|
||||
@@ -48,6 +48,9 @@ struct Drawn {
|
||||
height: u32,
|
||||
stats: Option<String>,
|
||||
hint: Option<String>,
|
||||
/// The mic-mute badge is up. Part of the damage key like everything else here — the badge
|
||||
/// is static once drawn, so a muted stream still re-renders nothing per frame.
|
||||
mic_muted: bool,
|
||||
/// The UI scale this was drawn at, in percent — part of the damage key so dragging the window
|
||||
/// to a differently-scaled monitor re-renders the chrome at the new size instead of keeping
|
||||
/// the stale one (the text is identical, so nothing else here would notice).
|
||||
@@ -390,7 +393,12 @@ impl Overlay for SkiaOverlay {
|
||||
// spinning through the damage gate; `+ 1` keeps an active resize's step nonzero
|
||||
// even on its first frame (phase 0) so the guard below doesn't skip it.
|
||||
let resize_step = resize_phase.map_or(0, |p| (p * 120.0) as u16 + 1);
|
||||
if ctx.stats.is_none() && ctx.hint.is_none() && banner_step == 0 && resize_step == 0 {
|
||||
if ctx.stats.is_none()
|
||||
&& ctx.hint.is_none()
|
||||
&& !ctx.mic_muted
|
||||
&& banner_step == 0
|
||||
&& resize_step == 0
|
||||
{
|
||||
self.drawn = Drawn::default(); // forget content so re-show re-renders
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -402,6 +410,7 @@ impl Overlay for SkiaOverlay {
|
||||
height: ctx.height,
|
||||
stats: ctx.stats.map(str::to_owned),
|
||||
hint: ctx.hint.map(str::to_owned),
|
||||
mic_muted: ctx.mic_muted,
|
||||
scale_pct: (scale * 100.0).round() as u16,
|
||||
banner_step,
|
||||
resize_step,
|
||||
@@ -437,6 +446,11 @@ impl Overlay for SkiaOverlay {
|
||||
if let Some(stats) = &want.stats {
|
||||
draw_osd_panel(canvas, font, stats, ctx.width, scale);
|
||||
}
|
||||
// Top-RIGHT, so it never collides with the stats panel or the bottom pill: the badge
|
||||
// has to stay readable at every stats tier, including Off.
|
||||
if want.mic_muted {
|
||||
draw_mic_muted_badge(canvas, font, ctx.width, scale);
|
||||
}
|
||||
if let Some(hint) = &want.hint {
|
||||
draw_hint_pill(canvas, font, hint, ctx.width, ctx.height, 1.0, scale);
|
||||
} else if banner_step > 0 {
|
||||
@@ -660,6 +674,49 @@ fn draw_osd_panel(canvas: &Canvas, base_font: &Font, text: &str, width: u32, sca
|
||||
}
|
||||
}
|
||||
|
||||
/// The mic-mute badge: a dot in the error colour plus the words, on the same translucent pill
|
||||
/// as the rest of the chrome, pinned to the TOP-RIGHT corner.
|
||||
///
|
||||
/// It is drawn from `FrameCtx::mic_muted` alone — not from the stats text — because it must
|
||||
/// survive the stats overlay being Off, which is where most people leave it. Words, not a
|
||||
/// glyph: the chrome font is a system monospace resolved at runtime and cannot be relied on to
|
||||
/// carry a crossed-out microphone. Persistent by design (no fade): a fading indicator answers
|
||||
/// "did the chord register?" but not "am I muted right now?", and the second question is the
|
||||
/// one that matters ten minutes later.
|
||||
fn draw_mic_muted_badge(canvas: &Canvas, base_font: &Font, width: u32, scale: f32) {
|
||||
const LABEL: &str = "Microphone muted";
|
||||
// Short line — it fits any window the stream runs in, so it takes the display scale as-is.
|
||||
let font = &chrome_font(base_font, scale);
|
||||
let (_, metrics) = font.metrics();
|
||||
let line_h = metrics.descent - metrics.ascent;
|
||||
let (pad_x, pad_y) = (base::PILL_PAD_X * scale, base::PILL_PAD_Y * scale);
|
||||
let dot_r = 4.0 * scale;
|
||||
let dot_gap = 8.0 * scale;
|
||||
let text_w = font.measure_str(LABEL, None).0;
|
||||
let w = text_w + 2.0 * dot_r + dot_gap + 2.0 * pad_x;
|
||||
let h = line_h + 2.0 * pad_y;
|
||||
let margin = base::OSD_MARGIN * scale;
|
||||
let (x, y) = (width as f32 - w - margin, margin);
|
||||
canvas.draw_rrect(
|
||||
RRect::new_rect_xy(Rect::from_xywh(x, y, w, h), h / 2.0, h / 2.0),
|
||||
&Paint::new(Color4f::new(0.0, 0.0, 0.0, 0.62), None),
|
||||
);
|
||||
canvas.draw_circle(
|
||||
Point::new(x + pad_x + dot_r, y + h / 2.0),
|
||||
dot_r,
|
||||
&Paint::new(crate::theme::ERROR, None),
|
||||
);
|
||||
canvas.draw_str(
|
||||
LABEL,
|
||||
Point::new(
|
||||
x + pad_x + 2.0 * dot_r + dot_gap,
|
||||
y + pad_y - metrics.ascent,
|
||||
),
|
||||
font,
|
||||
&Paint::new(Color4f::new(1.0, 1.0, 1.0, 0.92), None),
|
||||
);
|
||||
}
|
||||
|
||||
/// The mid-stream-resize cover: a full-screen dark scrim, the shared rotating spinner, and
|
||||
/// a "Resizing…" label centered over it — so the host's 0.3–2 s virtual-display + encoder
|
||||
/// rebuild reads as a deliberate pause rather than the stream stretching to the changed
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
use crate::anim::{approach, Spring, TRAY_C, TRAY_K};
|
||||
use crate::library::{BUMP_C, BUMP_K};
|
||||
use crate::theme::{brand, white, Fonts, PanelStroke, BRAND, FAINT, W, WHITE};
|
||||
use crate::theme::{brand, white, Fonts, PanelStroke, BRAND, DIM, FAINT, W, WHITE};
|
||||
use pf_client_core::gamepad::{MenuDir, MenuEvent, MenuPulse};
|
||||
use skia_safe::{Canvas, Paint, Path, RRect, Rect};
|
||||
|
||||
@@ -35,7 +35,9 @@ pub(crate) struct RowSpec {
|
||||
pub caret: bool,
|
||||
/// Show ‹ › chevrons while focused (left/right steps the value).
|
||||
pub adjustable: bool,
|
||||
/// Action rows render dimmed when not yet actionable.
|
||||
/// Rows render dimmed when they aren't actionable: an action row's centered label loses
|
||||
/// its brand tint, a value row's label greys — the look for a setting that depends on
|
||||
/// another one being on (Echo cancellation under Microphone).
|
||||
pub enabled: bool,
|
||||
}
|
||||
|
||||
@@ -229,7 +231,7 @@ impl MenuList {
|
||||
baseline,
|
||||
W::SemiBold,
|
||||
16.0 * k,
|
||||
WHITE,
|
||||
if row.enabled { WHITE } else { DIM },
|
||||
);
|
||||
let value = row.value.as_deref().unwrap_or_default();
|
||||
let vcolor = if row.value_dim {
|
||||
|
||||
@@ -117,10 +117,11 @@ pub(super) fn resolve_split_mode(bit_depth: u8, pixel_rate: u64) -> u32 {
|
||||
/// deserves a `warn`, a default being tuned an `info`. Callers LATCH this once next to their
|
||||
/// resolved subframe state (an env re-read at reconfigure would violate the "open and
|
||||
/// reconfigure present identical init params" invariant).
|
||||
/// Linux-cfg'd like its ONLY caller (the `nvenc_cuda` query_caps latch) — Windows sessions have
|
||||
/// `subframe == forced` by construction (env opt-in only) and never consult this; without the
|
||||
/// cfg it is dead code on every Windows leg (item-level dead_code, the recurring trap).
|
||||
#[cfg(target_os = "linux")]
|
||||
/// Both direct-SDK backends latch it now: Linux at the `nvenc_cuda` query_caps latch, Windows at
|
||||
/// session init since sub-frame defaults on there too (it used to be env opt-in only, so
|
||||
/// `subframe == forced` held by construction and the item was Linux-cfg'd to avoid being dead
|
||||
/// code on the Windows leg — the recurring item-level `dead_code` trap).
|
||||
#[cfg(any(target_os = "linux", windows))]
|
||||
pub(super) fn subframe_env_forced() -> bool {
|
||||
matches!(
|
||||
std::env::var("PUNKTFUNK_NVENC_SUBFRAME").as_deref(),
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
use super::nvenc_core::{
|
||||
apply_low_latency_config, build_init_params, cached_ceiling, codec_guid, plan_range_recovery,
|
||||
resolve_slices, resolve_split_mode, resolve_split_subframe, resolve_subframe, store_ceiling,
|
||||
CeilingKey, LowLatencyConfig, NvStatusExt, RangePlan,
|
||||
subframe_env_forced, CeilingKey, LowLatencyConfig, NvStatusExt, RangePlan,
|
||||
};
|
||||
use super::nvenc_status;
|
||||
use super::{AuChunk, ChromaFormat, Codec, EncodedFrame, Encoder, EncoderCaps};
|
||||
@@ -588,6 +588,10 @@ pub struct NvencD3d11Encoder {
|
||||
input_ring_depth: Option<usize>,
|
||||
/// `NV_ENC_CAPS_ASYNC_ENCODE_SUPPORT` from the caps probe — gates the async retrieve mode.
|
||||
async_supported: bool,
|
||||
/// `NV_ENC_CAPS_SUPPORT_SUBFRAME_READBACK` from the caps probe — gates the DEFAULT-on
|
||||
/// sub-frame readback (the Linux backend's rule since its Phase 3; Windows joined after the
|
||||
/// 2026-07-31 on-glass A/B), so a GPU without it never has sub-frame forced by default.
|
||||
subframe_cap: bool,
|
||||
/// (bitstream, mapped input resource to unmap after retrieval, pts_ns, recovery-anchor) per
|
||||
/// in-flight encode. The fourth field tags the first frame encoded after a successful
|
||||
/// [`invalidate_ref_frames`](Encoder::invalidate_ref_frames) — the clean re-anchor P-frame the
|
||||
@@ -748,6 +752,7 @@ impl NvencD3d11Encoder {
|
||||
async_rt: None,
|
||||
input_ring_depth: None,
|
||||
async_supported: false,
|
||||
subframe_cap: false,
|
||||
pending: VecDeque::new(),
|
||||
frame_idx: 0,
|
||||
force_kf: false,
|
||||
@@ -922,6 +927,7 @@ impl NvencD3d11Encoder {
|
||||
nv::NV_ENC_CAPS::NV_ENC_CAPS_SUPPORT_CUSTOM_VBV_BUF_SIZE,
|
||||
);
|
||||
let async_enc = self.get_cap(enc, nv::NV_ENC_CAPS::NV_ENC_CAPS_ASYNC_ENCODE_SUPPORT);
|
||||
let subframe = self.get_cap(enc, nv::NV_ENC_CAPS::NV_ENC_CAPS_SUPPORT_SUBFRAME_READBACK);
|
||||
let _ = (api().destroy_encoder)(enc);
|
||||
|
||||
// Reject an over-range mode with a clear message instead of an opaque InvalidParam.
|
||||
@@ -955,10 +961,12 @@ impl NvencD3d11Encoder {
|
||||
self.rfi_supported = rfi != 0;
|
||||
self.custom_vbv = custom_vbv != 0;
|
||||
self.async_supported = async_enc != 0;
|
||||
self.subframe_cap = subframe != 0;
|
||||
tracing::info!(
|
||||
rfi = self.rfi_supported,
|
||||
custom_vbv = self.custom_vbv,
|
||||
async_encode = self.async_supported,
|
||||
subframe_readback = self.subframe_cap,
|
||||
max = %format!("{wmax}x{hmax}"),
|
||||
ten_bit = ten_bit != 0,
|
||||
"NVENC capabilities probed"
|
||||
@@ -1152,11 +1160,14 @@ impl NvencD3d11Encoder {
|
||||
// VIDEO_CAP_MULTI_SLICE / Moonlight slices-per-frame client gets real slices.
|
||||
// `PUNKTFUNK_NVENC_SLICES` stays the operator override in both directions.
|
||||
self.slices = resolve_slices(self.codec, 4.min(self.max_slices));
|
||||
// Split × sub-frame arbitration (Phase 8) before the ladder/ceiling key. On Windows
|
||||
// sub-frame is env-opt-in only, so resolved == forced by construction.
|
||||
let subframe_req = resolve_subframe(false);
|
||||
// Split × sub-frame arbitration (Phase 8) before the ladder/ceiling key. Sub-frame
|
||||
// defaults ON where the GPU advertises SUBFRAME_READBACK (Linux parity; validated by
|
||||
// the 2026-07-31 .173 on-glass A/B — no regression, and slice-progressive clients
|
||||
// gain the encode/wire overlap); `PUNKTFUNK_NVENC_SUBFRAME` stays the tri-state
|
||||
// operator escape in both directions.
|
||||
let subframe_req = resolve_subframe(self.subframe_cap);
|
||||
let (split_mode, subframe_req) =
|
||||
resolve_split_subframe(self.codec, split_mode, subframe_req, subframe_req);
|
||||
resolve_split_subframe(self.codec, split_mode, subframe_req, subframe_env_forced());
|
||||
// Find the highest bitrate the GPU's codec LEVEL accepts and CLAMP to it. NVENC rejects
|
||||
// `initialize_encoder` (InvalidParam) when the bitrate exceeds the level ceiling (e.g. a
|
||||
// 1 Gbps request on HEVC). Strategy: try the requested rate; if the only problem is a forced
|
||||
@@ -1318,8 +1329,7 @@ impl NvencD3d11Encoder {
|
||||
self.session_async = use_async;
|
||||
// Sub-frame chunked poll (P2f, the Windows leg of the slice pipeline): sync
|
||||
// retrieve only — chunked poll is a depth-1 sync feature; the async retrieve's
|
||||
// thread owns the bitstream lock. Sub-frame write itself stays env-gated
|
||||
// (`PUNKTFUNK_NVENC_SUBFRAME=1`) until the Windows on-glass A/B validates it.
|
||||
// thread owns the bitstream lock.
|
||||
self.subframe_chunks = self.slices >= 2 && subframe_req && !use_async;
|
||||
if self.subframe_chunks {
|
||||
tracing::info!(
|
||||
@@ -1659,8 +1669,11 @@ impl Encoder for NvencD3d11Encoder {
|
||||
let anchor = std::mem::take(&mut self.pending_anchor) && flags == 0;
|
||||
// Submit-time IDR intent: chunked poll must flag an AU's EARLY chunks before the
|
||||
// driver reports `pictureType` (only the finishing lock sees it). Exact under
|
||||
// P-only + infinite GOP: IDRs happen only when forced.
|
||||
let idr_hint = flags != 0;
|
||||
// P-only + infinite GOP: IDRs happen only when forced — or on the session-opening
|
||||
// frame, which NVENC emits as an IDR regardless of pic flags (the Linux twin's
|
||||
// `is_idr`; without the `opening` term frame 1's early chunks went out unflagged
|
||||
// and the divergence WARN fired at every session start).
|
||||
let idr_hint = flags != 0 || opening;
|
||||
let mut pic = nv::NV_ENC_PIC_PARAMS {
|
||||
version: nv::NV_ENC_PIC_PARAMS_VER,
|
||||
inputWidth: self.width,
|
||||
|
||||
@@ -43,6 +43,11 @@ pub struct FrameCtx<'a> {
|
||||
pub stats: Option<&'a str>,
|
||||
/// The capture hint (bottom-center pill, "click to capture…"); `None` = hidden.
|
||||
pub hint: Option<&'a str>,
|
||||
/// The user muted their microphone mid-stream (Ctrl+Alt+Shift+V). Draws a persistent
|
||||
/// badge, deliberately independent of the stats tier: a muted mic is a fact about what
|
||||
/// the host is hearing, and "did my mute take?" must be answerable with the overlay off.
|
||||
/// False whenever this session has no mic uplink at all — the badge never invents one.
|
||||
pub mic_muted: bool,
|
||||
/// A mid-stream Match-window resize is in flight (design/midstream-resolution-resize.md,
|
||||
/// client UX): draw a full-screen scrim + spinner so the host's 0.3–2 s virtual-display
|
||||
/// and encoder rebuild reads as an intentional pause rather than the stream stretching to
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
//! overlay tier isn't Off (Ctrl+Alt+Shift+S cycles Off → Compact → Normal → Detailed;
|
||||
//! the stdout line always carries the full Detailed text so parsers see a stable
|
||||
//! shape). Logs go to stderr (the binary configures tracing so).
|
||||
//!
|
||||
//! In-stream chords all share the Ctrl+Alt+Shift prefix: Q release/engage, M mouse model,
|
||||
//! D disconnect, S stats tier, V microphone mute.
|
||||
|
||||
use crate::input::{Capture, FingerPhase};
|
||||
use crate::overlay::{FrameCtx, Overlay, OverlayAction, OverlayFrame, SessionPhase};
|
||||
@@ -681,6 +684,23 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
|
||||
tracing::info!(tier = ?stats_verbosity, "chord: stats verbosity");
|
||||
continue;
|
||||
}
|
||||
// Mic mute (B4) — "V" for voice; M and S are taken. Per session and never
|
||||
// persisted: this is the doorbell/cough key, not a settings change. The
|
||||
// uplink keeps running while muted (see `MicStreamer::spawn`); only the
|
||||
// sending stops. A session streaming no mic says so instead of silently
|
||||
// swallowing the chord — the overlay would have nothing to show either.
|
||||
if chord && sc == Scancode::V {
|
||||
if let Some(st) = &stream {
|
||||
match st.handle.mic.toggle() {
|
||||
Some(muted) => tracing::info!(muted, "chord: microphone mute"),
|
||||
None => tracing::info!(
|
||||
"chord: microphone mute — this session streams no \
|
||||
microphone (turn it on in Settings)"
|
||||
),
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// F11 or Alt+Enter (some keyboards' Fn layer sends a media key for
|
||||
// plain F11 — the Moonlight-standard alias always exists).
|
||||
let alt_enter =
|
||||
@@ -1208,6 +1228,10 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
|
||||
let resizing = stream
|
||||
.as_ref()
|
||||
.is_some_and(|st| st.connector.is_some() && st.resize_overlay.active());
|
||||
// Read live from the session's control rather than mirrored into StreamState: the
|
||||
// pump is what knows whether an uplink exists (it may have failed to open), and a
|
||||
// mirrored copy would be the thing that goes stale at session end.
|
||||
let mic_muted = stream.as_ref().is_some_and(|st| st.handle.mic.muted());
|
||||
let ctx = FrameCtx {
|
||||
width: pw,
|
||||
height: ph,
|
||||
@@ -1217,6 +1241,7 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
|
||||
scale: overlay_scale(window.display_scale(), osd_scale_pref),
|
||||
stats,
|
||||
hint,
|
||||
mic_muted,
|
||||
resizing,
|
||||
pad: pad.as_ref().map(|p| p.name.as_str()),
|
||||
pad_pref: pad.as_ref().map(|p| p.pref),
|
||||
@@ -2077,6 +2102,16 @@ fn stats_text(
|
||||
if s.lost > 0 {
|
||||
text.push_str(&format!("\nlost {} ({:.1}%)", s.lost, s.lost_pct));
|
||||
}
|
||||
// The mic uplink line renders only while voice is actually going out (a healthy 10 ms-frame
|
||||
// uplink reads ~100 f/s) and only in Detailed — drops here are the client shedding backlog.
|
||||
// A muted mic reads 0 and drops the line; the mute has its own always-on badge instead, so
|
||||
// this stays a throughput readout rather than doubling as a mute indicator.
|
||||
if detailed && (s.mic_sent > 0 || s.mic_dropped > 0) {
|
||||
text.push_str(&format!("\nmic {} f/s", s.mic_sent));
|
||||
if s.mic_dropped > 0 {
|
||||
text.push_str(&format!(" · dropped {}", s.mic_dropped));
|
||||
}
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
@@ -2322,6 +2357,8 @@ mod tests {
|
||||
decode_ms: 1.8,
|
||||
lost: 3,
|
||||
lost_pct: 0.4,
|
||||
mic_sent: 0,
|
||||
mic_dropped: 0,
|
||||
decoder: "vulkan",
|
||||
// Old-host baseline (no reported target, 4:2:0 never asked): the tier
|
||||
// texts stay exactly what they were before the target/chroma elements.
|
||||
@@ -2405,6 +2442,30 @@ mod tests {
|
||||
assert!(!line1(&s, StatsVerbosity::Detailed).contains("4:4:4"));
|
||||
}
|
||||
|
||||
/// The mic uplink line: Detailed-only, and only while the uplink is live.
|
||||
#[test]
|
||||
fn stats_text_mic_line() {
|
||||
let (mut s, p) = sample();
|
||||
let text = |s: &Stats, v| stats_text(v, "m", s, &p, false, false, None);
|
||||
assert!(
|
||||
!text(&s, StatsVerbosity::Detailed).contains("mic"),
|
||||
"no mic line while the mic is off"
|
||||
);
|
||||
s.mic_sent = 100;
|
||||
let detailed = text(&s, StatsVerbosity::Detailed);
|
||||
assert!(detailed.contains("\nmic 100 f/s"));
|
||||
assert!(
|
||||
!detailed.contains("dropped"),
|
||||
"a healthy uplink shows no drop term"
|
||||
);
|
||||
assert!(
|
||||
!text(&s, StatsVerbosity::Normal).contains("mic"),
|
||||
"mic line is Detailed-only"
|
||||
);
|
||||
s.mic_dropped = 7;
|
||||
assert!(text(&s, StatsVerbosity::Detailed).contains("mic 100 f/s · dropped 7"));
|
||||
}
|
||||
|
||||
/// Compact omits the latency term until the presenter's first e2e window lands.
|
||||
#[test]
|
||||
fn compact_waits_for_e2e() {
|
||||
|
||||
@@ -19,6 +19,43 @@ pub const DEFAULT_FEED_BASE: &str =
|
||||
/// One fetch's wall-clock budget.
|
||||
const FETCH_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
|
||||
/// Why a fetch didn't produce a manifest.
|
||||
///
|
||||
/// Exactly one failure mode is an *expected* steady state: a channel nobody has published to
|
||||
/// answers `manifest.json` with a 404. Collapsing that into the same string as a transport or
|
||||
/// signature failure is what made an empty stable feed render as "last check failed: feed
|
||||
/// returned HTTP 404" — telling operators their box is broken when the feed is merely empty.
|
||||
/// Everything else stays a real failure, loudly.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum FeedError {
|
||||
/// This channel has no manifest at all. Note the deliberate narrowness: only a 404 on
|
||||
/// `manifest.json` itself counts. A 404 on the *signature* means the manifest exists
|
||||
/// without its proof — a half-published pair (the publisher's manifest-then-signature
|
||||
/// window, or a botched upload), which must stay fail-closed and noisy.
|
||||
NotPublished,
|
||||
/// A real failure: transport, an HTTP status that isn't the empty-channel 404, the size
|
||||
/// cap, or signature/schema rejection.
|
||||
Failed(String),
|
||||
}
|
||||
|
||||
impl FeedError {
|
||||
/// Is this the benign "nothing published on this channel yet" state?
|
||||
pub fn is_not_published(&self) -> bool {
|
||||
matches!(self, Self::NotPublished)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for FeedError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::NotPublished => f.write_str("no release has been published on this channel yet"),
|
||||
Self::Failed(msg) => f.write_str(msg),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for FeedError {}
|
||||
|
||||
/// The feed base, with a `PUNKTFUNK_UPDATE_FEED` override for tests and dev feeds. This is
|
||||
/// operator config (an env var on the process), never request-time input; the `https://` (or
|
||||
/// loopback) requirement keeps a stray value from silently downgrading the transport.
|
||||
@@ -36,9 +73,11 @@ pub fn fetch_manifest_blocking(
|
||||
channel: &str,
|
||||
keys: &[PublicKey],
|
||||
user_agent: &str,
|
||||
) -> Result<Manifest, String> {
|
||||
) -> Result<Manifest, FeedError> {
|
||||
if keys.is_empty() {
|
||||
return Err("no update key is pinned in this build".into());
|
||||
return Err(FeedError::Failed(
|
||||
"no update key is pinned in this build".into(),
|
||||
));
|
||||
}
|
||||
let agent = ureq::AgentBuilder::new()
|
||||
.timeout(FETCH_TIMEOUT)
|
||||
@@ -48,29 +87,42 @@ pub fn fetch_manifest_blocking(
|
||||
let url = format!("{base}/{channel}/manifest.json");
|
||||
let sig_url = format!("{url}.sig");
|
||||
|
||||
let body = read_capped(agent.get(&url).call().map_err(fetch_err)?)?;
|
||||
// Only the MANIFEST leg can report an empty channel; see [`FeedError::NotPublished`].
|
||||
let body = read_capped(agent.get(&url).call().map_err(manifest_err)?)?;
|
||||
let sig = read_capped(agent.get(&sig_url).call().map_err(fetch_err)?)?;
|
||||
let sig_text = String::from_utf8(sig).map_err(|_| "signature file is not text".to_string())?;
|
||||
let sig_text = String::from_utf8(sig)
|
||||
.map_err(|_| FeedError::Failed("signature file is not text".into()))?;
|
||||
|
||||
manifest::verify_and_parse(&body, &sig_text, keys, channel).map_err(|e| format!("{e:#}"))
|
||||
manifest::verify_and_parse(&body, &sig_text, keys, channel)
|
||||
.map_err(|e| FeedError::Failed(format!("{e:#}")))
|
||||
}
|
||||
|
||||
fn fetch_err(e: ureq::Error) -> String {
|
||||
/// The manifest leg: a 404 here means the channel is empty, not broken.
|
||||
fn manifest_err(e: ureq::Error) -> FeedError {
|
||||
match e {
|
||||
ureq::Error::Status(code, _) => format!("feed returned HTTP {code}"),
|
||||
other => format!("feed fetch failed: {other}"),
|
||||
ureq::Error::Status(404, _) => FeedError::NotPublished,
|
||||
other => fetch_err(other),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_capped(resp: ureq::Response) -> Result<Vec<u8>, String> {
|
||||
fn fetch_err(e: ureq::Error) -> FeedError {
|
||||
FeedError::Failed(match e {
|
||||
ureq::Error::Status(code, _) => format!("feed returned HTTP {code}"),
|
||||
other => format!("feed fetch failed: {other}"),
|
||||
})
|
||||
}
|
||||
|
||||
fn read_capped(resp: ureq::Response) -> Result<Vec<u8>, FeedError> {
|
||||
use std::io::Read as _;
|
||||
let mut buf = Vec::new();
|
||||
let mut reader = resp.into_reader().take(MAX_MANIFEST_BYTES as u64 + 1);
|
||||
reader
|
||||
.read_to_end(&mut buf)
|
||||
.map_err(|e| format!("read failed: {e}"))?;
|
||||
.map_err(|e| FeedError::Failed(format!("read failed: {e}")))?;
|
||||
if buf.len() > MAX_MANIFEST_BYTES {
|
||||
return Err("response exceeds the manifest size cap".into());
|
||||
return Err(FeedError::Failed(
|
||||
"response exceeds the manifest size cap".into(),
|
||||
));
|
||||
}
|
||||
Ok(buf)
|
||||
}
|
||||
@@ -85,7 +137,42 @@ mod tests {
|
||||
// licence to trust whatever the feed serves.
|
||||
let err =
|
||||
fetch_manifest_blocking("https://127.0.0.1:1", "stable", &[], "test").unwrap_err();
|
||||
assert!(err.contains("no update key"), "{err}");
|
||||
assert!(err.to_string().contains("no update key"), "{err}");
|
||||
// And it is a real failure — never the benign empty-channel state.
|
||||
assert!(!err.is_not_published());
|
||||
}
|
||||
|
||||
fn status(code: u16) -> ureq::Error {
|
||||
ureq::Error::Status(code, ureq::Response::new(code, "status", "").unwrap())
|
||||
}
|
||||
|
||||
/// The whole point of the split: an empty channel is not a broken feed.
|
||||
#[test]
|
||||
fn manifest_404_is_not_published_but_other_statuses_are_failures() {
|
||||
assert_eq!(manifest_err(status(404)), FeedError::NotPublished);
|
||||
for code in [403, 500, 502] {
|
||||
let e = manifest_err(status(code));
|
||||
assert!(!e.is_not_published(), "HTTP {code} must stay a failure");
|
||||
assert!(e.to_string().contains(&code.to_string()), "{e}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A missing SIGNATURE is a half-published pair, not an empty channel — the manifest leg
|
||||
/// already answered 200. Treating it as "nothing published yet" would quietly excuse the
|
||||
/// one window where a manifest exists without its proof.
|
||||
#[test]
|
||||
fn signature_404_stays_a_failure() {
|
||||
let e = fetch_err(status(404));
|
||||
assert!(!e.is_not_published());
|
||||
assert_eq!(e.to_string(), "feed returned HTTP 404");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn not_published_reads_as_plain_english() {
|
||||
assert_eq!(
|
||||
FeedError::NotPublished.to_string(),
|
||||
"no release has been published on this channel yet"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -35,6 +35,7 @@ pub mod sig;
|
||||
pub mod version;
|
||||
|
||||
pub use detect::{InstallKind, Product};
|
||||
pub use feed::FeedError;
|
||||
pub use manifest::{Manifest, MAX_MANIFEST_BYTES, SCHEMA};
|
||||
pub use sig::{verify_signature, PublicKey};
|
||||
pub use version::{canary_run, is_newer, triple, Channel};
|
||||
|
||||
@@ -63,11 +63,45 @@ fn join_host_port(host: &str, port: u16) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// Outbound mic uplink queue depth: 5 ms Opus frames, so 64 is ~320 ms of audio — far beyond
|
||||
/// any worker stall a live mic session survives anyway. On overflow the FRESH frame is dropped
|
||||
/// (a tokio mpsc can't shed from the head; by the time 320 ms are queued the stream is broken
|
||||
/// either way, and the bound is about memory, not audio quality) and logged at debug.
|
||||
const MIC_QUEUE: usize = 64;
|
||||
/// Outbound mic uplink queue depth: desktop clients send 10 ms Opus frames (mobile still 20 ms),
|
||||
/// so 12 is ~120–240 ms of audio — the hard memory cap, not the working depth. (The old 64 was
|
||||
/// mislabeled "~320 ms of 5 ms frames"; the frames were 20 ms, so it really allowed 1.28 s, and
|
||||
/// since a filled tokio mpsc can only drop the FRESH frame, every queued frame became permanent
|
||||
/// standing latency once a stall filled it.) The pump's mic task now sheds the OLDEST frames
|
||||
/// whenever more than [`MIC_BACKLOG_MAX`] are still waiting, so a stall costs a short dropout
|
||||
/// and heals the moment the worker catches up. The producer stays non-blocking: on overflow it
|
||||
/// still drops the fresh frame (logged at debug) — the shed loop keeps that a rare event.
|
||||
const MIC_QUEUE: usize = 12;
|
||||
|
||||
/// The mic backlog the pump tolerates before shedding oldest-first: ~60 ms of 10 ms frames —
|
||||
/// enough slack to ride out an encode/send hiccup, small enough that voice stays conversational
|
||||
/// and never accrues a session-long standing delay.
|
||||
pub(crate) const MIC_BACKLOG_MAX: usize = 6;
|
||||
|
||||
/// Mic uplink counters, shared between the producer side ([`NativeClient::send_mic`]) and the
|
||||
/// pump's mic task. All monotonic for the session; a stats HUD windows them by diffing
|
||||
/// successive [`NativeClient::mic_stats`] snapshots (the `frames_dropped` pattern).
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct MicUplinkCounters {
|
||||
/// Frames handed to the QUIC datagram send (past every client-side queue).
|
||||
pub(crate) sent: AtomicU64,
|
||||
/// Frames shed at enqueue — the worker queue was full ([`MIC_QUEUE`]).
|
||||
pub(crate) dropped_full: AtomicU64,
|
||||
/// Frames shed by the pump's backlog governor (stale-oldest past [`MIC_BACKLOG_MAX`]).
|
||||
pub(crate) dropped_stale: AtomicU64,
|
||||
}
|
||||
|
||||
/// A [`NativeClient::mic_stats`] snapshot: cumulative mic uplink frame counts per stage.
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct MicUplinkStats {
|
||||
/// Frames handed to the QUIC datagram send.
|
||||
pub sent: u64,
|
||||
/// Frames shed at enqueue (worker queue full).
|
||||
pub dropped_full: u64,
|
||||
/// Frames shed by the pump's backlog governor (stale-oldest — see [`MIC_QUEUE`]'s
|
||||
/// self-healing note).
|
||||
pub dropped_stale: u64,
|
||||
}
|
||||
|
||||
/// Outbound control-request queue depth. The requests are sparse (mode switches, keyframe
|
||||
/// requests, ~1.3 loss reports/s, clock re-syncs) — 32 is hours of headroom; a full queue means
|
||||
@@ -101,9 +135,12 @@ pub struct NativeClient {
|
||||
cursor_state: Mutex<Receiver<crate::quic::CursorState>>,
|
||||
input_tx: tokio::sync::mpsc::UnboundedSender<InputEvent>,
|
||||
/// Outbound mic frames `(seq, pts_ns, opus)` → encoded as 0xCB datagrams by the worker.
|
||||
/// Bounded ([`MIC_QUEUE`]): a wedged worker drops fresh frames (logged) instead of queueing
|
||||
/// audio-latency (and memory) without limit — mic is best-effort end to end.
|
||||
/// Bounded ([`MIC_QUEUE`]): the pump sheds stale frames oldest-first and a full queue drops
|
||||
/// the fresh one (logged) instead of queueing audio-latency (and memory) without limit —
|
||||
/// mic is best-effort end to end, and a standing backlog is worse than a dropout.
|
||||
mic_tx: tokio::sync::mpsc::Sender<(u32, u64, Vec<u8>)>,
|
||||
/// Mic uplink counters (sent / dropped per stage) — see [`NativeClient::mic_stats`].
|
||||
mic_stats: Arc<MicUplinkCounters>,
|
||||
/// Outbound 0xCC rich-input plane, PRE-ENCODED datagrams: [`RichInput`] touchpad/motion
|
||||
/// (encoded in [`NativeClient::send_rich_input`]) and stylus [`crate::quic::PenBatch`]es
|
||||
/// (encoded in [`NativeClient::send_pen`]) share the channel — the worker's task just
|
||||
@@ -402,6 +439,7 @@ impl NativeClient {
|
||||
let probe = Arc::new(Mutex::new(ProbeState::default()));
|
||||
let frames_dropped = Arc::new(AtomicU64::new(0));
|
||||
let fec_recovered = Arc::new(AtomicU64::new(0));
|
||||
let mic_stats = Arc::new(MicUplinkCounters::default());
|
||||
let hot_tids = Arc::new(Mutex::new(Vec::new()));
|
||||
let clock_offset = Arc::new(AtomicI64::new(0));
|
||||
let decode_lat = Arc::new(Mutex::new(DecodeLatAcc::default()));
|
||||
@@ -416,6 +454,7 @@ impl NativeClient {
|
||||
let probe_w = probe.clone();
|
||||
let frames_dropped_w = frames_dropped.clone();
|
||||
let fec_recovered_w = fec_recovered.clone();
|
||||
let mic_stats_w = mic_stats.clone();
|
||||
let hot_tids_w = hot_tids.clone();
|
||||
let clock_offset_w = clock_offset.clone();
|
||||
let decode_lat_w = decode_lat.clone();
|
||||
@@ -481,6 +520,7 @@ impl NativeClient {
|
||||
probe: probe_w,
|
||||
frames_dropped: frames_dropped_w,
|
||||
fec_recovered: fec_recovered_w,
|
||||
mic_stats: mic_stats_w,
|
||||
hot_tids: hot_tids_w,
|
||||
clock_offset: clock_offset_w,
|
||||
decode_lat: decode_lat_w,
|
||||
@@ -516,6 +556,7 @@ impl NativeClient {
|
||||
cursor_state: Mutex::new(cursor_state_rx),
|
||||
input_tx,
|
||||
mic_tx,
|
||||
mic_stats,
|
||||
rich_input_tx,
|
||||
ctrl_tx,
|
||||
clip: Mutex::new(clip_event_rx),
|
||||
@@ -720,6 +761,18 @@ impl NativeClient {
|
||||
self.fec_recovered.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Cumulative mic uplink frame counts per stage: handed to the QUIC datagram send, shed at
|
||||
/// enqueue (worker queue full), and shed by the pump's backlog governor (stale-oldest — see
|
||||
/// [`MIC_QUEUE`]'s self-healing note). All monotonic for the session; a stats HUD windows
|
||||
/// them by diffing successive reads, like [`frames_dropped`](Self::frames_dropped).
|
||||
pub fn mic_stats(&self) -> MicUplinkStats {
|
||||
MicUplinkStats {
|
||||
sent: self.mic_stats.sent.load(Ordering::Relaxed),
|
||||
dropped_full: self.mic_stats.dropped_full.load(Ordering::Relaxed),
|
||||
dropped_stale: self.mic_stats.dropped_stale.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the underlying QUIC session has ended — the worker's connection-close watcher set the
|
||||
/// shutdown flag (`conn.closed()` fired: a host suspend / crash / network drop idle-timed the
|
||||
/// connection out, or the host closed it), or a deliberate [`disconnect_quit`](Self::disconnect_quit)
|
||||
@@ -1144,8 +1197,10 @@ impl NativeClient {
|
||||
match self.mic_tx.try_send((seq, pts_ns, opus)) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(TrySendError::Full(_)) => {
|
||||
// Bounded queue full = the worker stalled for ~MIC_QUEUE x 5 ms. Shed this
|
||||
// frame (mic is best-effort end to end) instead of queueing latency/memory.
|
||||
// Bounded queue full = the worker stalled long enough to outrun even the
|
||||
// pump's oldest-first shed. Drop this frame (mic is best-effort end to end)
|
||||
// instead of queueing latency/memory; the counter keeps the loss visible.
|
||||
self.mic_stats.dropped_full.fetch_add(1, Ordering::Relaxed);
|
||||
tracing::debug!("mic uplink queue full — dropping frame");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -68,6 +68,7 @@ pub(super) async fn run_pump(args: WorkerArgs) {
|
||||
probe,
|
||||
frames_dropped,
|
||||
fec_recovered,
|
||||
mic_stats,
|
||||
hot_tids,
|
||||
clock_offset,
|
||||
decode_lat,
|
||||
@@ -96,11 +97,20 @@ pub(super) async fn run_pump(args: WorkerArgs) {
|
||||
tokio::spawn(input_task::run(conn.clone(), input_rx, gamepad_snapshots));
|
||||
|
||||
// Mic task: embedder Opus mic frames → 0xCB uplink datagrams (best-effort, dropped on loss).
|
||||
// Self-healing latency bound: every frame still queued once this task catches up is standing
|
||||
// mic delay from then on, so a frame with more than [`MIC_BACKLOG_MAX`] successors already
|
||||
// waiting is shed as stale instead of sent — a stall costs a short dropout, not a
|
||||
// session-long lag (see [`MIC_QUEUE`]). The counters feed [`NativeClient::mic_stats`].
|
||||
let mic_conn = conn.clone();
|
||||
tokio::spawn(async move {
|
||||
while let Some((seq, pts_ns, opus)) = mic_rx.recv().await {
|
||||
if mic_rx.len() > MIC_BACKLOG_MAX {
|
||||
mic_stats.dropped_stale.fetch_add(1, Ordering::Relaxed);
|
||||
continue;
|
||||
}
|
||||
let d = crate::quic::encode_mic_datagram(seq, pts_ns, &opus);
|
||||
let _ = mic_conn.send_datagram(d.into());
|
||||
mic_stats.sent.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -66,6 +66,9 @@ pub(crate) struct WorkerArgs {
|
||||
pub(crate) probe: Arc<Mutex<ProbeState>>,
|
||||
pub(crate) frames_dropped: Arc<AtomicU64>,
|
||||
pub(crate) fec_recovered: Arc<AtomicU64>,
|
||||
/// Mic uplink counters (see [`NativeClient::mic_stats`]): the pump's mic task counts wire
|
||||
/// sends and its own stale-shed drops here; the producer counts queue-full drops.
|
||||
pub(crate) mic_stats: Arc<MicUplinkCounters>,
|
||||
pub(crate) hot_tids: Arc<Mutex<Vec<i32>>>,
|
||||
/// The live clock offset (see [`NativeClient::clock_offset`]): the worker seeds it with the
|
||||
/// connect-time estimate; the control task's mid-stream re-syncs update it.
|
||||
|
||||
@@ -114,7 +114,13 @@ pub use stats::Stats;
|
||||
/// v13: added `punktfunk_connection_send_pen` — the stylus wire plane
|
||||
/// (design/pen-tablet-input.md): a client sends `RICH_PEN` sample batches once the host
|
||||
/// advertises `HOST_CAP_PEN`. Additive and capability-gated, so [`WIRE_VERSION`] is unchanged.
|
||||
pub const ABI_VERSION: u32 = 13;
|
||||
/// v14: added `punktfunk_connection_report_phase` + the `PUNKTFUNK_CLIENT_CAP_PHASE_LOCK` mirror
|
||||
/// — the phase-locked capture reporter (design/phase-locked-capture.md): a client that advertises
|
||||
/// the cap reports its next display latch (already converted to host clock), the panel period, an
|
||||
/// uncertainty and the circular arrival-lead statistic the host's controller steers on. Additive;
|
||||
/// the wire grows only a new control message (`PhaseReport`, 0x32) an old host never reads and a
|
||||
/// strict-prefix append on the 0xCF host-timing tail, so [`WIRE_VERSION`] is unchanged.
|
||||
pub const ABI_VERSION: u32 = 14;
|
||||
|
||||
/// The punktfunk/1 **wire** version — what `Hello`/`Welcome` carry and hosts equality-check.
|
||||
/// Deliberately its own constant: [`ABI_VERSION`] tracks the embeddable **C surface**
|
||||
|
||||
@@ -115,6 +115,47 @@ pub trait VirtualMic: Send {
|
||||
fn channels(&self) -> u32 {
|
||||
CHANNELS as u32
|
||||
}
|
||||
|
||||
/// The adaptive de-jitter target (per-channel samples) the pump measured from uplink
|
||||
/// arrival jitter (see `mic_jitter`). A backend with a jitter ring primes around this PLUS
|
||||
/// one of its own consumer quanta: the ring must absorb arrival burstiness (the pump's
|
||||
/// number) and pull granularity (the backend's own), and neither may buy the other's depth
|
||||
/// — a 2048-frame recorder gets its one quantum, not three. Never called ⇒ the backend
|
||||
/// keeps its legacy fixed constants, which is also how `PUNKTFUNK_MIC_LEGACY_BUFFER=1`
|
||||
/// works (the pump simply never drives the target). Default: no ring, ignored.
|
||||
fn set_target_depth(&self, _samples_per_ch: usize) {}
|
||||
|
||||
/// `(buffered, prime_target)` of the backend's jitter ring in per-channel samples — read
|
||||
/// by the pump's creep trim + telemetry. `None` while unknown (consumer not yet running,
|
||||
/// or a backend without a ring).
|
||||
fn depth(&self) -> Option<(usize, usize)> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Reset-on-read telemetry counters (see [`MicBackendStats`]). Default: all zero.
|
||||
fn take_stats(&self) -> MicBackendStats {
|
||||
MicBackendStats::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset-on-read counters a [`VirtualMic`] backend reports into the pump's periodic mic
|
||||
/// telemetry line ("mic uplink health").
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub struct MicBackendStats {
|
||||
/// Full-drain re-prime arms: the ring emptied and gates on silence until the target depth
|
||||
/// rebuilds. One per talk spurt is normal; several per second mid-speech is the crackle.
|
||||
pub reprimes: u64,
|
||||
/// Per-channel samples dropped by the ring's overflow cap (drop-oldest).
|
||||
pub overflow_dropped: u64,
|
||||
}
|
||||
|
||||
/// One-release escape hatch (docs: configuration → Audio / microphone):
|
||||
/// `PUNKTFUNK_MIC_LEGACY_BUFFER=1` keeps the pre-adaptive fixed mic buffering — the pump never
|
||||
/// drives the backend target (so the rings stay on their legacy constants: 48 ms prime /
|
||||
/// 120 ms cap on Windows, the 3-quanta clamp on Linux) and never creep-trims depth.
|
||||
pub(crate) fn mic_legacy_buffer() -> bool {
|
||||
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
|
||||
*ON.get_or_init(|| std::env::var_os("PUNKTFUNK_MIC_LEGACY_BUFFER").is_some_and(|v| v != "0"))
|
||||
}
|
||||
|
||||
/// Open a virtual microphone with `channels` interleaved channels (1 or 2). Linux: a PipeWire
|
||||
@@ -152,5 +193,6 @@ mod wasapi_mic;
|
||||
#[path = "audio/wiring_plan.rs"]
|
||||
pub(crate) mod wiring_plan;
|
||||
|
||||
mod mic_jitter;
|
||||
mod mic_pump;
|
||||
pub use mic_pump::MicPump;
|
||||
pub use mic_pump::{MicFrame, MicPump};
|
||||
|
||||
@@ -29,10 +29,10 @@
|
||||
|
||||
mod stream_sink;
|
||||
|
||||
use super::{AudioCapturer, VirtualMic, SAMPLE_RATE};
|
||||
use super::{AudioCapturer, MicBackendStats, VirtualMic, SAMPLE_RATE};
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::mpsc::{sync_channel, Receiver, RecvTimeoutError};
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
@@ -218,6 +218,26 @@ pub struct PwMicSource {
|
||||
alive: Arc<AtomicBool>,
|
||||
/// One-shot flush request, consumed by the process callback (clears the jitter ring).
|
||||
flush: Arc<AtomicBool>,
|
||||
/// Ring policy/telemetry shared with the RT process callback (see [`MicRingShared`]).
|
||||
ring: Arc<MicRingShared>,
|
||||
}
|
||||
|
||||
/// Atomics shared between [`PwMicSource`] (the pump's side) and the RT process callback: the
|
||||
/// pump's adaptive de-jitter target in, ring depth/prime gauges + reset-on-read counters out.
|
||||
/// All `Relaxed` — a slowly-moving target and telemetry, not synchronization.
|
||||
#[derive(Default)]
|
||||
struct MicRingShared {
|
||||
/// Pump-set jitter target (per-channel samples). `0` = the pump never spoke (legacy mode,
|
||||
/// or its first estimate hasn't landed) → the callback keeps the historical 3-quanta clamp.
|
||||
target: AtomicUsize,
|
||||
/// Ring depth after the last callback (per-channel samples).
|
||||
depth: AtomicUsize,
|
||||
/// Effective prime target of the last callback (per-channel samples).
|
||||
prime: AtomicUsize,
|
||||
/// Full-drain re-prime arms (see [`MicBackendStats`]).
|
||||
reprimes: AtomicU64,
|
||||
/// Per-channel samples dropped by the overflow cap.
|
||||
overflow: AtomicU64,
|
||||
}
|
||||
|
||||
impl PwMicSource {
|
||||
@@ -234,11 +254,13 @@ impl PwMicSource {
|
||||
// service started before the user session) must surface as an open ERROR — engaging the
|
||||
// pump's backoff — not as an instantly-dead instance the pump would churn on.
|
||||
let (ready_tx, ready_rx) = sync_channel::<Result<()>>(1);
|
||||
let (alive_t, flush_t) = (alive.clone(), flush.clone());
|
||||
let ring = Arc::new(MicRingShared::default());
|
||||
let (alive_t, flush_t, ring_t) = (alive.clone(), flush.clone(), ring.clone());
|
||||
thread::Builder::new()
|
||||
.name("punktfunk-pw-mic".into())
|
||||
.spawn(move || {
|
||||
if let Err(e) = mic_pw_thread(pcm_rx, quit_rx, channels, flush_t, ready_tx) {
|
||||
if let Err(e) = mic_pw_thread(pcm_rx, quit_rx, channels, flush_t, ring_t, ready_tx)
|
||||
{
|
||||
// Reaching here is always a setup/open failure (once the mainloop runs it exits
|
||||
// Ok) — and it was already reported to the pump via the ready handshake, which
|
||||
// owns the throttled operator-facing warn. Keep only a debug breadcrumb.
|
||||
@@ -255,6 +277,7 @@ impl PwMicSource {
|
||||
quit: quit_tx,
|
||||
alive,
|
||||
flush,
|
||||
ring,
|
||||
}),
|
||||
Ok(Err(e)) => Err(e),
|
||||
Err(_) => Err(anyhow!("pipewire virtual-mic init timed out")),
|
||||
@@ -291,6 +314,20 @@ impl VirtualMic for PwMicSource {
|
||||
fn channels(&self) -> u32 {
|
||||
self.channels
|
||||
}
|
||||
fn set_target_depth(&self, samples_per_ch: usize) {
|
||||
self.ring.target.store(samples_per_ch, Ordering::Relaxed);
|
||||
}
|
||||
fn depth(&self) -> Option<(usize, usize)> {
|
||||
let prime = self.ring.prime.load(Ordering::Relaxed);
|
||||
// 0 = the process callback hasn't run yet (no consumer) — nothing meaningful to report.
|
||||
(prime > 0).then(|| (self.ring.depth.load(Ordering::Relaxed), prime))
|
||||
}
|
||||
fn take_stats(&self) -> MicBackendStats {
|
||||
MicBackendStats {
|
||||
reprimes: self.ring.reprimes.swap(0, Ordering::Relaxed),
|
||||
overflow_dropped: self.ring.overflow.swap(0, Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Producer-side state for the virtual-mic loop: incoming decoded PCM and a small ring buffer
|
||||
@@ -303,6 +340,8 @@ struct MicUserData {
|
||||
primed: bool,
|
||||
/// One-shot flush request from [`PwMicSource::discard`] (stale-audio drop after a gap).
|
||||
flush: Arc<AtomicBool>,
|
||||
/// Pump-driven ring policy + telemetry (see [`MicRingShared`]).
|
||||
shared: Arc<MicRingShared>,
|
||||
/// When the process callback last ran — a long gap means the ring content predates the
|
||||
/// current consumer (the stream idles with no recorder attached) and must be dropped.
|
||||
last_run: Option<std::time::Instant>,
|
||||
@@ -318,6 +357,7 @@ fn mic_pw_thread(
|
||||
quit_rx: pipewire::channel::Receiver<Terminate>,
|
||||
channels: u32,
|
||||
flush: Arc<AtomicBool>,
|
||||
shared: Arc<MicRingShared>,
|
||||
ready: std::sync::mpsc::SyncSender<Result<()>>,
|
||||
) -> Result<()> {
|
||||
use pipewire as pw;
|
||||
@@ -400,6 +440,7 @@ fn mic_pw_thread(
|
||||
channels: channels as usize,
|
||||
primed: false,
|
||||
flush,
|
||||
shared,
|
||||
last_run: None,
|
||||
};
|
||||
|
||||
@@ -473,15 +514,31 @@ fn mic_pw_thread(
|
||||
);
|
||||
}
|
||||
|
||||
// Adaptive jitter buffer. The client pushes 5 ms frames; the recorder pulls a
|
||||
// whole *quantum* (often 20–43 ms) from an independent clock. A drain of one
|
||||
// quantum must not outrun what's buffered, or every call underruns to silence
|
||||
// (the original ~58% gaps). So prime to ~3 quanta before producing, hold there,
|
||||
// and re-prime only after a genuine full drain (the client went quiet). The ring
|
||||
// is capped at a few quanta so latency stays bounded.
|
||||
let target = (3 * want).clamp(720 * ud.channels, 9600 * ud.channels);
|
||||
// Jitter buffer. The client pushes frames on its own clock; the recorder
|
||||
// pulls a whole *quantum* (often 20–43 ms) from an independent one. A drain
|
||||
// of one quantum must not outrun what's buffered, or every call underruns
|
||||
// to silence (the original ~58% gaps). Prime target = one quantum (the pull
|
||||
// granularity) + the pump's measured-jitter target (arrival burstiness —
|
||||
// see `mic_jitter`), re-priming only after a genuine full drain (the client
|
||||
// went quiet). Until the pump's first estimate lands — and forever under
|
||||
// PUNKTFUNK_MIC_LEGACY_BUFFER — the historical 3-quanta clamp applies; that
|
||||
// formula scaled with the RECORDING APP's quantum, so a 2048-frame recorder
|
||||
// bought 128+ ms of standing mic latency for jitter it never had.
|
||||
let pump_target = ud.shared.target.load(Ordering::Relaxed) * ud.channels;
|
||||
let target = if pump_target == 0 {
|
||||
(3 * want).clamp(720 * ud.channels, 9600 * ud.channels)
|
||||
} else {
|
||||
want + pump_target
|
||||
};
|
||||
let mut dropped = 0usize;
|
||||
while ud.ring.len() > target.max(want) + want {
|
||||
ud.ring.pop_front(); // bound latency: drop the oldest beyond ~1 quantum slack
|
||||
dropped += 1;
|
||||
}
|
||||
if dropped > 0 {
|
||||
ud.shared
|
||||
.overflow
|
||||
.fetch_add((dropped / ud.channels) as u64, Ordering::Relaxed);
|
||||
}
|
||||
if !ud.primed && ud.ring.len() >= target {
|
||||
ud.primed = true;
|
||||
@@ -501,9 +558,17 @@ fn mic_pw_thread(
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if ud.ring.is_empty() {
|
||||
if ud.primed && ud.ring.is_empty() {
|
||||
ud.primed = false; // fully drained — re-prime before producing again
|
||||
ud.shared.reprimes.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
// Publish depth/prime for the pump's creep trim + telemetry.
|
||||
ud.shared
|
||||
.depth
|
||||
.store(ud.ring.len() / ud.channels, Ordering::Relaxed);
|
||||
ud.shared
|
||||
.prime
|
||||
.store(target / ud.channels, Ordering::Relaxed);
|
||||
let chunk = data.chunk_mut();
|
||||
*chunk.offset_mut() = 0;
|
||||
*chunk.stride_mut() = stride as _;
|
||||
|
||||
@@ -0,0 +1,471 @@
|
||||
//! Backend-agnostic de-jitter for the client-mic uplink. The pump feeds every received `0xCB`
|
||||
//! frame (with its datagram sequence — see [`MicFrame`]) through [`MicDejitter`], which puts a
|
||||
//! two-frame reorder window in front of the Opus decoder, asks for libopus concealment on true
|
||||
//! sequence gaps (so a lost datagram doesn't drain the backend ring into a silence + re-prime
|
||||
//! cycle), and measures inter-arrival jitter into the adaptive target depth the backend rings
|
||||
//! prime against. Mirrors the client downlink's `AudioGapTracker` (punktfunk-core `audio.rs`):
|
||||
//! the same wrapping-sequence gap rules, grown the reorder hold and the jitter estimator the
|
||||
//! uplink needs — the downlink's jitter rings own their depth; the uplink's live in the
|
||||
//! backends and take their target from here.
|
||||
|
||||
use super::MicFrame;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// What the pump should do next, in playback order. `Frame` is a real frame to decode;
|
||||
/// `Conceal` is one frame of libopus packet-loss concealment (`decode_float(&[], …)`, sized to
|
||||
/// the last real frame) standing in for a lost one.
|
||||
pub(super) enum Deliver {
|
||||
Frame(MicFrame),
|
||||
Conceal,
|
||||
}
|
||||
|
||||
/// Most concealment frames one gap may synthesize (~100 ms at the common 20 ms frames) — the
|
||||
/// client downlink's `AudioGapTracker` cap, scaled to the uplink's bigger frames. libopus PLC
|
||||
/// fades to silence after a few frames anyway; past the cap the ring's underrun/re-prime path
|
||||
/// takes over, as before.
|
||||
const MAX_CONCEAL_FRAMES: u32 = 5;
|
||||
|
||||
/// How long one out-of-order frame may wait for its missing predecessor before the gap is
|
||||
/// concealed and the held frame plays: ~one 20 ms frame interval plus scheduling slack. With
|
||||
/// the incoming frame that is a two-frame window — anything needing more is treated as loss,
|
||||
/// exactly like the client downlink (which holds nothing at all).
|
||||
const HOLD_MAX: Duration = Duration::from_millis(30);
|
||||
|
||||
/// Arrival gaps above this are talk-spurt pauses (DTX, mute), not network jitter — folding
|
||||
/// them into the estimate would balloon the target after every pause.
|
||||
const PAUSE_GAP: Duration = Duration::from_millis(250);
|
||||
|
||||
/// The sliding jitter window: [`BUCKETS`] × [`BUCKET_LEN`] of per-bucket max inter-arrival gap.
|
||||
/// Long enough that the old bursty Mac cadence (~two 20 ms frames every ~42 ms — the 2026-07-03
|
||||
/// "crackling mic", see `wasapi_mic.rs`) is always represented; short enough that a one-off
|
||||
/// spike ages out in a few seconds.
|
||||
const BUCKETS: usize = 4;
|
||||
const BUCKET_LEN: Duration = Duration::from_millis(1000);
|
||||
|
||||
/// Target before the estimator has a full [`WARMUP`] of evidence: roughly the old fixed 48 ms
|
||||
/// prime minus the one consumer quantum the backends add — the crackle-safe assumption that
|
||||
/// the client might be an old bursty one. Modern 10 ms-cadence clients drop to ~15–25 ms once
|
||||
/// measured (after their first talk pause re-primes the ring at the lower target).
|
||||
const DEFAULT_TARGET_MS: u32 = 40;
|
||||
/// Headroom over the measured worst burst gap.
|
||||
const TARGET_PAD_MS: u32 = 5;
|
||||
/// How long the estimator distrusts its own (young) window and keeps the default as a floor.
|
||||
const WARMUP: Duration = Duration::from_secs(1);
|
||||
|
||||
/// Reset-on-read snapshot for the pump's telemetry line. Counters cover the window since the
|
||||
/// last read; `cadence_ms`/`frame_ms` are gauges (EWMAs).
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub(super) struct DejitterStats {
|
||||
pub seq_gaps: u64,
|
||||
pub concealed: u64,
|
||||
pub reorders: u64,
|
||||
pub late_drops: u64,
|
||||
/// EWMA inter-arrival gap — the uplink's burst cadence as seen by the pump.
|
||||
pub cadence_ms: f32,
|
||||
/// Nominal frame duration from consecutive frames' pts deltas (what the client encodes).
|
||||
pub frame_ms: f32,
|
||||
}
|
||||
|
||||
pub(super) struct MicDejitter {
|
||||
/// Sequence the uplink owes next; `None` before the first frame / after
|
||||
/// [`reset_stream`](Self::reset_stream).
|
||||
expected: Option<u32>,
|
||||
/// The reorder window: at most ONE newer frame parked ≤ [`HOLD_MAX`] for its predecessor.
|
||||
held: Option<(Instant, MicFrame)>,
|
||||
// --- adaptive target (inter-arrival jitter) ---
|
||||
first_arrival: Option<Instant>,
|
||||
last_arrival: Option<Instant>,
|
||||
bucket_start: Option<Instant>,
|
||||
bucket_idx: usize,
|
||||
gap_max: [f32; BUCKETS],
|
||||
ewma_gap_ms: f32,
|
||||
// --- pts bookkeeping ---
|
||||
/// `(seq, pts_ns)` of the newest ingested frame — a pts delta over exactly one seq step is
|
||||
/// the client's true frame duration.
|
||||
last_pts: Option<(u32, u64)>,
|
||||
frame_ms: f32,
|
||||
// --- counters (reset on read) ---
|
||||
seq_gaps: u64,
|
||||
concealed: u64,
|
||||
reorders: u64,
|
||||
late_drops: u64,
|
||||
}
|
||||
|
||||
impl MicDejitter {
|
||||
pub(super) fn new() -> Self {
|
||||
MicDejitter {
|
||||
expected: None,
|
||||
held: None,
|
||||
first_arrival: None,
|
||||
last_arrival: None,
|
||||
bucket_start: None,
|
||||
bucket_idx: 0,
|
||||
gap_max: [0.0; BUCKETS],
|
||||
ewma_gap_ms: 0.0,
|
||||
last_pts: None,
|
||||
frame_ms: 0.0,
|
||||
seq_gaps: 0,
|
||||
concealed: 0,
|
||||
reorders: 0,
|
||||
late_drops: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Feed one received frame; playback-ordered work is appended to `out`.
|
||||
pub(super) fn ingest(&mut self, now: Instant, frame: MicFrame, out: &mut Vec<Deliver>) {
|
||||
self.record_arrival(now, frame.seq, frame.pts_ns);
|
||||
self.accept(now, frame, out);
|
||||
}
|
||||
|
||||
fn accept(&mut self, now: Instant, frame: MicFrame, out: &mut Vec<Deliver>) {
|
||||
let Some(expected) = self.expected else {
|
||||
// First frame (or first after a reset): the chain starts here.
|
||||
self.expected = Some(frame.seq.wrapping_add(1));
|
||||
out.push(Deliver::Frame(frame));
|
||||
return;
|
||||
};
|
||||
let delta = frame.seq.wrapping_sub(expected);
|
||||
if delta == 0 {
|
||||
self.expected = Some(expected.wrapping_add(1));
|
||||
out.push(Deliver::Frame(frame));
|
||||
self.release_held_in_order(out);
|
||||
} else if delta > u32::MAX / 2 {
|
||||
// Duplicate of, or older than, something already played — the window moved on.
|
||||
self.late_drops += 1;
|
||||
} else if let Some(held_seq) = self.held.as_ref().map(|(_, h)| h.seq) {
|
||||
if held_seq == frame.seq {
|
||||
self.late_drops += 1; // duplicate of the held frame
|
||||
return;
|
||||
}
|
||||
// Two newer-than-expected frames in hand — waiting longer can't fill the gap
|
||||
// within the window. Play them in order, concealing what's genuinely missing.
|
||||
let (_, held) = self.held.take().expect("checked above");
|
||||
let (first, second) = if frame.seq.wrapping_sub(held.seq) > u32::MAX / 2 {
|
||||
(frame, held)
|
||||
} else {
|
||||
(held, frame)
|
||||
};
|
||||
self.give_up_and_play(first, out);
|
||||
self.accept(now, second, out); // depth ≤ 1: `held` is None here
|
||||
} else {
|
||||
self.held = Some((now, frame)); // wait ≤ HOLD_MAX for the predecessor
|
||||
}
|
||||
}
|
||||
|
||||
/// If the held frame's predecessor just played, the hold is over — the reorder repaired.
|
||||
fn release_held_in_order(&mut self, out: &mut Vec<Deliver>) {
|
||||
if let Some((_, h)) = &self.held {
|
||||
if Some(h.seq) == self.expected {
|
||||
let (_, h) = self.held.take().expect("checked above");
|
||||
self.expected = Some(h.seq.wrapping_add(1));
|
||||
self.reorders += 1;
|
||||
out.push(Deliver::Frame(h));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop waiting for `frame`'s predecessors: conceal the gap (capped) and play it.
|
||||
fn give_up_and_play(&mut self, frame: MicFrame, out: &mut Vec<Deliver>) {
|
||||
let gap = frame
|
||||
.seq
|
||||
.wrapping_sub(self.expected.expect("callers checked"));
|
||||
if gap > 0 {
|
||||
self.seq_gaps += 1;
|
||||
let conceal = gap.min(MAX_CONCEAL_FRAMES);
|
||||
self.concealed += u64::from(conceal);
|
||||
for _ in 0..conceal {
|
||||
out.push(Deliver::Conceal);
|
||||
}
|
||||
}
|
||||
self.expected = Some(frame.seq.wrapping_add(1));
|
||||
out.push(Deliver::Frame(frame));
|
||||
}
|
||||
|
||||
/// When the current hold (if any) must be given up — the pump's next wake deadline.
|
||||
pub(super) fn hold_deadline(&self) -> Option<Instant> {
|
||||
self.held.as_ref().map(|(t, _)| *t + HOLD_MAX)
|
||||
}
|
||||
|
||||
/// Give up an expired hold (the predecessor never came): conceal + play. Called on the
|
||||
/// pump's timeout wakes.
|
||||
pub(super) fn flush_expired_hold(&mut self, now: Instant, out: &mut Vec<Deliver>) {
|
||||
if self.hold_deadline().is_some_and(|d| now >= d) {
|
||||
let (_, h) = self.held.take().expect("deadline implies held");
|
||||
self.give_up_and_play(h, out);
|
||||
}
|
||||
}
|
||||
|
||||
/// Uplink pause / backend reopen: whatever is parked predates the gap, and the next frame
|
||||
/// restarts the chain (a pause is not loss — it must not conceal or count a gap).
|
||||
pub(super) fn reset_stream(&mut self) {
|
||||
self.expected = None;
|
||||
self.held = None;
|
||||
self.last_pts = None;
|
||||
}
|
||||
|
||||
// ---- adaptive target -------------------------------------------------------------------
|
||||
|
||||
fn record_arrival(&mut self, now: Instant, seq: u32, pts_ns: u64) {
|
||||
if let Some(last) = self.last_arrival {
|
||||
let gap = now.duration_since(last);
|
||||
if gap <= PAUSE_GAP {
|
||||
let ms = gap.as_secs_f32() * 1000.0;
|
||||
self.ewma_gap_ms = if self.ewma_gap_ms == 0.0 {
|
||||
ms
|
||||
} else {
|
||||
self.ewma_gap_ms * 0.9 + ms * 0.1
|
||||
};
|
||||
self.rotate_buckets(now);
|
||||
self.gap_max[self.bucket_idx] = self.gap_max[self.bucket_idx].max(ms);
|
||||
}
|
||||
}
|
||||
self.first_arrival.get_or_insert(now);
|
||||
self.last_arrival = Some(now);
|
||||
if let Some((last_seq, last_pts)) = self.last_pts {
|
||||
if seq == last_seq.wrapping_add(1) && pts_ns > last_pts {
|
||||
let d = (pts_ns - last_pts) as f32 / 1_000_000.0;
|
||||
if (1.0..=120.0).contains(&d) {
|
||||
self.frame_ms = if self.frame_ms == 0.0 {
|
||||
d
|
||||
} else {
|
||||
self.frame_ms * 0.9 + d * 0.1
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
self.last_pts = Some((seq, pts_ns));
|
||||
}
|
||||
|
||||
fn rotate_buckets(&mut self, now: Instant) {
|
||||
let Some(mut start) = self.bucket_start else {
|
||||
self.bucket_start = Some(now);
|
||||
return;
|
||||
};
|
||||
let mut advanced = 0;
|
||||
while now.duration_since(start) >= BUCKET_LEN {
|
||||
self.bucket_idx = (self.bucket_idx + 1) % BUCKETS;
|
||||
self.gap_max[self.bucket_idx] = 0.0;
|
||||
start += BUCKET_LEN;
|
||||
advanced += 1;
|
||||
if advanced >= BUCKETS {
|
||||
// Everything aged out (long pause) — snap the window to now.
|
||||
self.gap_max = [0.0; BUCKETS];
|
||||
start = now;
|
||||
break;
|
||||
}
|
||||
}
|
||||
self.bucket_start = Some(start);
|
||||
}
|
||||
|
||||
/// The jitter component of the backend ring's prime depth, in ms — the worst burst gap in
|
||||
/// the window plus a pad, clamped to [10, 60]. Backends add ONE consumer quantum on top
|
||||
/// (see [`VirtualMic::set_target_depth`](super::VirtualMic::set_target_depth)). Old bursty
|
||||
/// clients measure ~42 ms and land near the historical 48 ms prime; modern 10 ms-cadence
|
||||
/// clients settle at ~15–25 ms.
|
||||
pub(super) fn target_ms(&self, now: Instant) -> u32 {
|
||||
let measured =
|
||||
self.gap_max.iter().fold(0f32, |a, &b| a.max(b)).ceil() as u32 + TARGET_PAD_MS;
|
||||
let warm = self
|
||||
.first_arrival
|
||||
.is_some_and(|t| now.duration_since(t) >= WARMUP);
|
||||
let t = if warm {
|
||||
measured
|
||||
} else {
|
||||
measured.max(DEFAULT_TARGET_MS)
|
||||
};
|
||||
t.clamp(10, 60)
|
||||
}
|
||||
|
||||
/// Reset-on-read counters + gauges for the pump's telemetry line.
|
||||
pub(super) fn take_stats(&mut self) -> DejitterStats {
|
||||
let s = DejitterStats {
|
||||
seq_gaps: self.seq_gaps,
|
||||
concealed: self.concealed,
|
||||
reorders: self.reorders,
|
||||
late_drops: self.late_drops,
|
||||
cadence_ms: self.ewma_gap_ms,
|
||||
frame_ms: self.frame_ms,
|
||||
};
|
||||
self.seq_gaps = 0;
|
||||
self.concealed = 0;
|
||||
self.reorders = 0;
|
||||
self.late_drops = 0;
|
||||
s
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn f(seq: u32) -> MicFrame {
|
||||
MicFrame {
|
||||
seq,
|
||||
pts_ns: u64::from(seq) * 20_000_000,
|
||||
opus: vec![1],
|
||||
}
|
||||
}
|
||||
|
||||
/// Compact delivery shape: frame seqs as-is, concealment as -1.
|
||||
fn seqs(out: &[Deliver]) -> Vec<i64> {
|
||||
out.iter()
|
||||
.map(|d| match d {
|
||||
Deliver::Frame(fr) => i64::from(fr.seq),
|
||||
Deliver::Conceal => -1,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn in_order_passthrough() {
|
||||
let mut dj = MicDejitter::new();
|
||||
let t = Instant::now();
|
||||
let mut out = Vec::new();
|
||||
for s in 5..8 {
|
||||
dj.ingest(t, f(s), &mut out);
|
||||
}
|
||||
assert_eq!(seqs(&out), [5, 6, 7]);
|
||||
let st = dj.take_stats();
|
||||
assert_eq!(
|
||||
(st.seq_gaps, st.concealed, st.reorders, st.late_drops),
|
||||
(0, 0, 0, 0)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn swapped_pair_is_repaired_not_concealed() {
|
||||
let mut dj = MicDejitter::new();
|
||||
let t = Instant::now();
|
||||
let mut out = Vec::new();
|
||||
dj.ingest(t, f(5), &mut out);
|
||||
dj.ingest(t, f(7), &mut out); // held, waiting for 6
|
||||
assert_eq!(seqs(&out), [5]);
|
||||
dj.ingest(t, f(6), &mut out); // the missing one arrives late → both play, in order
|
||||
assert_eq!(seqs(&out), [5, 6, 7]);
|
||||
let st = dj.take_stats();
|
||||
assert_eq!(st.reorders, 1);
|
||||
assert_eq!(st.concealed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loss_is_concealed_when_the_window_moves_on() {
|
||||
let mut dj = MicDejitter::new();
|
||||
let t = Instant::now();
|
||||
let mut out = Vec::new();
|
||||
dj.ingest(t, f(5), &mut out);
|
||||
dj.ingest(t, f(7), &mut out); // held, waiting for 6
|
||||
dj.ingest(t, f(8), &mut out); // 6 is lost: one PLC frame, then 7 and 8
|
||||
assert_eq!(seqs(&out), [5, -1, 7, 8]);
|
||||
let st = dj.take_stats();
|
||||
assert_eq!(st.seq_gaps, 1);
|
||||
assert_eq!(st.concealed, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expired_hold_flushes_with_concealment() {
|
||||
let mut dj = MicDejitter::new();
|
||||
let t = Instant::now();
|
||||
let mut out = Vec::new();
|
||||
dj.ingest(t, f(5), &mut out);
|
||||
dj.ingest(t, f(7), &mut out);
|
||||
assert!(dj.hold_deadline().is_some());
|
||||
dj.flush_expired_hold(t + HOLD_MAX, &mut out);
|
||||
assert_eq!(seqs(&out), [5, -1, 7]);
|
||||
assert!(dj.hold_deadline().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn big_gap_conceal_is_capped() {
|
||||
let mut dj = MicDejitter::new();
|
||||
let t = Instant::now();
|
||||
let mut out = Vec::new();
|
||||
dj.ingest(t, f(5), &mut out);
|
||||
dj.ingest(t, f(100), &mut out); // held
|
||||
dj.flush_expired_hold(t + HOLD_MAX, &mut out);
|
||||
let conceals = out.iter().filter(|d| matches!(d, Deliver::Conceal)).count();
|
||||
assert_eq!(conceals, MAX_CONCEAL_FRAMES as usize);
|
||||
assert_eq!(dj.take_stats().seq_gaps, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicates_and_stale_late_frames_drop() {
|
||||
let mut dj = MicDejitter::new();
|
||||
let t = Instant::now();
|
||||
let mut out = Vec::new();
|
||||
dj.ingest(t, f(5), &mut out);
|
||||
dj.ingest(t, f(6), &mut out);
|
||||
dj.ingest(t, f(6), &mut out); // duplicate
|
||||
dj.ingest(t, f(3), &mut out); // ancient
|
||||
dj.ingest(t, f(8), &mut out); // held
|
||||
dj.ingest(t, f(8), &mut out); // duplicate of the held frame
|
||||
assert_eq!(seqs(&out), [5, 6]);
|
||||
assert_eq!(dj.take_stats().late_drops, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wraparound_is_a_reorder_not_an_eternity() {
|
||||
let mut dj = MicDejitter::new();
|
||||
let t = Instant::now();
|
||||
let mut out = Vec::new();
|
||||
dj.ingest(t, f(u32::MAX), &mut out);
|
||||
dj.ingest(t, f(0), &mut out); // in order across the wrap
|
||||
assert_eq!(seqs(&out), [i64::from(u32::MAX), 0]);
|
||||
dj.ingest(t, f(u32::MAX), &mut out); // pre-wrap replay: late, not a 2^31 gap
|
||||
assert_eq!(dj.take_stats().late_drops, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_restarts_the_chain_without_concealing() {
|
||||
let mut dj = MicDejitter::new();
|
||||
let t = Instant::now();
|
||||
let mut out = Vec::new();
|
||||
dj.ingest(t, f(5), &mut out);
|
||||
dj.reset_stream();
|
||||
dj.ingest(t, f(500), &mut out); // a pause is not loss
|
||||
assert_eq!(seqs(&out), [5, 500]);
|
||||
assert_eq!(dj.take_stats().concealed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bursty_uplink_grows_the_target_modern_uplink_shrinks_it() {
|
||||
// Old Mac cadence: two frames back-to-back every ~42 ms.
|
||||
let mut dj = MicDejitter::new();
|
||||
let t0 = Instant::now();
|
||||
let mut out = Vec::new();
|
||||
let mut now = t0;
|
||||
let mut seq = 0u32;
|
||||
for burst in 0u64..50 {
|
||||
now = t0 + Duration::from_millis(burst * 42);
|
||||
for _ in 0..2 {
|
||||
dj.ingest(now, f(seq), &mut out);
|
||||
seq += 1;
|
||||
}
|
||||
}
|
||||
let target = dj.target_ms(now);
|
||||
assert!((42..=60).contains(&target), "bursty target {target}");
|
||||
|
||||
// Modern 10 ms cadence: past warmup the target follows the small gaps down.
|
||||
let mut dj = MicDejitter::new();
|
||||
let mut now = t0;
|
||||
for i in 0u64..200 {
|
||||
now = t0 + Duration::from_millis(i * 10);
|
||||
dj.ingest(now, f(i as u32), &mut out);
|
||||
}
|
||||
let target = dj.target_ms(now);
|
||||
assert!((10..=20).contains(&target), "steady target {target}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warmup_holds_the_crackle_safe_default() {
|
||||
let mut dj = MicDejitter::new();
|
||||
let t0 = Instant::now();
|
||||
let mut out = Vec::new();
|
||||
dj.ingest(t0, f(0), &mut out);
|
||||
dj.ingest(t0 + Duration::from_millis(10), f(1), &mut out);
|
||||
// 10 ms of evidence must not shrink the target yet — the client might be bursty.
|
||||
assert_eq!(
|
||||
dj.target_ms(t0 + Duration::from_millis(20)),
|
||||
DEFAULT_TARGET_MS
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -5,15 +5,49 @@
|
||||
//! trait, its factory ([`open_virtual_mic`](super::open_virtual_mic)) and the audio-plane sample
|
||||
//! rate stay in `super`.
|
||||
|
||||
use super::mic_jitter::{Deliver, MicDejitter};
|
||||
use super::{VirtualMic, SAMPLE_RATE};
|
||||
use anyhow::Result;
|
||||
|
||||
/// Mic is 48 kHz stereo — matches the Opus stereo decoder and the host→client audio layout.
|
||||
pub const MIC_CHANNELS: u32 = 2;
|
||||
/// Bound for the shared mic frame queue (drop-newest when full): the host-lifetime queue is
|
||||
/// shared across all concurrent sessions and must not grow without limit under a near-line-rate
|
||||
/// flood (security-review 2026-06-28 S6). 64 × 5–20 ms frames ≈ 0.3–1.3 s of slack.
|
||||
const MIC_QUEUE_CAP: usize = 64;
|
||||
|
||||
/// One client mic frame off the wire (`0xCB` — `punktfunk_core::quic::decode_mic_datagram`).
|
||||
/// The datagram's seq + pts used to be decoded at ingest and thrown away; the pump's de-jitter
|
||||
/// needs them (reorder window, loss concealment, cadence/drift), so they ride the queue with
|
||||
/// the Opus payload.
|
||||
pub struct MicFrame {
|
||||
pub seq: u32,
|
||||
pub pts_ns: u64,
|
||||
pub opus: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Bound for the shared mic frame queue (drop-newest at the producer when full): the
|
||||
/// host-lifetime queue is shared across all concurrent sessions and must not grow without limit
|
||||
/// under a near-line-rate flood (security-review 2026-06-28 S6). 12 × 5–20 ms frames ≈
|
||||
/// 60–240 ms of slack — enough for a scheduling hiccup, and the consumer heals the rest (see
|
||||
/// [`DRAIN_ABOVE`]). The old cap of 64 could park 1.28 s of standing latency here that only a
|
||||
/// >600 ms silence gap ever flushed.
|
||||
const MIC_QUEUE_CAP: usize = 12;
|
||||
/// Consumer self-healing: a pump that wakes to a backlog deeper than this drops down to the
|
||||
/// newest [`DRAIN_KEEP`] frames before decoding — the oldest frames are already late, and
|
||||
/// playing them would turn a one-off stall into permanent mic delay.
|
||||
const DRAIN_ABOVE: usize = 6;
|
||||
const DRAIN_KEEP: usize = 4;
|
||||
|
||||
/// Cadence of the "mic uplink health" telemetry line (emitted only while frames flow).
|
||||
const TELEMETRY_EVERY: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
|
||||
/// Creep trim: the backend ring must sit more than this over its prime target, for
|
||||
/// [`TRIM_AFTER`], before the pump starts shedding — and then only near-silent frames, at most
|
||||
/// one per [`TRIM_SPACING`] (a 20 ms frame per 300 ms ≈ 7 ms per 100 ms). Depth built by a
|
||||
/// burst otherwise only ever comes back down at a full drain (the device consumes exactly what
|
||||
/// it plays).
|
||||
const TRIM_MARGIN_MS: usize = 15;
|
||||
const TRIM_AFTER: std::time::Duration = std::time::Duration::from_secs(2);
|
||||
const TRIM_SPACING: std::time::Duration = std::time::Duration::from_millis(300);
|
||||
/// Peak below which a decoded frame counts as a silence stretch (≈ −48 dBFS).
|
||||
const TRIM_SILENCE_PEAK: f32 = 0.004;
|
||||
|
||||
/// Tuning for [`MicPump`]'s open/reopen/flush behaviour — parameterized so the tests can run the
|
||||
/// real pump loop in milliseconds instead of seconds.
|
||||
@@ -56,20 +90,24 @@ const PUMP_TUNING: PumpTuning = PumpTuning {
|
||||
/// every push and on an idle heartbeat, and reopened with backoff. Sessions keep their
|
||||
/// senders; nothing upstream notices.
|
||||
/// - **Stale-flush**: buffered audio is discarded after an uplink gap (see [`PumpTuning`]).
|
||||
/// - **De-jittered**: frames pass a sequence-aware reorder window + loss concealment, and an
|
||||
/// adaptive target-depth estimator drives the backend rings' priming — see
|
||||
/// [`MicDejitter`](super::mic_jitter) (`PUNKTFUNK_MIC_LEGACY_BUFFER=1` restores the fixed
|
||||
/// pre-adaptive buffering).
|
||||
///
|
||||
/// Per-frame Opus DECODE errors stay non-fatal (dropped frame): the mic is shared across every
|
||||
/// concurrent session, so one paired client's junk frames must not deny everyone's mic
|
||||
/// (security-review 2026-06-28 S2). The thread exits when every sender is dropped (host
|
||||
/// shutdown), tearing the backend down.
|
||||
pub struct MicPump {
|
||||
tx: std::sync::mpsc::SyncSender<Vec<u8>>,
|
||||
tx: std::sync::mpsc::SyncSender<MicFrame>,
|
||||
}
|
||||
|
||||
impl MicPump {
|
||||
/// Start the host-lifetime pump (Linux/Windows). On platforms without a virtual-mic backend
|
||||
/// the thread just drains and drops frames (sessions still count the datagrams).
|
||||
pub fn start() -> MicPump {
|
||||
let (tx, rx) = std::sync::mpsc::sync_channel::<Vec<u8>>(MIC_QUEUE_CAP);
|
||||
let (tx, rx) = std::sync::mpsc::sync_channel::<MicFrame>(MIC_QUEUE_CAP);
|
||||
let spawned = std::thread::Builder::new()
|
||||
.name("punktfunk-mic-pump".into())
|
||||
.spawn(move || {
|
||||
@@ -90,7 +128,7 @@ impl MicPump {
|
||||
/// A sender a session forwards the client's Opus mic frames to (`try_send` — never block a
|
||||
/// datagram loop). Cloned per session; dropping a clone does NOT stop the pump (it holds
|
||||
/// the original sender for the host life).
|
||||
pub fn sender(&self) -> std::sync::mpsc::SyncSender<Vec<u8>> {
|
||||
pub fn sender(&self) -> std::sync::mpsc::SyncSender<MicFrame> {
|
||||
self.tx.clone()
|
||||
}
|
||||
}
|
||||
@@ -99,7 +137,7 @@ impl MicPump {
|
||||
/// never accumulates a stale backlog and senders never see a wedged queue. Returns `false` when
|
||||
/// every sender is gone (host shutdown).
|
||||
#[cfg_attr(not(any(target_os = "linux", target_os = "windows")), allow(dead_code))]
|
||||
fn drain_sleep(rx: &std::sync::mpsc::Receiver<Vec<u8>>, dur: std::time::Duration) -> bool {
|
||||
fn drain_sleep(rx: &std::sync::mpsc::Receiver<MicFrame>, dur: std::time::Duration) -> bool {
|
||||
use std::sync::mpsc::RecvTimeoutError;
|
||||
let deadline = std::time::Instant::now() + dur;
|
||||
loop {
|
||||
@@ -118,7 +156,7 @@ fn drain_sleep(rx: &std::sync::mpsc::Receiver<Vec<u8>>, dur: std::time::Duration
|
||||
/// The pump loop. `opener` is injected so the tests can run the REAL loop against a mock
|
||||
/// backend; production passes [`open_virtual_mic`](super::open_virtual_mic).
|
||||
#[cfg_attr(not(any(target_os = "linux", target_os = "windows")), allow(dead_code))]
|
||||
fn pump_thread<O>(rx: std::sync::mpsc::Receiver<Vec<u8>>, opener: O, tuning: PumpTuning)
|
||||
fn pump_thread<O>(rx: std::sync::mpsc::Receiver<MicFrame>, opener: O, tuning: PumpTuning)
|
||||
where
|
||||
O: Fn() -> Result<Box<dyn VirtualMic>>,
|
||||
{
|
||||
@@ -159,40 +197,65 @@ where
|
||||
let opened_at = Instant::now();
|
||||
|
||||
// Pump phase — runs until the backend dies (break) or the host shuts down (return).
|
||||
let legacy = super::mic_legacy_buffer();
|
||||
let mut decode_fails: u64 = 0;
|
||||
let mut drain_drops: u64 = 0;
|
||||
let mut trimmed: u64 = 0;
|
||||
let mut frames_seen: u64 = 0;
|
||||
let mut pcm = vec![0f32; 5760 * MIC_CHANNELS as usize]; // up to 120 ms scratch
|
||||
let mut last_push = Instant::now();
|
||||
loop {
|
||||
match rx.recv_timeout(tuning.heartbeat) {
|
||||
Ok(frame) => {
|
||||
if frame.is_empty() {
|
||||
continue; // DTX silence — the source underruns to silence on its own
|
||||
let mut batch: Vec<MicFrame> = Vec::new();
|
||||
let mut deliveries: Vec<Deliver> = Vec::new();
|
||||
let mut jitter = MicDejitter::new();
|
||||
// Concealment must match the last real frame's duration — libopus sizes PLC from the
|
||||
// output slice. One 20 ms frame until something decodes.
|
||||
let mut plc_samples: usize = 960;
|
||||
let mut applied_target_ms: u32 = 0;
|
||||
let mut over_since: Option<Instant> = None;
|
||||
let mut last_trim = Instant::now();
|
||||
let mut last_log = Instant::now();
|
||||
'pump: loop {
|
||||
// Wake for the next frame, the heartbeat, or a parked reorder hold aging out —
|
||||
// whichever is soonest.
|
||||
let timeout = jitter
|
||||
.hold_deadline()
|
||||
.map(|d| d.saturating_duration_since(Instant::now()))
|
||||
.unwrap_or(tuning.heartbeat)
|
||||
.min(tuning.heartbeat);
|
||||
deliveries.clear();
|
||||
match rx.recv_timeout(timeout) {
|
||||
Ok(first) => {
|
||||
// Take everything already queued in one gulp: normally that's just `first`,
|
||||
// but after a stall the backlog IS standing mic latency — heal by jumping
|
||||
// to the newest few frames instead of replaying the pile.
|
||||
batch.clear();
|
||||
batch.push(first);
|
||||
while batch.len() <= MIC_QUEUE_CAP {
|
||||
match rx.try_recv() {
|
||||
Ok(f) => batch.push(f),
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
if batch.len() > DRAIN_ABOVE {
|
||||
let drop_n = batch.len() - DRAIN_KEEP;
|
||||
drain_drops += drop_n as u64;
|
||||
batch.drain(..drop_n);
|
||||
}
|
||||
frames_seen += batch.len() as u64;
|
||||
if last_push.elapsed() > tuning.stale_gap {
|
||||
mic.discard();
|
||||
jitter.reset_stream();
|
||||
}
|
||||
match decoder.decode_float(&frame, &mut pcm, false) {
|
||||
Ok(samples_per_ch) => {
|
||||
let total = (samples_per_ch * MIC_CHANNELS as usize).min(pcm.len());
|
||||
if !mic.push(&pcm[..total]) {
|
||||
tracing::warn!("virtual mic backend died — reopening");
|
||||
break;
|
||||
}
|
||||
last_push = Instant::now();
|
||||
decode_fails = 0;
|
||||
}
|
||||
Err(e) => {
|
||||
// Malformed/garbage frame: drop it, keep the shared mic + decoder
|
||||
// (see the struct docs). Throttled log (1, 2, 4, … fails).
|
||||
decode_fails += 1;
|
||||
if decode_fails.is_power_of_two() {
|
||||
tracing::warn!(error = %e, fails = decode_fails,
|
||||
"mic opus decode failed — dropping frame");
|
||||
}
|
||||
}
|
||||
let now = Instant::now();
|
||||
for frame in batch.drain(..) {
|
||||
jitter.ingest(now, frame, &mut deliveries);
|
||||
}
|
||||
// A hold whose window expired while traffic kept flowing (only late
|
||||
// duplicates arriving) must still flush — the timeout arm never fires then.
|
||||
jitter.flush_expired_hold(now, &mut deliveries);
|
||||
}
|
||||
Err(RecvTimeoutError::Timeout) => {
|
||||
jitter.flush_expired_hold(Instant::now(), &mut deliveries);
|
||||
if !mic.alive() {
|
||||
tracing::warn!("virtual mic backend died while idle — reopening");
|
||||
break;
|
||||
@@ -203,6 +266,123 @@ where
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
for d in deliveries.drain(..) {
|
||||
let samples_per_ch = match d {
|
||||
Deliver::Frame(frame) => {
|
||||
if frame.opus.is_empty() {
|
||||
continue; // DTX silence — the source underruns to silence on its own
|
||||
}
|
||||
match decoder.decode_float(&frame.opus, &mut pcm, false) {
|
||||
Ok(n) => {
|
||||
plc_samples = n.max(120); // ≥ 2.5 ms keeps PLC well-formed
|
||||
decode_fails = 0;
|
||||
n
|
||||
}
|
||||
Err(e) => {
|
||||
// Malformed/garbage frame: drop it, keep the shared mic +
|
||||
// decoder (see the struct docs). Throttled log (1, 2, 4, …).
|
||||
decode_fails += 1;
|
||||
if decode_fails.is_power_of_two() {
|
||||
tracing::warn!(error = %e, fails = decode_fails,
|
||||
"mic opus decode failed — dropping frame");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
Deliver::Conceal => {
|
||||
// A lost datagram: an empty-input decode synthesizes libopus PLC for
|
||||
// exactly the slice's duration, so the backend ring doesn't starve
|
||||
// into a silence + re-prime cycle over one missing frame.
|
||||
let want = (plc_samples * MIC_CHANNELS as usize).min(pcm.len());
|
||||
match decoder.decode_float(&[], &mut pcm[..want], false) {
|
||||
Ok(n) => n,
|
||||
Err(_) => continue, // nothing decoded yet — nothing to extend
|
||||
}
|
||||
}
|
||||
};
|
||||
let total = (samples_per_ch * MIC_CHANNELS as usize).min(pcm.len());
|
||||
// Creep trim: ring depth persistently over target sheds a near-silent frame at
|
||||
// a time (a few ms per 100 ms) — never speech, never a hard clear. Without it,
|
||||
// depth built by a burst only ever comes back down at a full drain.
|
||||
let mut shed = false;
|
||||
if !legacy {
|
||||
match mic.depth() {
|
||||
Some((buffered, target))
|
||||
if buffered > target + TRIM_MARGIN_MS * SAMPLE_RATE as usize / 1000 =>
|
||||
{
|
||||
let since = *over_since.get_or_insert_with(Instant::now);
|
||||
if since.elapsed() >= TRIM_AFTER
|
||||
&& last_trim.elapsed() >= TRIM_SPACING
|
||||
&& pcm[..total].iter().all(|s| s.abs() < TRIM_SILENCE_PEAK)
|
||||
{
|
||||
shed = true;
|
||||
last_trim = Instant::now();
|
||||
trimmed += 1;
|
||||
}
|
||||
}
|
||||
_ => over_since = None,
|
||||
}
|
||||
}
|
||||
if shed {
|
||||
last_push = Instant::now(); // the uplink is live — a trim is not a stale gap
|
||||
continue;
|
||||
}
|
||||
if !mic.push(&pcm[..total]) {
|
||||
tracing::warn!("virtual mic backend died — reopening");
|
||||
break 'pump;
|
||||
}
|
||||
last_push = Instant::now();
|
||||
}
|
||||
|
||||
// Drive the backend ring's prime target from the measured jitter. Legacy mode
|
||||
// never calls this, which keeps the backend on its fixed constants (see
|
||||
// `VirtualMic::set_target_depth`).
|
||||
if !legacy {
|
||||
let t = jitter.target_ms(Instant::now());
|
||||
if t != applied_target_ms {
|
||||
applied_target_ms = t;
|
||||
mic.set_target_depth(t as usize * SAMPLE_RATE as usize / 1000);
|
||||
}
|
||||
}
|
||||
|
||||
// One structured health line per interval while the mic is live (reset-on-read).
|
||||
if last_log.elapsed() >= TELEMETRY_EVERY {
|
||||
if frames_seen > 0 {
|
||||
let js = jitter.take_stats();
|
||||
let bs = mic.take_stats();
|
||||
let (depth_ms, target_ms) = mic
|
||||
.depth()
|
||||
.map(|(d, t)| {
|
||||
(
|
||||
d * 1000 / SAMPLE_RATE as usize,
|
||||
t * 1000 / SAMPLE_RATE as usize,
|
||||
)
|
||||
})
|
||||
.unwrap_or((0, 0));
|
||||
tracing::info!(
|
||||
depth_ms,
|
||||
target_ms,
|
||||
cadence_ms = (js.cadence_ms * 10.0).round() / 10.0,
|
||||
frame_ms = (js.frame_ms * 10.0).round() / 10.0,
|
||||
frames = frames_seen,
|
||||
gaps = js.seq_gaps,
|
||||
concealed = js.concealed,
|
||||
reorders = js.reorders,
|
||||
late = js.late_drops,
|
||||
drained = drain_drops,
|
||||
trimmed,
|
||||
reprimes = bs.reprimes,
|
||||
overflow_ms = bs.overflow_dropped * 1000 / SAMPLE_RATE as u64,
|
||||
"mic uplink health"
|
||||
);
|
||||
}
|
||||
last_log = Instant::now();
|
||||
frames_seen = 0;
|
||||
drain_drops = 0;
|
||||
trimmed = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Death triage: an instance that lived is a one-off (PipeWire/audio-engine restart) —
|
||||
@@ -252,7 +432,7 @@ mod pump_tests {
|
||||
}
|
||||
|
||||
struct Harness {
|
||||
tx: std::sync::mpsc::SyncSender<Vec<u8>>,
|
||||
tx: std::sync::mpsc::SyncSender<MicFrame>,
|
||||
opens: Arc<AtomicUsize>,
|
||||
alive: Arc<Mutex<Option<Arc<AtomicBool>>>>, // latest instance's kill switch
|
||||
pushed: Arc<AtomicUsize>,
|
||||
@@ -265,7 +445,7 @@ mod pump_tests {
|
||||
/// pre-killed (exercises the rapid-death churn guard). `stable_after` mirrors the tuning
|
||||
/// field (ZERO = every death counts as stable → immediate reopen, keeping tests fast).
|
||||
fn start_tuned(fail_first: usize, dead_on_arrival: bool, stable_after: Duration) -> Harness {
|
||||
let (tx, rx) = std::sync::mpsc::sync_channel::<Vec<u8>>(MIC_QUEUE_CAP);
|
||||
let (tx, rx) = std::sync::mpsc::sync_channel::<MicFrame>(MIC_QUEUE_CAP);
|
||||
let opens = Arc::new(AtomicUsize::new(0));
|
||||
let alive = Arc::new(Mutex::new(None::<Arc<AtomicBool>>));
|
||||
let pushed = Arc::new(AtomicUsize::new(0));
|
||||
@@ -317,7 +497,7 @@ mod pump_tests {
|
||||
}
|
||||
|
||||
fn wait_until(what: &str, mut cond: impl FnMut() -> bool) {
|
||||
for _ in 0..200 {
|
||||
for _ in 0..600 {
|
||||
if cond() {
|
||||
return;
|
||||
}
|
||||
@@ -326,6 +506,27 @@ mod pump_tests {
|
||||
panic!("timed out waiting for: {what}");
|
||||
}
|
||||
|
||||
/// Keep a live uplink running until audio reaches the backend.
|
||||
///
|
||||
/// A single frame is NOT enough to prove a reopen recovered: a freshly opened backend
|
||||
/// deliberately drops whatever queued while it was down (the `try_recv` drain after the
|
||||
/// open — that audio predates the device), and `opens` counts from the START of the open,
|
||||
/// so a frame sent the moment the counter moves lands inside that drain and is discarded
|
||||
/// by design. A real uplink keeps sending, so the test does too. The sequence has to keep
|
||||
/// advancing or the de-jitter reads the repeats as duplicates and drops them.
|
||||
fn wait_until_pushed(what: &str, h: &Harness, from_seq: u32) {
|
||||
let mut seq = from_seq;
|
||||
for _ in 0..600 {
|
||||
let _ = h.tx.try_send(mic_frame(seq));
|
||||
seq = seq.wrapping_add(1);
|
||||
if h.pushed.load(Ordering::SeqCst) > 0 {
|
||||
return;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
panic!("timed out waiting for: {what}");
|
||||
}
|
||||
|
||||
fn opus_frame() -> Vec<u8> {
|
||||
let mut enc = opus::Encoder::new(48_000, opus::Channels::Stereo, opus::Application::Voip)
|
||||
.expect("opus encoder");
|
||||
@@ -336,6 +537,15 @@ mod pump_tests {
|
||||
out
|
||||
}
|
||||
|
||||
/// A wire-shaped frame: `seq` with the matching 20 ms pts, payload from [`opus_frame`].
|
||||
fn mic_frame(seq: u32) -> MicFrame {
|
||||
MicFrame {
|
||||
seq,
|
||||
pts_ns: seq as u64 * 20_000_000,
|
||||
opus: opus_frame(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Eager: the backend opens (after transient failures) with NO frame ever sent.
|
||||
#[test]
|
||||
fn opens_eagerly_with_backoff() {
|
||||
@@ -352,7 +562,7 @@ mod pump_tests {
|
||||
fn decodes_and_pushes() {
|
||||
let h = start(0);
|
||||
wait_until("open", || h.alive.lock().unwrap().is_some());
|
||||
h.tx.send(opus_frame()).unwrap();
|
||||
h.tx.send(mic_frame(0)).unwrap();
|
||||
wait_until("pcm pushed", || h.pushed.load(Ordering::SeqCst) > 0);
|
||||
drop(h.tx);
|
||||
h.join.join().unwrap();
|
||||
@@ -388,10 +598,9 @@ mod pump_tests {
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.store(false, Ordering::Release);
|
||||
h.tx.send(opus_frame()).unwrap(); // push sees death → reopen
|
||||
h.tx.send(mic_frame(0)).unwrap(); // push sees death → reopen
|
||||
wait_until("reopen", || h.opens.load(Ordering::SeqCst) >= 2);
|
||||
h.tx.send(opus_frame()).unwrap();
|
||||
wait_until("pcm after reopen", || h.pushed.load(Ordering::SeqCst) > 0);
|
||||
wait_until_pushed("pcm after reopen", &h, 1);
|
||||
drop(h.tx);
|
||||
h.join.join().unwrap();
|
||||
}
|
||||
@@ -416,15 +625,32 @@ mod pump_tests {
|
||||
h.join.join().unwrap();
|
||||
}
|
||||
|
||||
/// A lost datagram is concealed: seq 0 then 2 (1 missing) must still push ~3 frames of PCM
|
||||
/// (decode, PLC, decode) once the reorder window expires — the backend ring never starves
|
||||
/// over a single missing frame.
|
||||
#[test]
|
||||
fn seq_gap_is_concealed() {
|
||||
let h = start(0);
|
||||
wait_until("instance", || h.alive.lock().unwrap().is_some());
|
||||
h.tx.send(mic_frame(0)).unwrap();
|
||||
h.tx.send(mic_frame(2)).unwrap();
|
||||
// 0 plays immediately; 2 is held ≤ 30 ms for 1, then the gap is concealed and 2 plays.
|
||||
wait_until("conceal + late frame pushed", || {
|
||||
h.pushed.load(Ordering::SeqCst) >= 3 * 960 * 2
|
||||
});
|
||||
drop(h.tx);
|
||||
h.join.join().unwrap();
|
||||
}
|
||||
|
||||
/// An uplink gap discards buffered-stale audio before the next frame plays.
|
||||
#[test]
|
||||
fn discards_after_gap() {
|
||||
let h = start(0);
|
||||
wait_until("instance", || h.alive.lock().unwrap().is_some());
|
||||
h.tx.send(opus_frame()).unwrap();
|
||||
h.tx.send(mic_frame(0)).unwrap();
|
||||
wait_until("first push", || h.pushed.load(Ordering::SeqCst) > 0);
|
||||
std::thread::sleep(Duration::from_millis(150)); // > stale_gap
|
||||
h.tx.send(opus_frame()).unwrap();
|
||||
h.tx.send(mic_frame(1)).unwrap();
|
||||
wait_until("discard on gap", || h.discards.load(Ordering::SeqCst) >= 1);
|
||||
drop(h.tx);
|
||||
h.join.join().unwrap();
|
||||
|
||||
@@ -19,19 +19,21 @@
|
||||
//! returns `false` and the pump reopens (re-planning, so endpoint churn re-resolves). Before this
|
||||
//! existed, the first device change silently killed mic passthrough for the rest of the host's life.
|
||||
//!
|
||||
//! `push` enqueues decoded interleaved-f32 PCM into a bounded ring (drop-oldest beyond ~120 ms so
|
||||
//! mic latency stays bounded); a dedicated COM-apartment thread renders it event-driven through an
|
||||
//! adaptive jitter buffer (prime → hold → re-prime, see the render loop — clients arrive in bursts,
|
||||
//! the device pulls per-period), filling silence when the client isn't talking. WASAPI objects are
|
||||
//! `!Send`, so they live entirely on that thread (mirrors `WasapiLoopbackCapturer`).
|
||||
//! `push` enqueues decoded interleaved-f32 PCM into a bounded ring (drop-oldest so mic latency
|
||||
//! stays bounded — the bound follows the adaptive prime threshold, or the legacy ~120 ms until
|
||||
//! the pump drives it); a dedicated COM-apartment thread renders it event-driven through a
|
||||
//! jitter buffer (prime → hold → re-prime, see the render loop — clients arrive in bursts, the
|
||||
//! device pulls per-period) whose prime depth the mic pump sets from measured uplink jitter
|
||||
//! ([`VirtualMic::set_target_depth`]), filling silence when the client isn't talking. WASAPI
|
||||
//! objects are `!Send`, so they live entirely on that thread (mirrors `WasapiLoopbackCapturer`).
|
||||
|
||||
// Every `unsafe` block in this file carries a `// SAFETY:` proof; enforce it.
|
||||
#![deny(clippy::undocumented_unsafe_blocks)]
|
||||
|
||||
use super::{audio_control, VirtualMic, SAMPLE_RATE};
|
||||
use super::{audio_control, MicBackendStats, VirtualMic, SAMPLE_RATE};
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::mpsc::{sync_channel, SyncSender};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread::{self, JoinHandle};
|
||||
@@ -41,26 +43,53 @@ use wasapi::{Direction, SampleType, StreamMode, WaveFormat};
|
||||
const CHANNELS: u32 = 2;
|
||||
/// 48 kHz stereo f32: 2 channels * 4 bytes.
|
||||
const BLOCK_ALIGN: usize = 2 * 4;
|
||||
/// Jitter-buffer priming depth (~48 ms): the render loop emits pure silence until this much PCM
|
||||
/// is queued, then plays from the cushion. Clients deliver mic audio in BURSTS (the Mac client's
|
||||
/// input tap yields ~two 20 ms Opus packets every ~42 ms) while WASAPI pulls a small block every
|
||||
/// device period (~10 ms) — with no cushion the queue sits near-empty and most periods insert
|
||||
/// mid-stream silence: the "crackling mic" (heard live, Mac → Windows host 2026-07-03; the Linux
|
||||
/// backend's process callback primes the same way and the identical stream was clean there). The
|
||||
/// depth must cover the worst inter-burst gap (~42 ms), so ~48 ms with re-prime on a full drain.
|
||||
/// LEGACY jitter-buffer priming depth (~48 ms): the render loop emits pure silence until this
|
||||
/// much PCM is queued, then plays from the cushion. Old clients deliver mic audio in BURSTS
|
||||
/// (the Mac client's input tap yields ~two 20 ms Opus packets every ~42 ms) while WASAPI pulls
|
||||
/// a small block every device period (~10 ms) — with no cushion the queue sits near-empty and
|
||||
/// most periods insert mid-stream silence: the "crackling mic" (heard live, Mac → Windows host
|
||||
/// 2026-07-03; the Linux backend's process callback primes the same way and the identical
|
||||
/// stream was clean there). The depth had to cover the worst inter-burst gap (~42 ms), so
|
||||
/// ~48 ms with re-prime on a full drain. Today the pump MEASURES that gap and drives the prime
|
||||
/// threshold per client ([`VirtualMic::set_target_depth`] — bursty clients still get ~48 ms,
|
||||
/// modern 10 ms-cadence ones ~25–35 ms); this constant remains the fallback until the pump's
|
||||
/// first estimate, and forever under `PUNKTFUNK_MIC_LEGACY_BUFFER=1`.
|
||||
const PRIME_BYTES: usize = (SAMPLE_RATE as usize * 48 / 1000) * BLOCK_ALIGN;
|
||||
/// Bound the inject queue at ~120 ms so the passed-through mic stays low-latency (drop oldest
|
||||
/// beyond): the priming cushion (~48 ms) plus arrival-burst headroom.
|
||||
/// LEGACY bound for the inject queue at ~120 ms (drop oldest beyond): the fixed priming
|
||||
/// cushion plus arrival-burst headroom. Applies only while the pump isn't driving the target;
|
||||
/// adaptive mode bounds the queue at prime + [`CAP_HEADROOM_BYTES`] instead (≈ 50–105 ms).
|
||||
const MAX_QUEUE_BYTES: usize = (SAMPLE_RATE as usize * 120 / 1000) * BLOCK_ALIGN;
|
||||
/// Producer-side overflow headroom (~32 ms) over the render loop's prime threshold when the
|
||||
/// adaptive target drives the ring.
|
||||
const CAP_HEADROOM_BYTES: usize = (SAMPLE_RATE as usize * 32 / 1000) * BLOCK_ALIGN;
|
||||
|
||||
pub struct WasapiVirtualMic {
|
||||
queue: Arc<Mutex<VecDeque<u8>>>,
|
||||
stop: Arc<AtomicBool>,
|
||||
/// False once the render thread has exited (device error or stop) — the pump's reopen signal.
|
||||
alive: Arc<AtomicBool>,
|
||||
/// Ring policy/telemetry shared with the render thread (see [`RingShared`]).
|
||||
ring: Arc<RingShared>,
|
||||
join: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
/// Atomics shared between the pump-facing handle and the render thread: the pump's adaptive
|
||||
/// de-jitter target in, the effective prime threshold + reset-on-read counters out. All
|
||||
/// `Relaxed` — a slowly-moving target and telemetry, not synchronization.
|
||||
#[derive(Default)]
|
||||
struct RingShared {
|
||||
/// Pump-set jitter target in bytes. `0` = the pump never spoke (legacy mode, or its first
|
||||
/// estimate hasn't landed) → the render loop keeps the fixed [`PRIME_BYTES`] and `push`
|
||||
/// keeps the fixed [`MAX_QUEUE_BYTES`].
|
||||
target_bytes: AtomicUsize,
|
||||
/// Effective prime threshold (bytes) of the last render iteration.
|
||||
prime_bytes: AtomicUsize,
|
||||
/// Full-drain re-prime arms (see [`MicBackendStats`]).
|
||||
reprimes: AtomicU64,
|
||||
/// Per-channel samples dropped by the overflow cap.
|
||||
overflow: AtomicU64,
|
||||
}
|
||||
|
||||
impl WasapiVirtualMic {
|
||||
pub fn open(channels: u32) -> Result<Self> {
|
||||
anyhow::ensure!(
|
||||
@@ -70,14 +99,15 @@ impl WasapiVirtualMic {
|
||||
let queue = Arc::new(Mutex::new(VecDeque::<u8>::new()));
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let alive = Arc::new(AtomicBool::new(true));
|
||||
let ring = Arc::new(RingShared::default());
|
||||
// Bring-up handshake: report the resolved device (or the error) before returning, so a missing
|
||||
// virtual-mic device surfaces as Err (the caller retries with backoff) not a silent dead thread.
|
||||
let (ready_tx, ready_rx) = sync_channel::<Result<String>>(1);
|
||||
let (q, st, al) = (queue.clone(), stop.clone(), alive.clone());
|
||||
let (q, st, rg, al) = (queue.clone(), stop.clone(), ring.clone(), alive.clone());
|
||||
let join = thread::Builder::new()
|
||||
.name("punktfunk-wasapi-mic".into())
|
||||
.spawn(move || {
|
||||
if let Err(e) = render_thread(q, st, ready_tx) {
|
||||
if let Err(e) = render_thread(q, st, rg, ready_tx) {
|
||||
tracing::error!(error = %format!("{e:#}"), "wasapi virtual-mic thread failed");
|
||||
}
|
||||
// Normal stop or device error alike: this instance is done — the pump reopens.
|
||||
@@ -92,6 +122,7 @@ impl WasapiVirtualMic {
|
||||
queue,
|
||||
stop,
|
||||
alive,
|
||||
ring,
|
||||
join: Some(join),
|
||||
})
|
||||
}
|
||||
@@ -122,10 +153,25 @@ impl VirtualMic for WasapiVirtualMic {
|
||||
for &s in pcm {
|
||||
q.extend(s.to_le_bytes());
|
||||
}
|
||||
// Drop-oldest to keep latency bounded (mic is real-time; stale audio is worse than dropped).
|
||||
if q.len() > MAX_QUEUE_BYTES {
|
||||
let excess = q.len() - MAX_QUEUE_BYTES;
|
||||
// Drop-oldest to keep latency bounded (mic is real-time; stale audio is worse than
|
||||
// dropped). With the pump driving the target, the bound follows the render loop's prime
|
||||
// threshold + headroom; otherwise (legacy / no estimate yet) the fixed 120 ms applies.
|
||||
let cap = if self.ring.target_bytes.load(Ordering::Relaxed) == 0 {
|
||||
MAX_QUEUE_BYTES
|
||||
} else {
|
||||
// `max(PRIME_BYTES)` covers the one render period before the loop first publishes.
|
||||
self.ring
|
||||
.prime_bytes
|
||||
.load(Ordering::Relaxed)
|
||||
.max(PRIME_BYTES)
|
||||
+ CAP_HEADROOM_BYTES
|
||||
};
|
||||
if q.len() > cap {
|
||||
let excess = q.len() - cap;
|
||||
q.drain(..excess);
|
||||
self.ring
|
||||
.overflow
|
||||
.fetch_add((excess / BLOCK_ALIGN) as u64, Ordering::Relaxed);
|
||||
}
|
||||
true
|
||||
}
|
||||
@@ -143,6 +189,28 @@ impl VirtualMic for WasapiVirtualMic {
|
||||
fn channels(&self) -> u32 {
|
||||
CHANNELS
|
||||
}
|
||||
|
||||
fn set_target_depth(&self, samples_per_ch: usize) {
|
||||
self.ring
|
||||
.target_bytes
|
||||
.store(samples_per_ch * BLOCK_ALIGN, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn depth(&self) -> Option<(usize, usize)> {
|
||||
let prime = self.ring.prime_bytes.load(Ordering::Relaxed);
|
||||
if prime == 0 {
|
||||
return None; // render loop hasn't run yet
|
||||
}
|
||||
let q = self.queue.lock().ok()?;
|
||||
Some((q.len() / BLOCK_ALIGN, prime / BLOCK_ALIGN))
|
||||
}
|
||||
|
||||
fn take_stats(&self) -> MicBackendStats {
|
||||
MicBackendStats {
|
||||
reprimes: self.ring.reprimes.swap(0, Ordering::Relaxed),
|
||||
overflow_dropped: self.ring.overflow.swap(0, Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the mic inject target from the wiring plan, auto-installing the Steam Streaming pair
|
||||
@@ -273,6 +341,7 @@ fn try_install_steam_audio(inf_name: &str) -> bool {
|
||||
fn render_thread(
|
||||
queue: Arc<Mutex<VecDeque<u8>>>,
|
||||
stop: Arc<AtomicBool>,
|
||||
shared: Arc<RingShared>,
|
||||
ready: SyncSender<Result<String>>,
|
||||
) -> Result<()> {
|
||||
if let Err(e) = wasapi::initialize_mta()
|
||||
@@ -284,7 +353,7 @@ fn render_thread(
|
||||
}
|
||||
// Open + start the render stream. The WASAPI objects must outlive the loop, so build them here and
|
||||
// keep them (a closure that *returned* them would drop them); on any failure report Err and exit.
|
||||
let setup = (|| -> Result<(wasapi::AudioClient, wasapi::AudioRenderClient, wasapi::Handle, String)> {
|
||||
let setup = (|| -> Result<(wasapi::AudioClient, wasapi::AudioRenderClient, wasapi::Handle, i64, String)> {
|
||||
let (device, name) = resolve_target()?;
|
||||
let mut audio_client = device.get_iaudioclient().context("IAudioClient")?;
|
||||
// 48 kHz stereo f32; autoconvert lets WASAPI shared-mode SRC match the device mix format.
|
||||
@@ -312,9 +381,9 @@ fn render_thread(
|
||||
let buf_frames = audio_client.get_buffer_size().context("buffer size")? as usize;
|
||||
let _ = render_client.write_to_device(buf_frames, &vec![0u8; buf_frames * BLOCK_ALIGN], None);
|
||||
audio_client.start_stream().context("start render stream")?;
|
||||
Ok((audio_client, render_client, h_event, name))
|
||||
Ok((audio_client, render_client, h_event, default_period, name))
|
||||
})();
|
||||
let (audio_client, render_client, h_event, name) = match setup {
|
||||
let (audio_client, render_client, h_event, default_period, name) = match setup {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
let _ = ready.send(Err(anyhow!("{e:#}")));
|
||||
@@ -322,18 +391,26 @@ fn render_thread(
|
||||
}
|
||||
};
|
||||
let _ = ready.send(Ok(name));
|
||||
// One device period in bytes (period is in 100 ns units; floor 10 ms if it reads absurd) —
|
||||
// the device-side pull granularity the adaptive prime threshold builds on.
|
||||
let period_bytes = ((default_period.max(0) as usize * SAMPLE_RATE as usize / 10_000_000)
|
||||
.max(SAMPLE_RATE as usize / 100))
|
||||
* BLOCK_ALIGN;
|
||||
|
||||
// Any error below (endpoint invalidated/removed, engine restart) propagates out of the loop,
|
||||
// ending the thread — the `alive` flag flips in the spawn wrapper and the pump reopens.
|
||||
//
|
||||
// Adaptive jitter buffer (mirrors the Linux backend's process callback): clients push mic
|
||||
// audio in bursts on their own clock while the device pulls a block every period from an
|
||||
// independent clock, so a greedy per-period drain leaves the queue near-empty and pads most
|
||||
// periods with mid-stream silence — audible as constant crackling. Instead: emit silence
|
||||
// until [`PRIME_BYTES`] is buffered, then play from the cushion (zero-filling only a
|
||||
// momentary shortfall), and re-prime only after a genuine FULL drain (the client went quiet —
|
||||
// between talk spurts the cushion rebuilds, and [`VirtualMic::discard`] resets it across
|
||||
// session gaps).
|
||||
// Jitter buffer (mirrors the Linux backend's process callback): clients push mic audio in
|
||||
// bursts on their own clock while the device pulls a block every period from an independent
|
||||
// clock, so a greedy per-period drain leaves the queue near-empty and pads most periods
|
||||
// with mid-stream silence — audible as constant crackling. Instead: emit silence until the
|
||||
// prime threshold is buffered, then play from the cushion (zero-filling only a momentary
|
||||
// shortfall), and re-prime only after a genuine FULL drain (the client went quiet — between
|
||||
// talk spurts the cushion rebuilds, and [`VirtualMic::discard`] resets it across session
|
||||
// gaps). The threshold = one device period + the pump's measured-jitter target
|
||||
// ([`VirtualMic::set_target_depth`]); the fixed [`PRIME_BYTES`] until the pump's first
|
||||
// estimate, and forever under `PUNKTFUNK_MIC_LEGACY_BUFFER=1` (the pump then never drives
|
||||
// the target).
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
let mut primed = false;
|
||||
while !stop.load(Ordering::Relaxed) {
|
||||
@@ -351,11 +428,18 @@ fn render_thread(
|
||||
if buf.len() < need {
|
||||
buf.resize(need, 0);
|
||||
}
|
||||
let target = shared.target_bytes.load(Ordering::Relaxed);
|
||||
let prime = if target == 0 {
|
||||
PRIME_BYTES
|
||||
} else {
|
||||
period_bytes + target
|
||||
};
|
||||
shared.prime_bytes.store(prime, Ordering::Relaxed);
|
||||
// Silence base; overwrite with queued mic PCM once the cushion is primed.
|
||||
buf[..need].fill(0);
|
||||
{
|
||||
let mut q = queue.lock().unwrap();
|
||||
if !primed && q.len() >= PRIME_BYTES {
|
||||
if !primed && q.len() >= prime {
|
||||
primed = true;
|
||||
}
|
||||
if primed {
|
||||
@@ -365,6 +449,7 @@ fn render_thread(
|
||||
}
|
||||
if q.is_empty() {
|
||||
primed = false; // fully drained — re-prime before producing again
|
||||
shared.reprimes.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,11 +69,18 @@ fn capture_for(mic_render_lname: &str) -> &'static [&'static str] {
|
||||
}
|
||||
|
||||
/// A render endpoint no loopback should capture: the VB-CABLE (reserved for the mic even when it
|
||||
/// isn't the chosen target — capturing a cable someone else feeds echoes too) and the Steam
|
||||
/// Streaming Speakers, whose loopback is silent (validated live). Also the capture-side
|
||||
/// watchdog's test for "the operator's new default can never work — snap back to the plan".
|
||||
/// isn't the chosen target — capturing a cable someone else feeds echoes too), the Steam
|
||||
/// Streaming Speakers, whose loopback is silent (validated live), and any VoiceMeeter or
|
||||
/// generically-"virtual" endpoint. VoiceMeeter's strips share one internal mixer: capturing ANY
|
||||
/// of its render endpoints re-captures what the mic wrote into another one — a digital feedback
|
||||
/// loop with no acoustic path to break it (mic=`Voicemeeter Input`, loopback=`Voicemeeter Aux
|
||||
/// Input` used to pass the old name checks). Also the capture-side watchdog's test for "the
|
||||
/// operator's new default can never work — snap back to the plan".
|
||||
pub(crate) fn excluded_from_loopback(lname: &str) -> bool {
|
||||
lname.contains("cable") || lname.contains("steam streaming speakers")
|
||||
lname.contains("cable")
|
||||
|| lname.contains("steam streaming speakers")
|
||||
|| lname.contains("voicemeeter")
|
||||
|| lname.contains("virtual")
|
||||
}
|
||||
|
||||
/// A render endpoint that is SILENT on the host but loopback-capturable — the client-only audio
|
||||
@@ -140,10 +147,14 @@ pub(crate) fn plan(
|
||||
.iter()
|
||||
.find(|(n, id)| not_mic(id) && silent_sink(&n.to_lowercase()))
|
||||
};
|
||||
// `virtualish` here too: a virtual endpoint that slipped past `excluded_from_loopback`'s
|
||||
// name list (a future cable/mixer sibling) is an internal-feedback loop waiting to happen —
|
||||
// no loopback is the honest answer, exactly like the cable-only case.
|
||||
let leftover = || {
|
||||
renders
|
||||
.iter()
|
||||
.find(|(n, id)| not_mic(id) && !excluded_from_loopback(&n.to_lowercase()))
|
||||
renders.iter().find(|(n, id)| {
|
||||
let ln = n.to_lowercase();
|
||||
not_mic(id) && !excluded_from_loopback(&ln) && !virtualish(&ln)
|
||||
})
|
||||
};
|
||||
let loopback_render = if host_audio {
|
||||
real_hw().or_else(silent).or_else(leftover)
|
||||
@@ -359,4 +370,59 @@ mod tests {
|
||||
assert!(w.mic_render.is_none());
|
||||
assert_eq!(w.loopback_render.unwrap().0, "Speakers (Realtek HD Audio)");
|
||||
}
|
||||
|
||||
/// VoiceMeeter box with real hardware: no cable, so the mic takes `Voicemeeter Input` — and
|
||||
/// the loopback must NEVER be `Voicemeeter Aux Input` (same internal mixer: capturing any
|
||||
/// VoiceMeeter render re-captures what the mic wrote — a digital feedback loop). Real
|
||||
/// hardware wins the loopback in both modes.
|
||||
#[test]
|
||||
fn voicemeeter_aux_never_pairs_with_voicemeeter_mic() {
|
||||
let renders = [
|
||||
ep("Voicemeeter Input (VB-Audio Voicemeeter VAIO)"),
|
||||
ep("Voicemeeter Aux Input (VB-Audio Voicemeeter AUX VAIO)"),
|
||||
ep("Speakers (Realtek HD Audio)"),
|
||||
];
|
||||
let captures = [ep("Voicemeeter Out B1 (VB-Audio Voicemeeter VAIO)")];
|
||||
for host_audio in [false, true] {
|
||||
let w = plan(&renders, &captures, None, host_audio);
|
||||
assert_eq!(
|
||||
w.mic_render.as_ref().unwrap().0,
|
||||
"Voicemeeter Input (VB-Audio Voicemeeter VAIO)",
|
||||
"host_audio={host_audio}"
|
||||
);
|
||||
assert_eq!(
|
||||
w.loopback_render.as_ref().unwrap().0,
|
||||
"Speakers (Realtek HD Audio)",
|
||||
"host_audio={host_audio}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Only VoiceMeeter endpoints (headless mixer box): mic wins one, the loopback is honestly
|
||||
/// absent — like the cable-only case, never another strip of the same mixer.
|
||||
#[test]
|
||||
fn voicemeeter_only_no_loopback() {
|
||||
let renders = [
|
||||
ep("Voicemeeter Input (VB-Audio Voicemeeter VAIO)"),
|
||||
ep("Voicemeeter Aux Input (VB-Audio Voicemeeter AUX VAIO)"),
|
||||
];
|
||||
for host_audio in [false, true] {
|
||||
let w = plan(&renders, &[], None, host_audio);
|
||||
assert!(w.mic_render.is_some(), "host_audio={host_audio}");
|
||||
assert!(w.loopback_render.is_none(), "host_audio={host_audio}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A generically-"virtual" leftover (unknown vendor cable) is refused too: `leftover()`
|
||||
/// applies `virtualish`, so a virtual endpoint that slips past the name list can't become
|
||||
/// the loopback.
|
||||
#[test]
|
||||
fn unknown_virtual_never_loopback() {
|
||||
let renders = [
|
||||
ep("CABLE Input (VB-Audio Virtual Cable)"),
|
||||
ep("Speakers (Some Virtual Audio Device)"),
|
||||
];
|
||||
let w = plan(&renders, &[], None, false);
|
||||
assert!(w.loopback_render.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -234,9 +234,10 @@ pub fn start(
|
||||
});
|
||||
}
|
||||
|
||||
/// Stub — the audio plane needs an audio-capture backend (PipeWire on Linux, WASAPI on Windows)
|
||||
/// + libopus; this keeps the remaining targets (e.g. macOS) compiling (crate doc: "the crate
|
||||
/// compiles everywhere"). Reports failure the same way the real stream thread does: clears `running`.
|
||||
/// Stub — the audio plane needs an audio-capture backend (PipeWire on Linux, WASAPI on
|
||||
/// Windows) + libopus; this keeps the remaining targets (e.g. macOS) compiling (crate doc:
|
||||
/// "the crate compiles everywhere"). Reports failure the same way the real stream thread
|
||||
/// does: clears `running`.
|
||||
#[cfg(not(any(target_os = "linux", target_os = "windows")))]
|
||||
pub fn start(
|
||||
running: std::sync::Arc<std::sync::atomic::AtomicBool>,
|
||||
|
||||
@@ -1649,10 +1649,11 @@ async fn hooks_get_shape_and_put_validation() {
|
||||
|
||||
// ------------------------------------------------------------------ library scanners
|
||||
|
||||
/// The scanner list is platform-shaped and read-only-safe; the toggle rejects unknown ids with
|
||||
/// 404. (A successful toggle PUT would write the developer's real `library-scanners.json`, so the
|
||||
/// write path is exercised only through the unknown-id rejection here — the settings round-trip
|
||||
/// itself is unit-tested in `library::scanners` against pure shapes.)
|
||||
/// The scanner list is platform-shaped and read-only-safe; the toggle rejects unknown ids
|
||||
/// with 404. (A successful toggle PUT would write the developer's real
|
||||
/// `library-scanners.json`, so the write path is exercised only through the unknown-id
|
||||
/// rejection here — the settings round-trip itself is unit-tested in `library::scanners`
|
||||
/// against pure shapes.)
|
||||
#[tokio::test]
|
||||
async fn library_scanner_list_and_unknown_toggle() {
|
||||
let app = test_app(test_state(), None);
|
||||
|
||||
@@ -83,6 +83,12 @@ pub(crate) struct UpdateStatus {
|
||||
pub last_checked_unix: Option<u64>,
|
||||
/// Why the last check failed, verbatim, if it did.
|
||||
pub last_error: Option<String>,
|
||||
/// The check reached the feed and found this channel has **no release published yet** —
|
||||
/// an expected state (a channel nobody has announced to answers with a 404), not a
|
||||
/// failure. Mutually exclusive with `last_error`, so a UI can say "nothing published yet"
|
||||
/// instead of painting an empty feed as a broken host. Never set once a manifest has been
|
||||
/// seen for this channel: a feed that loses a document it used to serve stays an error.
|
||||
pub not_published: bool,
|
||||
/// This install could one-click apply, but the operator hasn't opted in yet — the
|
||||
/// command to run (Linux: join the `punktfunk-update` group).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -142,6 +148,7 @@ fn status_from(snap: update::Snapshot) -> UpdateStatus {
|
||||
}),
|
||||
last_checked_unix: snap.checked.as_ref().map(|c| c.fetched_unix),
|
||||
last_error: snap.last_error,
|
||||
not_published: snap.not_published,
|
||||
opt_in_hint: update::opt_in_hint(),
|
||||
job,
|
||||
last_result: snap.last_result.as_ref().map(|r| UpdateResultInfo {
|
||||
@@ -186,7 +193,7 @@ pub(crate) async fn get_update_status() -> Json<UpdateStatus> {
|
||||
tag = "update",
|
||||
operation_id = "forceUpdateCheck",
|
||||
responses(
|
||||
(status = OK, description = "Refreshed update-check state (`last_error` carries a failed check)", body = UpdateStatus),
|
||||
(status = OK, description = "Refreshed update-check state (`last_error` carries a failed check; `not_published` an empty channel, which is not one)", body = UpdateStatus),
|
||||
(status = CONFLICT, description = "Update checks are disabled on this host", body = ApiError),
|
||||
(status = TOO_MANY_REQUESTS, description = "A forced check ran less than 30 s ago", body = ApiError),
|
||||
(status = UNAUTHORIZED, description = "Missing or invalid bearer token", body = ApiError),
|
||||
|
||||
@@ -760,7 +760,7 @@ async fn serve_session(
|
||||
opts: &Punktfunk1Options,
|
||||
audio_cap: &AudioCapSlot,
|
||||
inj_tx: std::sync::mpsc::Sender<InputEvent>,
|
||||
mic_tx: std::sync::mpsc::SyncSender<Vec<u8>>,
|
||||
mic_tx: std::sync::mpsc::SyncSender<crate::audio::MicFrame>,
|
||||
host_fp: &[u8; 32],
|
||||
np: &NativePairing,
|
||||
last_pairing: &std::sync::Mutex<Option<std::time::Instant>>,
|
||||
@@ -1178,11 +1178,17 @@ async fn serve_session(
|
||||
tokio::spawn(async move {
|
||||
let (mut input_count, mut mic_count, mut rich_count) = (0u64, 0u64, 0u64);
|
||||
while let Ok(d) = input_conn.read_datagram().await {
|
||||
if let Some((_seq, _pts, opus)) = punktfunk_core::quic::decode_mic_datagram(&d) {
|
||||
if let Some((seq, pts, opus)) = punktfunk_core::quic::decode_mic_datagram(&d) {
|
||||
mic_count += 1;
|
||||
// Host-lifetime mic service (bounded queue): `try_send` drops the frame when the
|
||||
// service is full or gone, never blocking this datagram loop (security-review S6).
|
||||
let _ = mic_tx.try_send(opus.to_vec());
|
||||
// seq + pts ride along — the pump's de-jitter reorders, conceals losses and
|
||||
// tracks cadence with them (they used to be decoded here and thrown away).
|
||||
let _ = mic_tx.try_send(crate::audio::MicFrame {
|
||||
seq,
|
||||
pts_ns: pts,
|
||||
opus: opus.to_vec(),
|
||||
});
|
||||
} else if let Some(rich) = punktfunk_core::quic::RichInput::decode(&d) {
|
||||
rich_count += 1;
|
||||
if rich_tx.send(ClientInput::Rich(rich)).is_err() {
|
||||
|
||||
@@ -27,7 +27,7 @@ pub(crate) use pf_update_check::manifest;
|
||||
pub(crate) mod windows;
|
||||
|
||||
use manifest::Manifest;
|
||||
use pf_update_check::PublicKey;
|
||||
use pf_update_check::{FeedError, PublicKey};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
@@ -141,6 +141,9 @@ pub(crate) struct Checked {
|
||||
struct Runtime {
|
||||
checked: Option<Checked>,
|
||||
last_error: Option<String>,
|
||||
/// The channel has nothing published (and never had, for us) — an expected state, kept
|
||||
/// out of `last_error` so a console never paints an empty feed as a broken host.
|
||||
not_published: bool,
|
||||
/// Refresh in flight (status kicks at most one).
|
||||
refreshing: bool,
|
||||
/// Wall-clock guard for the forced-check rate limit.
|
||||
@@ -214,7 +217,7 @@ fn store_floor(path: &Path, channel: &str, serial: u64) {
|
||||
|
||||
/// Fetch + verify the channel manifest through the shared checker. Blocking — call from a
|
||||
/// blocking thread.
|
||||
fn fetch_manifest_blocking(channel: &str) -> Result<Manifest, String> {
|
||||
fn fetch_manifest_blocking(channel: &str) -> Result<Manifest, FeedError> {
|
||||
pf_update_check::feed::fetch_manifest_blocking(
|
||||
&pf_update_check::feed::feed_base(),
|
||||
channel,
|
||||
@@ -227,18 +230,18 @@ fn fetch_manifest_blocking(channel: &str) -> Result<Manifest, String> {
|
||||
}
|
||||
|
||||
/// One full refresh: fetch, verify, enforce + raise the serial floor, update the cache,
|
||||
/// announce a newly available release on the event bus. Returns the user-facing error string
|
||||
/// on failure (also cached for status).
|
||||
pub(crate) fn refresh_blocking() -> Result<Checked, String> {
|
||||
/// announce a newly available release on the event bus. Returns the user-facing error on
|
||||
/// failure (also cached for status).
|
||||
pub(crate) fn refresh_blocking() -> Result<Checked, FeedError> {
|
||||
let (kind, channel) = detect::detect();
|
||||
let result = fetch_manifest_blocking(channel.as_str()).and_then(|m| {
|
||||
let path = state_path();
|
||||
let floor = load_floor(&path, channel.as_str());
|
||||
if m.serial < floor {
|
||||
return Err(format!(
|
||||
return Err(FeedError::Failed(format!(
|
||||
"manifest serial {} is older than the last accepted {} — refusing rollback",
|
||||
m.serial, floor
|
||||
));
|
||||
)));
|
||||
}
|
||||
store_floor(&path, channel.as_str(), m.serial);
|
||||
Ok(m)
|
||||
@@ -268,16 +271,32 @@ pub(crate) fn refresh_blocking() -> Result<Checked, String> {
|
||||
});
|
||||
}
|
||||
rt.last_error = None;
|
||||
rt.not_published = false;
|
||||
rt.checked = Some(checked.clone());
|
||||
Ok(checked)
|
||||
}
|
||||
Err(e) => {
|
||||
rt.last_error = Some(e.clone());
|
||||
let (last_error, not_published) = classify_failure(&e, rt.checked.is_some());
|
||||
rt.last_error = last_error;
|
||||
rt.not_published = not_published;
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Split a failed refresh into `(last_error, not_published)` — the two are never both set.
|
||||
///
|
||||
/// An empty channel is only benign while we have never seen a manifest for it. Once a check
|
||||
/// has succeeded, the same 404 means the feed LOST a document it used to serve, which is a
|
||||
/// regression that must stay a loud error rather than being excused as "nothing yet".
|
||||
fn classify_failure(e: &FeedError, had_manifest: bool) -> (Option<String>, bool) {
|
||||
if e.is_not_published() && !had_manifest {
|
||||
(None, true)
|
||||
} else {
|
||||
(Some(e.to_string()), false)
|
||||
}
|
||||
}
|
||||
|
||||
/// The status handler's read: current cache + errors, kicking a background refresh when the
|
||||
/// cache is cold and checks are enabled.
|
||||
pub(crate) fn snapshot_and_maybe_refresh() -> Snapshot {
|
||||
@@ -296,6 +315,7 @@ pub(crate) fn snapshot_and_maybe_refresh() -> Snapshot {
|
||||
Snapshot {
|
||||
checked: rt.checked.clone(),
|
||||
last_error: rt.last_error.clone(),
|
||||
not_published: rt.not_published,
|
||||
job: rt.job.clone(),
|
||||
last_result: jobs::read_result(&jobs::result_path()),
|
||||
}
|
||||
@@ -329,6 +349,7 @@ pub(crate) async fn force_check() -> Result<Snapshot, ForceError> {
|
||||
Ok(Snapshot {
|
||||
checked: rt.checked.clone(),
|
||||
last_error: rt.last_error.clone(),
|
||||
not_published: rt.not_published,
|
||||
job: rt.job.clone(),
|
||||
last_result: jobs::read_result(&jobs::result_path()),
|
||||
})
|
||||
@@ -570,6 +591,8 @@ pub(crate) fn reconcile_at_boot() {
|
||||
pub(crate) struct Snapshot {
|
||||
pub checked: Option<Checked>,
|
||||
pub last_error: Option<String>,
|
||||
/// The channel simply has no release yet — mutually exclusive with `last_error`.
|
||||
pub not_published: bool,
|
||||
/// The live apply job, when one runs. When the process was restarted mid-apply this is
|
||||
/// `None` but a fresh intent still reads as in-flight — the API layer surfaces that via
|
||||
/// [`Snapshot::applying_from_intent`].
|
||||
@@ -640,6 +663,34 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// The console paints `last_error` as a failure and `not_published` as a plain sentence,
|
||||
/// so the two must never arrive together — and the benign reading must not survive a
|
||||
/// channel that has already served us a manifest.
|
||||
#[test]
|
||||
fn empty_channel_is_benign_only_until_a_manifest_has_been_seen() {
|
||||
let (err, not_published) = classify_failure(&FeedError::NotPublished, false);
|
||||
assert_eq!(err, None);
|
||||
assert!(not_published);
|
||||
|
||||
// Same 404, but the feed used to answer — that is a regression, not "nothing yet".
|
||||
let (err, not_published) = classify_failure(&FeedError::NotPublished, true);
|
||||
assert_eq!(
|
||||
err.as_deref(),
|
||||
Some("no release has been published on this channel yet")
|
||||
);
|
||||
assert!(!not_published);
|
||||
|
||||
// Every real failure stays an error whether or not we have a cached manifest.
|
||||
for had_manifest in [false, true] {
|
||||
let (err, not_published) = classify_failure(
|
||||
&FeedError::Failed("feed returned HTTP 500".into()),
|
||||
had_manifest,
|
||||
);
|
||||
assert_eq!(err.as_deref(), Some("feed returned HTTP 500"));
|
||||
assert!(!not_published);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pinned_keys_skip_empty_rotation_slot() {
|
||||
let keys = pinned_keys();
|
||||
@@ -664,6 +715,7 @@ mod tests {
|
||||
fetched_unix: now_unix(),
|
||||
}),
|
||||
last_error: None,
|
||||
not_published: false,
|
||||
job: None,
|
||||
last_result: None,
|
||||
};
|
||||
|
||||
@@ -99,7 +99,20 @@ claims a sink advertising exactly that many channels, so applications produce re
|
||||
from a stereo endpoint is an upmix, not new channels. Offered everywhere except the Decky plugin.
|
||||
|
||||
**Microphone** — *default: off on Linux, Windows, Android, the console home and Decky; on in the
|
||||
Apple app.* Sends this device's microphone to the host's virtual mic.
|
||||
Apple app.* Sends this device's microphone to the host's virtual mic. On Linux and Windows the
|
||||
row is spelled *Stream microphone*, and **Ctrl+Alt+Shift+V** mutes it mid-stream without ending
|
||||
anything — see [Muting your microphone](/docs/input#muting-your-microphone).
|
||||
|
||||
**Echo cancellation** — *default: on.* Stops the host's audio, playing out of this device's
|
||||
speakers, from being picked up by the microphone and sent straight back. It hands the microphone
|
||||
to the system's own canceller rather than doing the work itself: on **Linux** that means capturing
|
||||
from an echo-cancelled PipeWire source when your desktop provides one, on **Windows** asking WASAPI
|
||||
for the Communications stream category so the endpoint's processing engages, and on **Apple** and
|
||||
**Android** the platform's voice-processing mode. Turn it off if your microphone already runs its
|
||||
own processing, or if the canceller makes your voice sound thin. The row sits under the microphone
|
||||
toggle and greys out while the microphone is off. Offered by the Linux, Windows, Apple, Android and
|
||||
console-home clients; Decky has no toggle. What it can and can't fix is in
|
||||
[Why do I hear myself](/docs/echo).
|
||||
|
||||
**Speaker** and **Microphone** device pickers — *default: System default.* Which endpoint stream
|
||||
audio plays out of, and which input feeds the uplink. Only the Linux app (PipeWire nodes) and the
|
||||
|
||||
@@ -150,6 +150,7 @@ See your desktop page ([KDE](/docs/kde), [GNOME](/docs/gnome)) for when to set t
|
||||
|---|---|---|
|
||||
| `PUNKTFUNK_AUDIO_GAIN` | float (default `1.0`) | **(Moonlight/GameStream sessions only)** Linear gain applied to captured desktop audio — bump it for a quiet source. The native `punktfunk/1` path ignores it; adjust the source's own volume there instead. |
|
||||
| `PUNKTFUNK_MIC_DEVICE` | name substring | **(Windows)** Target mic-uplink device by friendly-name substring (first match wins). |
|
||||
| `PUNKTFUNK_MIC_LEGACY_BUFFER` | `1` | Restore the fixed pre-adaptive mic buffering (a ~48 ms prime and ~120 ms cap on Windows; a buffer scaled to the recording app's audio quantum on Linux) instead of the adaptive per-client jitter target. One-release escape hatch: if the microphone coming out of the host only sounds right *with* this set, that's a bug — please report it. |
|
||||
| `PUNKTFUNK_NO_MIC_INSTALL` | set | **(Windows)** Skip installing the virtual-mic driver (e.g. when the host runs as SYSTEM). |
|
||||
| `PUNKTFUNK_HOST_AUDIO` | set | **(Windows)** Also play the stream's audio on the host's own speakers. While a session is capturing desktop audio the host parks the default playback device on a silent sink, so sound comes out of the *client* only — that's why the PC goes quiet when a stream starts. Set this to prefer a real output device instead (audible on both ends). The default is put back when the capture closes. |
|
||||
| `PUNKTFUNK_KEEP_DEFAULT` | set | **(Windows)** Never touch the default playback/recording devices at all — the host leaves whatever you chose in Sound settings in place. The mic uplink still picks a target device; you may then have to select it yourself. |
|
||||
@@ -239,6 +240,7 @@ A few knobs are read by the native **clients**, not the host:
|
||||
| `PUNKTFUNK_DECODER` | `software` · `vaapi` · `vulkan` (Linux) · `d3d11va` (Windows) | Force the decode path. Default auto-selects hardware per GPU vendor and falls back on its own: **Linux** — Vulkan Video first on NVIDIA and AMD, VAAPI first on Intel and anything else; **Windows** — Vulkan Video first on NVIDIA and AMD, D3D11VA first on Intel and anything else. Whichever isn't first is the next thing tried, with software last. |
|
||||
| `PUNKTFUNK_PREFER_PYROWAVE` | `1` | Ask for the [PyroWave](/docs/pyrowave) wavelet codec on a wired link, where the client's own setting isn't reachable (the gamepad console, a headless launch). |
|
||||
| `PUNKTFUNK_OSD_SCALE` | multiplier, e.g. `1.5` *(default `1`)* | Size of the in-stream overlay — the stats OSD, the capture hint and the start banner. They already follow your display's scaling setting (200 % display → twice the pixels), so set this only to nudge that: bigger for a TV across the room, smaller if your compositor reports an aggressive scale. Clamped to 0.5×–4×, and a line that would run off the screen is shrunk to fit. |
|
||||
| `PUNKTFUNK_NO_AEC` | `1` | Turn the microphone's echo cancellation off for this run, whatever **Echo cancellation** says in [client settings](/docs/client-settings#audio). One-way: it can only switch the processing off, never back on, and the setting is the normal way to control it. Linux and Windows clients. |
|
||||
|
||||
## Bitrate
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
---
|
||||
title: Why do I hear myself
|
||||
description: Echo while streaming — the four places it can come from (client speakers, Windows monitoring loops, host speakers, virtual mixers) and how to stop each one.
|
||||
---
|
||||
|
||||
You talk into your device's microphone and hear your own voice come back a beat later. The echo
|
||||
is almost never "in the stream" itself — it's a loop in one of four well-known places. Work
|
||||
through them in order; the first two cover nearly every report.
|
||||
|
||||
## Your device's speakers
|
||||
|
||||
If the stream's audio plays out of the **speakers of the device you're streaming on** — a
|
||||
phone, tablet or laptop without headphones — its microphone picks that sound back up and sends
|
||||
it to the host along with your voice. Everyone in your voice chat hears the game twice, and you
|
||||
hear yourself whenever anything routes the mic back.
|
||||
|
||||
**Fix: use headphones on the device you're streaming on.** Clients also try to cancel it for
|
||||
you: **Echo cancellation** in [client settings](/docs/client-settings#audio) is on by default on
|
||||
Linux, Windows, the console home, the Apple apps and Android, and hands the microphone to the
|
||||
system's own canceller. How much it removes depends on the device — a laptop with a good array
|
||||
mic can clear it entirely, a cheap USB mic next to a speaker can't — so headphones remain the
|
||||
reliable fix everywhere.
|
||||
|
||||
If you just need to stop talking for a moment, **Ctrl+Alt+Shift+V** mutes the microphone without
|
||||
leaving the stream; see [Input](/docs/input#getting-your-input-back).
|
||||
|
||||
## "Listen to this device" and app monitoring (Windows hosts)
|
||||
|
||||
Windows can play a microphone straight out of the speakers. If **Listen to this device** is
|
||||
ticked for the Punktfunk mic (usually *CABLE Output*), your voice plays on the host's output —
|
||||
which the stream then captures and sends right back to you.
|
||||
|
||||
Open **Sound settings → More sound settings → Recording**, double-click *CABLE Output*, and on
|
||||
the **Listen** tab untick *Listen to this device*.
|
||||
|
||||
The same loop hides in apps: **Discord's** *Mic Test* / input monitoring, **OBS's** *Monitor
|
||||
audio* on a mic source, and similar monitoring features in other tools all play your mic into
|
||||
the host's output. Turn the monitoring off rather than the mic.
|
||||
|
||||
## The host's own speakers
|
||||
|
||||
If you set `PUNKTFUNK_HOST_AUDIO` (Windows) so the stream's sound also plays in the room, and
|
||||
you're streaming **from that same room**, your device's mic hears the host's speakers. Remove
|
||||
the setting while you stream from nearby, or turn the host's volume down.
|
||||
|
||||
## Virtual mixers (VoiceMeeter and friends)
|
||||
|
||||
VoiceMeeter's virtual devices all share one internal mixer. Older Punktfunk hosts could pick one
|
||||
VoiceMeeter strip as the microphone target and *another* as the audio capture — a feedback loop
|
||||
with no acoustics involved at all. Current hosts refuse to capture VoiceMeeter or other virtual
|
||||
endpoints for desktop audio, so this fixes itself with an update. If you route audio through
|
||||
VoiceMeeter on purpose, make sure no strip that hears the Punktfunk mic feeds the output being
|
||||
streamed.
|
||||
|
||||
## What the host log can tell you
|
||||
|
||||
While your microphone is in use, the host writes a **`mic uplink health`** line to its log every
|
||||
30 seconds (web console → **Logs**). It shows how much of your voice is buffered on the host
|
||||
(`depth_ms` vs `target_ms`), how much the network lost (`gaps`, `concealed`), and how steadily
|
||||
your client is delivering audio (`cadence_ms`). It won't point at an echo loop directly — echo
|
||||
is a routing problem, not a network one — but if your voice also sounds choppy or delayed,
|
||||
include that line in a bug report. The startup log also names exactly which devices the host
|
||||
picked for the microphone and for audio capture, which is the quickest way to spot a monitoring
|
||||
loop like the ones above.
|
||||
@@ -21,9 +21,10 @@ window — see [Mouse modes](#mouse-modes) below.
|
||||
| **Ctrl+Alt+Shift+M** | Switch the mouse mode (capture ⇄ desktop) |
|
||||
| **Ctrl+Alt+Shift+D** | Disconnect |
|
||||
| **Ctrl+Alt+Shift+S** | Cycle the [stats overlay](/docs/stats) — off · compact · normal · detailed |
|
||||
| **Ctrl+Alt+Shift+V** | Mute or unmute your microphone |
|
||||
| **F11** or **Alt+Enter** | Toggle fullscreen |
|
||||
|
||||
While input is released the session window prints the list over the stream:
|
||||
While input is released the session window prints the shortest list over the stream:
|
||||
|
||||
```text
|
||||
Click the stream to capture input · Ctrl+Alt+Shift+Q releases · Ctrl+Alt+Shift+M mouse mode ·
|
||||
@@ -31,7 +32,25 @@ Ctrl+Alt+Shift+D disconnects · Ctrl+Alt+Shift+S stats
|
||||
```
|
||||
|
||||
With a controller in use the same hint names the controller chord instead of the mouse-mode and
|
||||
stats entries.
|
||||
stats entries. The full list is always available without a stream running — see below.
|
||||
|
||||
### Muting your microphone
|
||||
|
||||
**Ctrl+Alt+Shift+V** stops sending your microphone to the host, and pressing it again resumes.
|
||||
The uplink itself keeps running underneath, so unmuting is instant rather than a second of the
|
||||
device warming back up.
|
||||
|
||||
While you are muted a **Microphone muted** badge sits in the top-right corner of the stream. It
|
||||
is deliberately separate from the [stats overlay](/docs/stats): it shows even with stats off,
|
||||
because "am I still muted?" is a question you ask ten minutes later.
|
||||
|
||||
The mute lasts for that stream only — the next session starts unmuted, and nothing is written to
|
||||
your settings. If the stream isn't sending a microphone at all (**Stream microphone** off in
|
||||
[client settings](/docs/client-settings#audio)) the shortcut does nothing and no badge appears,
|
||||
rather than pretending to mute something.
|
||||
|
||||
This is on the **Linux and Windows** clients. The Apple, Android and Decky clients have no mute
|
||||
shortcut yet; turn **Stream microphone** off in their settings instead.
|
||||
|
||||
Alt-Tabbing away releases input on its own and takes it back when you return. A release you asked
|
||||
for with the chord stays released until you opt back in. Either way, keys and buttons you were
|
||||
@@ -39,11 +58,13 @@ holding are released on the host, so nothing sticks down.
|
||||
|
||||
You can look the shortcuts up again without a stream running: the Linux client has **Keyboard
|
||||
Shortcuts** in its main menu, and the Windows client has a **Shortcuts** screen reached from its
|
||||
host list.
|
||||
host list. Both list the microphone mute; the in-stream hint over the video doesn't, to keep it
|
||||
to one readable line.
|
||||
|
||||
### On the other clients
|
||||
|
||||
- **macOS** honours the same four combos, written **⌃⌥⇧Q / M / D / S**. **⌘⎋** also toggles capture,
|
||||
- **macOS** honours the release, mouse-mode, disconnect and stats combos, written
|
||||
**⌃⌥⇧Q / M / D / S** — but not the microphone mute. **⌘⎋** also toggles capture,
|
||||
**⌃⌘F** toggles fullscreen, and **⌃⌥⇧C** starts or stops [clipboard sharing](/docs/clipboard). The
|
||||
**Stream** menu lists them all except the mouse-mode combo, which works but has no menu item.
|
||||
- **iPhone and iPad** with a hardware keyboard: **⌃⌥⇧Q** releases input while it is captured, and
|
||||
|
||||
@@ -47,6 +47,7 @@
|
||||
"wake-on-lan",
|
||||
"---Troubleshooting---",
|
||||
"troubleshooting",
|
||||
"echo",
|
||||
"stats",
|
||||
"forgot-password",
|
||||
"---Project---",
|
||||
|
||||
@@ -40,7 +40,7 @@ in-stream:
|
||||
| Android · iPhone | a **three-finger tap** |
|
||||
|
||||
**Ctrl+Alt+Shift+S** is one of a small set of shortcuts a stream reserves; the others — release
|
||||
captured input, switch mouse mode, disconnect — are in
|
||||
captured input, switch mouse mode, disconnect, mute the microphone — are in
|
||||
[Getting your input back](/docs/input#getting-your-input-back).
|
||||
|
||||
**Compact** is a one-line pill (fps · end-to-end ms · Mb/s, plus a loss flag when frames are being
|
||||
|
||||
@@ -1,14 +1,10 @@
|
||||
Wire-compatible with 0.21.x and 0.22.x — nothing about streaming changed, and everything already paired keeps working. This release repairs the Windows installer and gives every host something new: the web console now tells you when a newer Punktfunk is out, and can install it for you where the platform allows. The apps on your phone, tablet, Mac and TV are unchanged.
|
||||
Wire-compatible with 0.21.x and 0.22.x — nothing about streaming changed, and everything already paired keeps working. This release repairs the Windows installer and gives every host something new: the web console now tells you when a newer Punktfunk is out. The apps on your phone, tablet, Mac and TV are unchanged.
|
||||
|
||||
**If you stream to a Windows PC, update that PC.** The 0.22.1 and 0.22.2 installers were built without the web console in them at all — not broken, absent. If the tray menu on your PC says "Open web console (not responding)" and the page never loads no matter what you try, that is this, and reinstalling those versions could never have fixed it.
|
||||
|
||||
## New
|
||||
|
||||
- **The console tells you when a newer host is out.** The Host page has an **Updates** card: the version you're running, the channel you follow (stable or canary), and — once something newer exists — release notes and the exact way to update *your* kind of install. The check is a small signed file the host verifies against keys built into it, so nothing between you and the project can feed it a fake update, and a feed that ever tried to roll you backwards is refused outright. It contacts only the Punktfunk forge, and `PUNKTFUNK_UPDATE_CHECK=0` turns it off entirely. Scripts can react too: the host emits `update.available` and, after a successful update, `update.applied` on its event stream.
|
||||
|
||||
- **On Windows, updating the host is now one click.** The card grows an **Update now** button: it asks for your console password again (a remembered login alone can't restart your PC's host), then downloads the installer, checks it against the signed release manifest *and* its code signature, and runs it silently — the service restarts and the page reconnects by itself. If someone is streaming you're warned first, because updating drops the stream. Every attempt leaves its outcome in the card even across the restart, with the installer's log path when something went wrong — and if a freshly installed version ever crash-loops, the host puts the previous installer back on its own and says so. `PUNKTFUNK_UPDATE_APPLY=0` in `host.env` removes the button on hosts that should never have it.
|
||||
|
||||
- **Linux hosts can opt in to the same button.** apt, Fedora, Bazzite (sysext and rpm-ostree) installs get one-click updating through a deliberately tiny root helper the packages now ship. It's off until you decide otherwise — enabling it is one command, `sudo usermod -aG punktfunk-update $USER`, and that group membership is the entire grant: the button can only ever run your package manager's normal update for the Punktfunk packages, from the same signed repositories you already use, and it never picks versions or URLs. rpm-ostree updates are staged and the card asks you to reboot (it never reboots for you). On Arch the button additionally requires opting into a full `pacman -Syu` in `/etc/punktfunk/update.conf`, because a partial upgrade is how Arch boxes break — we won't run one. Steam Deck on-device installs get the button with no opt-in at all: it runs the same rebuild `update.sh` always did, just from the couch. The full story is on the new [Updating the Host](https://punktfunk.unom.io/docs/updating) docs page.
|
||||
- **The console tells you when a newer host is out.** The Host page has an **Updates** card: the version you're running, the channel you follow (stable or canary), and — once something newer exists — release notes and the exact way to update *your* kind of install. The check is a small signed file the host verifies against keys built into it, so nothing between you and the project can feed it a fake update, and a feed that ever tried to roll you backwards is refused outright. It contacts only the Punktfunk forge, and `PUNKTFUNK_UPDATE_CHECK=0` turns it off entirely. Scripts can react too: the host emits `update.available` on its event stream. Actually *applying* an update from the card arrives in the next release.
|
||||
|
||||
## Fixed
|
||||
|
||||
@@ -33,6 +29,5 @@ Wire-compatible with 0.21.x and 0.22.x — nothing about streaming changed, and
|
||||
- `PUNKTFUNK_ABR_PROBE_KBPS` sets the startup capacity probe's burst target for embedders. Unset, zero or unparseable keeps the 2 Gbps default, so existing sessions are unchanged; `PUNKTFUNK_ABR_PROBE=0` remains a poor substitute because it pins the climb ceiling at the negotiated ~20 Mbps.
|
||||
- The console's log export serializes the filters' full result rather than the rendered tail — the 1000-row cap is a DOM budget and has no business truncating a file destined for a bug report — and stamps each line with a local ISO 8601 timestamp plus numeric offset. Web Share level 2 is probed at runtime after mount (SSR has no `navigator`, and guessing there would mismatch on hydration), falling back to the clipboard, and the button is omitted only where neither exists.
|
||||
- The update check's truth is a per-channel Ed25519-signed manifest (`…/generic/punktfunk-update/{stable,canary}/manifest.json` + `.sig`), verified against keys pinned in the host binary via the plugin-index machinery, with a persisted monotonic serial as the anti-rollback floor and the channel name bound into the signed document. Stable manifests publish from `announce.yml` — the existing manual fleet-green gate doubles as the gate for the fleet learning about a release; canary rides `windows-host.yml`. TLS and the registry are transport, never trust.
|
||||
- `/api/v1/update/*` is admin-lane only: whole-prefix denied to the plugin token and absent from the paired-cert allowlist. Apply requests carry no version, URL, or channel — the privileged side derives everything from its own verified state — and the console's BFF re-verifies the console password per apply (sharing the login throttle) while every mutating route now requires a same-origin `Sec-Fetch-Site`. Outcomes survive the host's own restart via an intent record reconciled at boot.
|
||||
- The Linux packages grow four pieces: the dep-free root helper `/usr/libexec/punktfunk/pf-update`, the fixed-ExecStart oneshot `punktfunk-update.service`, a polkit rule granting exactly that unit's `start` verb to the `punktfunk-update` group, and the group itself — created empty by every postinst, populated by no one.
|
||||
- `/api/v1/update/*` is admin-lane only: whole-prefix denied to the plugin token and absent from the paired-cert allowlist. This release exposes the check surface only — there is no apply route in it.
|
||||
- No wire, ABI or driver-protocol changes: wire protocol 2, C ABI 13, Windows virtual-gamepad channel 3, virtual-display driver protocol 6 — identical to 0.22.0, 0.22.1 and 0.22.2.
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
Wire-compatible with 0.22.x — everything you have already paired keeps working, and you can update one side at a time. Every new ability here is negotiated per device: a phone, TV or host that does not know about one simply streams the way it does today.
|
||||
|
||||
This is a big release, and it pulls in four directions. **Latency:** Android phones and tablets get a new frame scheduler that cut glass-to-glass delay by roughly a third on the devices we measured, a host can line its capture up with your device's display instead of drifting against it, and a frame can start decoding before the rest of it has arrived. **Your microphone:** it can be muted mid-stream on every client now, echo cancellation is on by default, and the uplink itself was rebuilt to be about half as slow. **Settings that do what they say:** the Linux app has been quietly streaming at defaults since 0.22.0 — bitrate, resolution, codec, HDR, profiles — and that is fixed, along with HDR and full chroma no longer being mutually exclusive on Windows. **The web console:** it now follows the host live instead of polling it, shows you what just happened, hands a phone a link to connect with, and installs to a home screen. Plus HDR by default on Linux hosts that run their games through gamescope, a DualSense over USB on Android, and a real **Update now** button.
|
||||
|
||||
## New
|
||||
|
||||
- **Your microphone has an off switch you can reach mid-stream.** Muting used to mean leaving the stream and switching the microphone off in Settings — an app-wide setting doing the job of a moment-to-moment one. Every client can mute now: **Ctrl+Alt+Shift+V** on Windows and Linux, **⌃⌥⇧A** or the HUD button on Mac, iPhone and iPad, and a tap on the pill (or **Select + Y** on a controller) on Android. A "Microphone muted" badge sits over the stream independently of the stats overlay, because "am I muted?" is not a statistic and the overlay is exactly what a player turns off — and on touch, tapping the badge is the way back. Muting is local and instant: the host is never asked and never told. It is per session and deliberately not remembered, so a mute you made three days ago never follows you into a call.
|
||||
|
||||
- **Echo cancellation, and a switch for it.** If you play through speakers, the host used to hear its own game audio coming back through your microphone. Every client now runs the microphone through the operating system's echo canceller by default — on Mac and iPhone that meant putting playback and capture on one engine so the voice processor can see both, and on Android it meant opening the microphone in the mode that keeps the phone's hardware canceller switched on. There is an **Echo cancellation** toggle beside the microphone setting on every platform if you want it off. A new [Why do I hear myself](https://docs.punktfunk.unom.io/docs/echo) page walks through the four different ways an echo loop can form and which switch stops each one.
|
||||
|
||||
- **Android streams reach the glass sooner.** The app used to hand each decoded frame to the screen the instant it appeared, so every frame inherited whatever jitter the network and decoder had just added, and bursts piled up behind the display. Frames are now scheduled onto the panel's own refresh. Measured end to end on a 120 Hz phone under game load: 43.7 ms down to 26.4 ms; on a second phone against a Linux host, 22–24 ms down to 14–18 ms. Two ingredients were plain bugs — Android quietly down-rates a game's frame callbacks to 60 Hz and then *reports* 60 Hz as the panel's rate, so the app was pacing a 120 fps session at half speed, and the panel was not being pinned to the stream's refresh at all. Under **Decoding** you can pick **Lowest latency** (the default) or **Smoothness**, matching the iPhone and Apple TV apps.
|
||||
|
||||
- **The host can line its capture up with your screen.** A host's capture and your device's display are two independent clocks, so the wait between "captured" and "shown" slowly sweeps across a whole refresh — a large part of residual judder, and something no client can fix alone. Your device can now report the phase of its own display schedule and the host walks its capture timing until the two agree. It engages only when the reports are steady enough to trust, which in practice means a clean or wired link; over noisy Wi-Fi it stays out of the way rather than chasing noise. Android, the Apple apps and the Windows and Linux desktop clients all report it.
|
||||
|
||||
- **A frame starts decoding before it has finished arriving.** Video is cut into slices during encoding, and those slices now travel as they are produced instead of waiting for the whole picture, so a decoder can work on the front of a frame while its tail is still on the wire. It is on by default between hosts and clients that both understand it; devices that can only handle whole frames keep receiving exactly what they did before.
|
||||
|
||||
- **HDR and full chroma stop being mutually exclusive on Windows.** An HDR desktop used to cost you 4:4:4: the only 10-bit capture format available threw away colour detail, *after* the host had already told the client it was sending full chroma. Both now work together on HDR sessions. Separately, full chroma can now stay on your graphics card's decoder instead of falling back to software, and the stats overlay tells you the truth about what you got — it prints the encoder's actual target next to the measured rate (`19.4 Mb/s · target 20 Mb/s (auto)`) and shows `4:4:4→4:2:0` when the host declined. That missing number is the reason the Linux settings bug below survived four releases: measured throughput alone cannot tell "the encoder is capped at 20" from "my 200 Mb/s allowance met a cheap scene".
|
||||
|
||||
- **Linux hosts stream HDR from a gamescope session by default.** If your host runs its games through gamescope — a Bazzite or SteamOS machine in Game Mode, a handheld, an HTPC — a 10-bit HDR session used to need a build of gamescope most people did not have *and* a setting nobody knew to turn on. So capable machines quietly streamed SDR, and an RX 5700 reported "10-bit HEVC unsupported" when the GPU was never the problem. HDR is attempted by default now, and the Linux install routes carry the gamescope build that can actually capture it: the Bazzite and Arch system extensions, the NixOS module, and a new script that builds it on a Steam Deck's own on-device install. A host still running stock gamescope keeps exactly the 8-bit behaviour it has today.
|
||||
|
||||
- **The web console follows the host instead of interrogating it.** It used to poll ten endpoints on timers, so a change could be five seconds stale and two pages could disagree while you looked at them — and the Library page did not poll at all, so a game you installed in Steam never appeared until you reloaded. The console now subscribes to the host's live event stream and refreshes exactly what each event affects. Three things come with it: a **Recent activity** feed on the dashboard, so a client that connected and left while you were on another page leaves a trace; a **Connect a device** card with the host's address and a copyable link that opens an installed client straight onto this host; and an **Automation** page for the start/stop hooks the host has always run and the console never showed. The console also installs to a phone's home screen, and warns you when another Moonlight-compatible server such as Sunshine or Apollo is running on the same machine — the most common reason a host looks fine but no client can reach it.
|
||||
|
||||
- **A DualSense or DualShock 4 plugged into an Android phone behaves like a real one.** Over USB the app now talks to the pad directly: rumble, adaptive trigger effects, the lightbar and player LEDs, the gyro and the touchpad, on the DualSense, DualSense Edge and DualShock 4. Android offers no way to do any of this over Bluetooth, so USB is the route — plug the pad in and the app asks for permission the moment it appears rather than hiding the switch in Settings.
|
||||
|
||||
- **Updating the host is one click.** The Updates card grows an **Update now** button. On Windows it asks for your console password again (a remembered login should not be able to restart your PC), downloads the installer, checks it against the signed release manifest *and* its code signature, and runs it silently — the service restarts and the page reconnects by itself. If a freshly installed version ever crash-loops, the host puts the previous installer back on its own and tells you. On Linux the button is opt-in: `sudo usermod -aG punktfunk-update $USER` is the entire grant, and all it can ever do is run your own package manager's normal update for the Punktfunk packages from repositories you already trust. Fedora Silverblue-style installs stage the update and ask you to reboot; Arch additionally needs you to accept a full `pacman -Syu`, because a partial upgrade is how Arch boxes break. A Steam Deck on-device install gets the button with no opt-in at all. `PUNKTFUNK_UPDATE_APPLY=0` removes it entirely. Details on the [Updating the Host](https://docs.punktfunk.unom.io/docs/updating) page.
|
||||
|
||||
- **The Linux app can tell you it is out of date, and update itself.** `punktfunk-client --check-update` verifies the same signed release information the host does and reports what can be done on *your* kind of install; `--apply-update` does it. Where nothing on the box can install an update — a system extension, a Nix profile, a source build — it says so and gives you the command rather than a button that could only fail, and a check that *failed* reads as a failure rather than "up to date". The Steam Deck plugin now follows the client you actually launch, instead of offering updates only for the Flatpak and leaving every other kind silently stale.
|
||||
|
||||
- **A dead tray icon on Windows is one command away.** `punktfunk-host tray start`, `stop` and `status` — previously the icon only ever appeared at sign-in, so anything that killed it left you without one until you signed out and back in.
|
||||
|
||||
- **"Capture system shortcuts" finally decides where Alt+Tab goes.** The switch has been in settings for a while and nothing read it: Windows always grabbed the keyboard whether you asked or not, Linux never grabbed at all. Both honour it now, on Wayland and X11 as well. It is on by default, so **on Linux the stream now keeps chords like Alt+Tab and Super by default** — release input capture, or turn the switch off, to hand them back.
|
||||
|
||||
- **Host cards show which system a machine actually runs.** Bazzite, CachyOS and Nobara used to collapse onto their parent distribution's logo and Windows machines wore the Windows 8 flag; all have their own mark now. On Android every non-square logo was being stretched sideways — fixed too.
|
||||
|
||||
- **An Apple Pencil near the screen keeps its full sampling rate.** Drawing over a 60 fps stream on a 120 Hz iPad silently halved how often the Pencil was sampled, which is exactly the workload that feels it. The panel is lifted to its top rate whenever the Pencil is in range and drops back afterwards.
|
||||
|
||||
## Improved
|
||||
|
||||
- **The microphone is about half as slow, and heals itself.** The uplink used to send 20 ms stereo packets built out of buffers up to 42 ms long, and duplicated a mono voice into two channels for no reason. Every client now sends 10 ms mono packets with forward error correction, so a lost packet's audio is reconstructed from the next one instead of leaving a hole. The queues behind it were worse than the format: on some paths a single stall could park over a second of standing delay that never drained, and now a backlog is dropped rather than played out late. On the host side the microphone gets a proper jitter buffer that sizes itself to the client it actually has, between 10 and 60 ms, instead of the fixed worst-case buffer everyone used to pay for. The stats overlay's Detailed tier shows whether your microphone frames are arriving.
|
||||
|
||||
- **Windows hosts encode with less delay by default.** Reading each slice out of the encoder while the frame is still being encoded — which Linux hosts have done for a while — now switches itself on wherever the graphics card reports support, instead of needing an environment variable.
|
||||
|
||||
- **Android picks AV1 when the silicon prefers it.** Under **Automatic**, a device that hardware-decodes AV1 and cannot use slice-progressive delivery now asks for AV1, measured about 1.2 ms faster end to end than HEVC in an identical A/B. An explicit choice still wins, and the host only honours the preference within what both ends actually support.
|
||||
|
||||
- **The picture quality control notices when the decoder is drowning.** If your device keeps asking for a fresh keyframe, that is the decoder saying it cannot keep up — but with nothing lost on the network the bitrate used to hold steady while the picture kept breaking up. Repeated requests now back it off.
|
||||
|
||||
- **The console shows numbers it was already being told.** How long a session took to start, what a mid-stream resolution change cost, and packet loss and error-correction recovery *while the capture runs* rather than only in a saved recording. There is also a card that names each plugin which has added games to your library, counts them, and can remove them in one go — previously a plugin's entries were stuck there once it was gone.
|
||||
|
||||
- **A controller no longer disappears over a brief hiccup.** A momentary gap in controller state used to tear the virtual pad down and build a new one, which Windows treats as a full unplug and replug. It waits a fraction of a second before believing it now.
|
||||
|
||||
- **Windows logs explain a stutter instead of guessing.** When frames stop arriving the host now says whether the *game* went quiet (a menu, a loading screen, an ordinary hitch) or the display path did, rather than blaming the display path by default — and a long degraded stretch is summarised in one line at the end instead of falling silent exactly while you suffer.
|
||||
|
||||
- **The documentation caught up with five releases.** Around 280 corrections and nine new pages, including what works on which platform, mouse/touch/pen input and the in-stream chords, client settings, profiles and links, the game library, clipboard, Wake-on-LAN, HDR and uninstalling. The five-minute quickstart could not actually work as written; it can now. Debian is no longer claimed as supported, because it was never tested and its system libraries are too old.
|
||||
|
||||
## Fixed
|
||||
|
||||
- **The Linux app ignored every setting you chose.** Since 0.22.0 the desktop app handed the stream a set of defaults instead of your settings, so bitrate, resolution, refresh rate, render scale, codec, decoder, HDR, full chroma, audio channels, microphone, touch and mouse mode, scroll direction, the stats overlay and the controller type all did nothing — the report that started this was "my bitrate seems to be stuck at 20 Mbps", which is what the host falls back to when the app sends nothing. Profiles never applied at all. The GPU and audio-device pickers kept working the whole time, which is what made it so confusing. Everything is read again, and connecting from a script or the Steam Deck plugin with `--profile` now actually selects that profile.
|
||||
|
||||
- **Full chroma (4:4:4) did nothing on Linux or Windows.** It was announced in 0.22.0 and only the Apple apps ever asked the host for it. Both desktop apps ask now.
|
||||
|
||||
- **Settings saves stopped reverting each other.** The settings file has several writers and no merge, so saving from one screen silently undid what another had stored — most visibly the window size the app itself had just remembered. Every whole-file save now re-reads the file first. Windows also gained the speaker and microphone pickers it never had, its decoder list no longer offers a macOS-only option while hiding the actual Windows one, and its controller picker learned Steam Deck.
|
||||
|
||||
- **A host that was reinstalled locked your client out forever.** Regenerate a host's identity — a reinstall, a wiped configuration — and the desktop clients refused it permanently with "Host identity rejected", *including* right after a successful re-pair, showing two cards for one address with no way to tell which to forget. Pairing was appending a second record and the lookup kept returning the dead one. Trust decisions now retire the stale record and carry your settings across, so a reinstall no longer costs you what you had configured.
|
||||
|
||||
- **The bottom of the picture stopped smearing.** On some 1080p streams the last few rows repeated the final row of pixels and the whole image looked slightly stretched. Video is decoded into a buffer taller than the picture itself — 1088 rows for a 1080-row frame — and the colour conversion was sampling that padding into view, squashing the picture by 0.7%.
|
||||
|
||||
- **HDR stopped leaking out of the stream and blowing out the interface.** Connect to an HDR host, disconnect, and the gamepad interface came back overblown with wrong colours until you restarted. The interface is drawn into the same surface the stream used, and nothing ever switched that surface back out of HDR when the stream ended.
|
||||
|
||||
- **A Steam Deck streaming to a Windows PC had a stuck stick and a stuck d-pad.** Every virtual controller was presented to Windows as a DualSense regardless of what it was, so a Deck's button data was read as a DualSense's — a stick jammed hard left and a d-pad permanently pressed up. Controllers enumerate as what they are now; DualShock 4 and DualSense Edge pads were affected the same way.
|
||||
|
||||
- **Chromecast with Google TV 4K stopped crashing.** Since 0.17.0, streaming to one froze on the first frame and usually crashed and rebooted the device — with Moonlight too, not just Punktfunk. Linux hosts had started cutting every frame into four slices, which that class of TV decoder cannot handle. The host now asks what the client can take.
|
||||
|
||||
- **A muted microphone is a pause, not a hole.** On Android the frame counter kept running while muted, so the first frame after unmuting looked to the host like lost audio and it played up to five frames of stale voice trying to cover the gap. Quick mute-unmute toggles were the visible case.
|
||||
|
||||
- **A Windows host could feed your own voice back into the stream.** On a machine with VoiceMeeter, the host could pick two strips of the same virtual mixer as the microphone and the loopback — a digital feedback loop with no acoustic path to break. Virtual mixer outputs are excluded from loopback now.
|
||||
|
||||
- **Installing a plugin no longer fails on some machines.** If any folder above the plugins directory happened to contain a `package.json` — entirely normal in a home directory — the install landed there instead, and the console reported the plugin as "not present after install" or failed outright. A newly installed plugin also shows up in the sidebar on its own now, in about seven seconds rather than whenever you next reloaded.
|
||||
|
||||
- **The console's Logs page came back after a host restart.** It died permanently every time the host restarted — which the console's own update flow does — showing stale lines with no error and no way out short of a full reload. Following also stopped following once the log got busy enough to matter, and pausing did not actually pause.
|
||||
|
||||
- **One wrong password stopped locking out the whole console.** The login throttle was documented as per-address and was not: every attempt from anywhere was charged to one shared bucket, so five wrong guesses from any machine on your network locked out everybody, including you, and including the update button. Logging out now genuinely revokes the session too — previously it only deleted the browser's copy of the cookie, and a captured one stayed valid for its full seven days, surviving even a restart.
|
||||
|
||||
- **The console's charts stopped lying about time.** Every sample was drawn as an evenly spaced step, so a capture that idled for two minutes rendered that gap as a single notch — in the one view you open specifically to find where the time went. They use a real time axis now, and no longer join two separate sessions into one continuous line.
|
||||
|
||||
- **A fresh Linux install could not start the host at all.** `systemctl --user enable --now punktfunk-host` died with "Failed to load environment files", because the service demanded a configuration file that no package creates and only the docs told you to copy. It is optional now — without it you get the defaults, which is what it always should have meant.
|
||||
|
||||
- **A Linux host with no configuration file served a test pattern.** Moonlight and other GameStream clients connecting to a packaged install that had never been configured got synthetic test video instead of the screen.
|
||||
|
||||
- **A brand-new host stopped looking broken.** The Updates card reported "feed returned HTTP 404" on the stable channel, which is simply what an empty channel answers before anyone has published to it. It says so plainly now instead of showing a failure banner.
|
||||
|
||||
- **Streaming from a Steam Deck-style session no longer starts a login storm.** On some images the login manager tried to restart the gaming session about three times a second for the entire stream — over eight minutes one test box logged 1481 login sessions, the journal overflowed, and a 5070 Ti "could not hold 240 fps".
|
||||
|
||||
- **The web console on Windows stops going quiet.** It is started and watched by the host service itself now, restarted whenever it exits with a widening delay, and its output finally lands in a log file. Three separate outages in one week were the same missing supervision. A separate defect let a half-finished console deployment report success while every page served an error under a 200 status.
|
||||
|
||||
- **Updating no longer kills the tray icon for good**, and the script and plugin runner on Windows can start at all — it could not read its own program file, and the error that said so was thrown away.
|
||||
|
||||
- **A cold-start connect on Android no longer loses HDR.** Opening a link from a cold start could reach the connect before the app was attached to a display, and the checks silently fell back to their worst answers: SDR, and 1080p60.
|
||||
|
||||
- **Drawing no longer loses parts of a stroke.** A stroke held still for more than a fifth of a second while the app was busy could be dropped by the host's safety timer, and a fast burst of samples was truncated rather than split — losing exactly the geometry that matters most, at exactly the moment the app hitched. Fixed on iPad and Android.
|
||||
|
||||
- **Rumble survives a game that hammers it.** A game driving a virtual DualSense's outputs at over 2000 messages a second overflowed a queue and silenced rumble for the whole storm, while filling the log with about 230 warnings a second.
|
||||
|
||||
- **A game link on Windows launches its game.** A `punktfunk://` link carrying a game opened a plain desktop session instead — the id was parsed, validated, planned, and dropped at the last step. A link to a host you have not paired with now runs the pairing ceremony instead of refusing outright.
|
||||
|
||||
- **Android tablets and foldables stop being forced into landscape**, and the stream handles camera cutouts explicitly so the notch area behaves the same on every Android version.
|
||||
|
||||
- **A PC whose only screen is a switched-off TV stops warning on every disconnect**, and **the Apple and Android apps credit their icons again** — the bundled third-party notices had drifted behind the project's own.
|
||||
|
||||
## Under the hood (for developers)
|
||||
|
||||
- **Versions.** Wire protocol 2, virtual-display driver protocol 6 and the Windows virtual-gamepad channel 3 are unchanged from 0.22.x. **C ABI 13 → 14**: the client surface grew `punktfunk_connection_report_phase` and the `PUNKTFUNK_CLIENT_CAP_PHASE_LOCK` mirror constant. Additive and capability-gated — an embedder that never calls the new function needs no change — and the wire grows only a control message an old host never reads plus a strict-prefix append on the host-timing tail, which is why `WIRE_VERSION` stays 2.
|
||||
- **Slice-streamed access units.** `USER_FLAG_SLICE_STREAM` marks frames whose FEC blocks are cut at slice boundaries rather than at a fixed byte count, so blocks are variable-size and can no longer be addressed by the uniform `block_index * K_max` formula: sentinel blocks reuse `frame_bytes` as a shard-aligned base offset, the final block's base derives from the totals, and the receiver pins positionally (sentinels must sit strictly below the final block's base; a mixed-flag or out-of-range header is dropped before it can place a byte). The host sets the flag only toward clients advertising `STREAMED_AU` + `VIDEO_CAP_MULTI_SLICE`; `PUNKTFUNK_SLICE_STREAM=0` pins legacy for an A/B.
|
||||
- **Partial delivery.** `connect`'s `frame_parts` opt-in yields each AU's newly-contiguous prefix as `Frame::part` pieces (offset tiling, first/last marked, the completing push carrying only the suffix). The reassembler walks a per-frame cursor over successfully-completed blocks, coalesces out-of-order completions, keeps probe filler whole and stops short of the final block so the zero-padded tail still trims. Per-AU accounting keeps its units — OWD/ABR feeds, the inter-arrival series and the jump-to-live detector count only completing deliveries, and `FrameChannel::depth()` counts AUs. Never flows without the opt-in, never on PyroWave. Consumer contract: a gap or orphan part means the AU is lost — abandon, flush, resync on the next `first`. Field note: on the NP3 all three `c2.qti` low-latency decoders refuse `FEATURE_PartialFrame`, and forcing it through `debug.punktfunk.force_parts` showed they accept the pieces but only assemble them — codec time unchanged, so the overlap is dead on SM8735 either way.
|
||||
- **Multi-slice gating and sub-frame readback.** `VIDEO_CAP_MULTI_SLICE` is `0x80`, the video-caps byte's last free bit; the next cap needs a second byte and an ABI bump. GameStream reads `x-nv-video[0].videoEncoderSlicesPerFrame` (absent or out of range ⇒ 1). The Windows direct-NVENC session now seeds `resolve_subframe` from `NV_ENC_CAPS_SUPPORT_SUBFRAME_READBACK` instead of a hard false, so `PUNKTFUNK_NVENC_SUBFRAME` is the same tri-state escape it is on Linux. The submit-time IDR hint gained the Linux twin's `opening` term — NVENC emits the session-opening frame as an IDR regardless of pic flags, which was firing the divergence check at every stream start.
|
||||
- **Phase-locked capture.** `PhaseReport` (control `0x32`) carries the client's next display latch already converted to host clock, the panel period, an uncertainty and a circular arrival-lead statistic with a coherence permille; the `0xCF` host-timing tail grows a 29-byte `applied_phase_ns` ACK under the existing strict-prefix append discipline. The controller locks submits to an absolute grid (`epoch + k × period + offset`) and walks only the offset, so phase actuation is linear by construction and a saturating plant disengages rather than free-running or parking a hold. Coherence floor 300‰, antipode damping within 1 ms of ±period/2, decay-to-zero on incoherence or travel-budget exhaustion. `PUNKTFUNK_PHASE_LOCK=0` disarms. The desktop clients have the best latch signal of the three — true on-glass stamps via `VK_KHR_present_wait` — and advertise the cap only when present timing is real. `punktfunk_core::phase::circular_latch` is shared by every reporter and by a closed-loop simulation harness that turns both on-glass failures (the median-driven orbit and the parked-hold e2e tax) into CI regressions.
|
||||
- **Android presenter.** An `AChoreographer` thread publishes the vsync grid and frame timelines; releases go through `releaseOutputBufferAtTime` against the timeline's expected present minus a latch margin that starts at 0 and widens by 500 µs per 1 Hz window only when the paced counter shows real misses, one-way per stream. The panel period is learned downward from observed timeline spacing because `Display.getRefreshRate` reports the frame-rate-category override rather than the panel. `debug.punktfunk.latch_margin_us` (0..=8000) and `debug.punktfunk.presenter=arrival` pin the margin and restore the legacy path without a rebuild. `nativeVideoStats` is now 33 doubles — the decode stage splits into feed (received→queued) and codec (queued→decoded) at 30/31, with a skipped counter at 32 that separates benign newest-wins pacing from parked-AU overflow; 0–25 remain frozen and the JNI KDoc is the authoritative index list.
|
||||
- **Microphone path.** Uplink is 10 ms mono Opus at 48 kbps with in-band FEC against 10% assumed loss, on every client; the wire is unchanged (the host decodes any Opus frame ≤ 120 ms and its stereo decoder upmixes mono). `MIC_QUEUE` drops 64 → 12 and sheds oldest-first past a ~60 ms backlog — the old constant claimed 320 ms of 5 ms frames but the frames were 20 ms, so it really allowed 1.28 s of standing delay that only a >600 ms silence gap ever flushed. `MicFrame {seq, pts_ns, opus}` now survives ingest, feeding a real host-side de-jitter (`audio/mic_jitter.rs`): two-frame reorder window, libopus concealment up to 5 frames, adaptive target depth from inter-arrival jitter clamped to 10–60 ms, with `PUNKTFUNK_MIC_LEGACY_BUFFER=1` as a one-release escape hatch and a 30 s "mic uplink health" line. Sequence deliberately freezes while muted on every client — a pause is not loss and must not be concealed or counted as a gap. `Settings::echo_cancel` (default on, `#[serde(default)]`) gates what `PUNKTFUNK_NO_AEC` gated; the env var still wins one-way and can only turn AEC off.
|
||||
- **HDR × 4:4:4 on Windows.** `HdrRgb10Converter` is one full-res pass from the FP16 scRGB desktop to packed `R10G10B10A2` in BT.2020 PQ, reusing the P010 shader's `scrgb_to_pq2020` verbatim so both HDR outputs share bit-identical colour math — it simply stops before the RGB→YUV matrix, the studio-range squeeze and the chroma decimation, and NVENC does the CSC to YUV 4:4:4 itself under FREXT. No swizzle is involved despite appearances: NVENC names packed formats MSB-down, so its `ABGR10` is bit-identical to DXGI `R10G10B10A2_UNORM`. `capturer_supports_444` no longer has a depth it cannot serve, so the chroma resolved before the Welcome is the chroma the wire carries. AV1 is deliberately untouched — Range Extensions are HEVC-only. On the client side `vkframe_plane_views` accepts the 2-plane 4:4:4 pool formats (8-bit and the 10-bit 3PACK16 sibling); 3-plane 4:4:4 stays rejected and demotes cleanly. No capability probe gates `VIDEO_CAP_444` on purpose — software decode is the guaranteed display floor, and a probe-gated bit would go inert on exactly the boxes relying on the fallback.
|
||||
- **Vulkan decode crop.** The CSC pass sampled decoded planes with the fullscreen triangle's 0..1 UVs while its render target is the *cropped* size. FFmpeg sizes the decode pool from `avctx->coded_*` and H.264 codes `16 * mb_height`, so a 1080-row picture decodes into a 1088-row pool and destination row 1079 sampled source row ~1087.5. `VkVideoFrame` carries the pool extent now and `record_csc` takes a `uv_scale`. Only the Vulkan-Video path passes a scale below 1.0; every other call site passes `[1.0, 1.0]` and is inert.
|
||||
- **Console architecture.** The event subscription needed four separate defects fixed before it could work, each found by measuring: Nitro's `localFetch` accumulates the whole response so nothing streams through the deployed Bun server (SSE gets its own route handing back a web `Response` wrapping the upstream stream); hydration mounts the shell and discards it ~15 ms later, so the subscription is a refcounted module singleton with a grace period; `getRouter()` runs more than once in the browser and each call built its own QueryClient; and `invalidateQueries` only refetches queries with a live observer. Events never carry data into the cache — they only mark staleness — so an unknown future kind costs nothing and a missed event degrades to the slow safety-net polling underneath.
|
||||
- **Console security.** The login throttle's per-IP bucket was always `undefined` because Nitro's synthetic request has no `remoteAddress`; the Bun entry stamps the real peer into a header (deleting any client-supplied copy first). Session cookies carry an epoch persisted next to the host config, so logout revokes across a restart — it was a module-level counter that reset with the process. Installing an unreviewed package or adding a catalog source now re-ask for the console password, the plugin-UI response filter became an allowlist (`Clear-Site-Data` from a plugin error page could sign the operator out on our own origin), the ui-credential denylist matches the normalised path, and a half-configured TLS setup refuses to start rather than serving with the login password in the clear. `@unom/ui/button` reached two game-UI sprite sheets through `sound/defaults.js` — a 4.8 MB `.wav` and a 2.2 MB `.mp3` that rode into the Windows installer and the .deb and could never be played; a build-time rewrite takes the asset payload from 8.2 MB to 1.5 MB.
|
||||
- **Known-hosts store.** `upsert` matches on fingerprint, which is what lets a host that moved address keep its record; a host that changed *identity* matched nothing and appended a second record, and `find_by_addr` returned whichever came first. Trust decisions funnel through `upsert_trusted`, which retires — deletes, not demotes — any other record for that address, carrying the box-describing fields across (MAC, OS chain, bound profile, pinned cards, last used) while `paired` and `clipboard_sync` are re-decided, being decisions about one specific certificate. Only trust decisions may retire; the wake path's address re-key and every learn-from-advert path stay on plain `upsert`, since letting an unauthenticated mDNS advert delete a saved host by claiming its address trades this bug for a much worse one. `find_by_addr` stops being positional: a real fingerprint beats a placeholder, newest trust decision wins among real ones.
|
||||
- **Environment.** `PUNKTFUNK_GAMESCOPE_HDR` defaults **on** (explicit-off grammar); the NixOS module's `gamescopeHdr` only controls whether the patched binary is on `PATH`. `PUNKTFUNK_VIDEO_SOURCE` defaults to `virtual` — unset previously fell through to the synthetic test pattern, reachable once `host.env` became optional. `PUNKTFUNK_SECURE_DDA` is read by nothing and is no longer written into a fresh `host.env`; `PUNKTFUNK_INPUT_BACKEND` documented a `uinput` value that does not exist and omitted `kwin`. `PFVD_NO_RT_GPU` makes the virtual display's realtime GPU priority A/B-able without a rebuild. `PUNKTFUNK_STALL_PROBES=0` opts a box out of the capture micro-probe engine.
|
||||
@@ -52,7 +52,13 @@
|
||||
// v13: added `punktfunk_connection_send_pen` — the stylus wire plane
|
||||
// (design/pen-tablet-input.md): a client sends `RICH_PEN` sample batches once the host
|
||||
// advertises `HOST_CAP_PEN`. Additive and capability-gated, so [`WIRE_VERSION`] is unchanged.
|
||||
#define ABI_VERSION 13
|
||||
// v14: added `punktfunk_connection_report_phase` + the `PUNKTFUNK_CLIENT_CAP_PHASE_LOCK` mirror
|
||||
// — the phase-locked capture reporter (design/phase-locked-capture.md): a client that advertises
|
||||
// the cap reports its next display latch (already converted to host clock), the panel period, an
|
||||
// uncertainty and the circular arrival-lead statistic the host's controller steers on. Additive;
|
||||
// the wire grows only a new control message (`PhaseReport`, 0x32) an old host never reads and a
|
||||
// strict-prefix append on the 0xCF host-timing tail, so [`WIRE_VERSION`] is unchanged.
|
||||
#define ABI_VERSION 14
|
||||
|
||||
// The punktfunk/1 **wire** version — what `Hello`/`Welcome` carry and hosts equality-check.
|
||||
// Deliberately its own constant: [`ABI_VERSION`] tracks the embeddable **C surface**
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
into the self-contained .output (no node_modules, nothing for bun to fail to resolve). The
|
||||
console runs as a supervised child of the PunktfunkHost service (bun {app}\web\.output\server\
|
||||
index.mjs on :47992), so the swap is: stop the service (its kill-on-close job takes the console's
|
||||
bun down and unlocks the files), replace {app}\web\.output, start the service — the supervisor
|
||||
bun down and unlocks the files), replace {app}\web\.output, start the service - the supervisor
|
||||
brings the console back by itself. Needs an elevated shell (service control + Program Files).
|
||||
#>
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
@@ -543,6 +543,8 @@
|
||||
"update_checking": "Prüfe…",
|
||||
"update_last_checked": "Zuletzt geprüft",
|
||||
"update_never_checked": "Noch nicht geprüft",
|
||||
"update_none_published": "Noch keine veröffentlicht",
|
||||
"update_not_published": "Auf dem Kanal {channel} wurde noch nichts veröffentlicht. Die Prüfung funktioniert — es gibt nur noch keine Version zum Vergleichen.",
|
||||
"update_stale": "Der Update-Feed hat sich seit über 45 Tagen nicht geändert — Prüfungen gelingen, aber es kommt nichts Neues an. Falls das nicht stimmen kann, prüfe die Verbindung dieses Hosts zu git.unom.io.",
|
||||
"update_disabled": "Update-Prüfungen sind auf diesem Host deaktiviert (PUNKTFUNK_UPDATE_CHECK=0).",
|
||||
"update_error": "Letzte Prüfung fehlgeschlagen:",
|
||||
|
||||
@@ -543,6 +543,8 @@
|
||||
"update_checking": "Checking…",
|
||||
"update_last_checked": "Last checked",
|
||||
"update_never_checked": "Not checked yet",
|
||||
"update_none_published": "None published yet",
|
||||
"update_not_published": "Nothing has been published to the {channel} channel yet. The check itself is working — there's just no release to compare against.",
|
||||
"update_stale": "The update feed hasn't changed in over 45 days — checks succeed but nothing new arrives. If that seems wrong, check this host's connectivity to git.unom.io.",
|
||||
"update_disabled": "Update checks are disabled on this host (PUNKTFUNK_UPDATE_CHECK=0).",
|
||||
"update_error": "Last check failed:",
|
||||
|
||||
@@ -162,7 +162,9 @@ export const UpdateCard: FC<{
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-sm text-muted-foreground">
|
||||
{m.update_never_checked()}
|
||||
{s.not_published
|
||||
? m.update_none_published()
|
||||
: m.update_never_checked()}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
@@ -211,6 +213,17 @@ export const UpdateCard: FC<{
|
||||
{m.update_stale()}
|
||||
</p>
|
||||
)}
|
||||
{/* An empty channel is a normal state, not a fault: it looks like a
|
||||
404 down at the transport, but it means "nobody has announced a
|
||||
release here yet". Rendering it in the failure style told
|
||||
operators their host was broken when nothing was. The host keeps
|
||||
the two apart (`not_published` is never set alongside
|
||||
`last_error`), so this stays a straight either/or. */}
|
||||
{!inFlight && s.not_published && (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{m.update_not_published({ channel: s.channel })}
|
||||
</p>
|
||||
)}
|
||||
{!inFlight && s.last_error && (
|
||||
<p className="text-sm text-destructive">
|
||||
{m.update_error()} {s.last_error}
|
||||
|
||||
Reference in New Issue
Block a user