feat(codec): make client codec selection real — GPU-aware native advertisement

The clients' codec pickers sent a preference the host threw away:
host_wire_caps() hardcoded HEVC for every GPU backend, so the native
path never emitted H.264/AV1 regardless of what the user chose.

Host: advertise what the backend actually encodes, mirroring the
GameStream serverinfo logic — software openh264 emits H.264; probed
backends (Linux VAAPI, Windows AMF/QSV) advertise their per-GPU
CodecSupport via the new wire_mask(); NVENC keeps the
Moonlight-validated H.264|HEVC|AV1 static superset; an empty probe
falls back to the superset so auto clients still resolve HEVC. Gate
10-bit to HEVC (like the 4:4:4 gate) now that a client can steer the
codec — Main10 is the only 10-bit encode path. Fix the web-console
stats label hardcoded to "hevc" (new Codec::label(), shared with the
GameStream register_session mapping).

Android: replace the hardcoded H264|HEVC Hello advertisement with a
videoCodecs param fed by VideoDecoders.decodableCodecBits() (AV1 bit
only when a real hardware video/av01 decoder exists — the decode loop
was already mime-driven); offer AV1 in the codec picker on capable
devices.

Decky: add the missing "Video codec" dropdown (auto/hevc/h264/av1) to
the plugin settings, round-tripping the same codec key the flatpak
client reads.

Apple: unchanged by design (AnnexB.swift is NAL-only, AV1 is never
advertised); refresh the stale "hosts don't emit AV1 on the native
path yet" comments here and host-side.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

clients/android/app/src/main/kotlin/io/unom/punktfunk/HostConnect.kt

@@ -3,6 +3,7 @@ package io.unom.punktfunk
 import android.content.Context
 import io.unom.punktfunk.kit.Gamepad
 import io.unom.punktfunk.kit.NativeBridge
+import io.unom.punktfunk.kit.VideoDecoders
 import io.unom.punktfunk.kit.security.ClientIdentity
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
@@ -41,7 +42,10 @@ suspend fun connectToHost(
             host, port, w, h, hz,
             identity.certPem, identity.privateKeyPem, pinHex,
             settings.bitrateKbps, settings.compositor, gamepadPref,
-            hdrEnabled, settings.audioChannels, settings.preferredCodec(), timeoutMs,
+            hdrEnabled, settings.audioChannels,
+            // What this device can decode (H.264|HEVC always, AV1 when a real decoder exists) +
+            // the user's soft codec preference — the host resolves the emitted codec from both.
+            VideoDecoders.decodableCodecBits(), settings.preferredCodec(), timeoutMs,
             launch,
         )
     }

clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt

@@ -26,8 +26,9 @@ data class Settings(
     /** Requested audio channel count: 2 (stereo), 6 (5.1) or 8 (7.1). The host clamps to what it
      * can capture; the resolved count drives the decoder + AAudio layout. */
     val audioChannels: Int = 2,
-    /** Preferred video codec: `"auto"` (host decides), `"hevc"`, or `"h264"`. A soft preference — the
-     * host emits it when it can, else falls back. AMediaCodec decodes whichever the host resolves. */
+    /** Preferred video codec: `"auto"` (host decides), `"hevc"`, `"h264"`, or `"av1"`. A soft
+     * preference — the host emits it when it can, else falls back. AMediaCodec decodes whichever
+     * the host resolves (AV1 is only advertised/offered when the device has a real AV1 decoder). */
     val codec: String = "auto",
     val micEnabled: Boolean = false,
     /**
@@ -271,14 +272,17 @@ val AUDIO_CHANNEL_OPTIONS = listOf(
     8 to "7.1 Surround",
 )
 
-/** (stored value, label) for the preferred video codec. `"auto"` = host decides. */
+/** (stored value, label) for the preferred video codec. `"auto"` = host decides. The `"av1"` row
+ * only makes sense on a device with a real AV1 decoder — SettingsScreen filters it out otherwise. */
 val CODEC_OPTIONS = listOf(
     "auto" to "Automatic",
     "hevc" to "HEVC (H.265)",
     "h264" to "H.264 (AVC)",
+    "av1" to "AV1",
 )
 
-/** The [Settings.codec] string as a `quic::CODEC_*` preference byte (`0` = auto). H264=1, HEVC=2. */
+/** The [Settings.codec] string as a `quic::CODEC_*` preference byte (`0` = auto). H264=1, HEVC=2,
+ * AV1=4. */
 fun Settings.preferredCodec(): Int = when (codec) {
     "h264" -> 1
     "hevc" -> 2

clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt

@@ -65,6 +65,7 @@ import androidx.compose.ui.graphics.vector.ImageVector
 import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.unit.dp
 import androidx.core.content.ContextCompat
+import io.unom.punktfunk.kit.VideoDecoders
 
 /**
  * Stream settings, organised as an iOS-Settings / Android-system-settings style list of category
@@ -300,7 +301,12 @@ private fun DisplaySettings(s: Settings, update: (Settings) -> Unit, context: an
             update(s.copy(bitrateKbps = kbps))
         }
 
-        SettingDropdown(label = "Video codec", options = CODEC_OPTIONS, selected = s.codec) { c ->
+        // AV1 is only offered when the device has a real AV1 decoder (it's never advertised to the
+        // host otherwise, so preferring it would be a dead setting). A stored "av1" from a capable
+        // device stays visible so the selection is always representable.
+        val av1Capable = remember { VideoDecoders.pickDecoder("video/av01") != null }
+        val codecOptions = CODEC_OPTIONS.filter { (v, _) -> v != "av1" || av1Capable || s.codec == "av1" }
+        SettingDropdown(label = "Video codec", options = codecOptions, selected = s.codec) { c ->
             update(s.copy(codec = c))
         }
 

clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt

@@ -48,6 +48,9 @@ object NativeBridge {
         gamepadPref: Int,
         hdrEnabled: Boolean,
         audioChannels: Int,
+        /** `quic::CODEC_*` bitfield of codecs this device decodes ([VideoDecoders.decodableCodecBits]);
+         *  `0` falls back to H.264|HEVC. The host resolves the emitted codec from this ∩ its GPU. */
+        videoCodecs: Int,
         /** Preferred video codec as a `quic::CODEC_*` bit (`0` = auto). Soft — the host falls back. */
         preferredCodec: Int,
         timeoutMs: Int,

clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/VideoDecoders.kt

@@ -44,6 +44,15 @@ object VideoDecoders {
      * Pick the best decoder for [mime] (`"video/hevc"` / `"video/avc"` / `"video/av01"`), or `null`
      * to let the platform resolve its default. Enumerates once — call at stream start.
      */
+    /**
+     * The `quic::CODEC_*` bitfield of codecs this device can decode, advertised in the Hello so the
+     * host never emits a codec the decode loop can't open: H.264 (1) and HEVC (2) always (universal
+     * on Android hardware), plus AV1 (4) only when [pickDecoder] finds a real (hardware, non-blocked)
+     * `video/av01` decoder. Enumerates `MediaCodecList` — call at connect time, not per frame.
+     */
+    fun decodableCodecBits(): Int =
+        1 or 2 or (if (pickDecoder("video/av01") != null) 4 else 0)
+
     fun pickDecoder(mime: String): DecoderChoice? {
         if (mime.isEmpty()) return null
         val infos = runCatching { MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos }

clients/android/native/src/session/connect.rs

@@ -76,6 +76,7 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo
     gamepad_pref: jint,
     hdr_enabled: jboolean,
     audio_channels: jint,
+    video_codecs: jint,
     preferred_codec: jint,
     timeout_ms: jint,
     launch: JString<'local>,
@@ -142,10 +143,23 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo
         // decoder + AAudio layout (read in `crate::audio::AudioPlayback::start`). Anything else
         // normalizes to stereo here.
         punktfunk_core::audio::normalize_channels(audio_channels.clamp(0, u8::MAX as jint) as u8),
-        // Codecs this device can decode — AMediaCodec decodes both HEVC and H.264 (AV1 isn't wired;
-        // hosts don't emit it on the native path yet). The host resolves the emitted codec from these
-        // + the soft `preferred_codec` and echoes it in `connector.codec`, which drives the mime below.
-        punktfunk_core::quic::CODEC_H264 | punktfunk_core::quic::CODEC_HEVC,
+        // Codecs this device can decode, ranked on the Kotlin side (`VideoDecoders.decodableCodecBits`:
+        // H.264 + HEVC always, AV1 when a real `video/av01` decoder exists — AMediaCodec is
+        // mime-driven, see `codec_mime`). Mask to the known bits and fall back to the pre-AV1
+        // H.264|HEVC pair on 0 so a bogus value can't advertise nothing and kill the handshake.
+        // The host resolves the emitted codec from these + the soft `preferred_codec` and echoes it
+        // in `connector.codec`, which drives the mime below.
+        {
+            let bits = (video_codecs.clamp(0, u8::MAX as jint) as u8)
+                & (punktfunk_core::quic::CODEC_H264
+                    | punktfunk_core::quic::CODEC_HEVC
+                    | punktfunk_core::quic::CODEC_AV1);
+            if bits == 0 {
+                punktfunk_core::quic::CODEC_H264 | punktfunk_core::quic::CODEC_HEVC
+            } else {
+                bits
+            }
+        },
         preferred_codec.clamp(0, u8::MAX as jint) as u8,
         launch,   // a store-qualified library id to boot into a game, or None for the desktop
         pin,      // Some → Crypto on host-fp mismatch

clients/apple/Sources/PunktfunkClient/Session/SessionModel.swift

@@ -194,9 +194,10 @@ final class SessionModel: ObservableObject {
             if want444, canDecode444 {
                 videoCaps |= PunktfunkConnection.videoCap444
             }
-            // This client's VideoToolbox path decodes H.264 and HEVC (AV1 isn't wired — hosts don't
-            // emit it on the native path yet). The host resolves the emitted codec from these + the
-            // soft `preferredCodec`; `resolvedCodec` reflects what it chose.
+            // This client's VideoToolbox path decodes H.264 and HEVC (AV1 depacketization isn't
+            // wired — AnnexB.swift is NAL-only — so it must never be advertised here). The host
+            // resolves the emitted codec from these + the soft `preferredCodec`; `resolvedCodec`
+            // reflects what it chose.
             let videoCodecs = PunktfunkConnection.codecH264 | PunktfunkConnection.codecHEVC
             let result = Result { try PunktfunkConnection(
                 host: host.address, port: host.port,

clients/apple/Sources/PunktfunkClient/Settings/SettingsOptions.swift

@@ -38,8 +38,8 @@ enum SettingsOptions {
         HUDPlacement.allCases.map { ($0.label, $0.rawValue) }
 
     /// Video-codec preference (`DefaultsKey.codec`) — a soft preference the host falls back from.
-    /// No AV1: this client's VideoToolbox path decodes H.264/HEVC only (hosts don't emit AV1 on
-    /// the native path yet).
+    /// No AV1: this client's VideoToolbox path decodes H.264/HEVC only (AnnexB.swift is NAL-only),
+    /// so it never advertises AV1 — offering it here would be a dead setting.
     static let codecs: [(label: String, tag: String)] = [
         ("Automatic", "auto"),
         ("HEVC (H.265)", "hevc"),

clients/apple/Sources/PunktfunkKit/Video/AnnexB.swift

@@ -6,8 +6,8 @@
 // buffers whose NALs are 4-byte-length-prefixed. This file converts between the two, for
 // the codec the host resolved in the Welcome (`connection.videoCodec`) — HEVC and H.264
 // differ only in NAL-header layout and which parameter sets exist (HEVC adds a VPS). AV1
-// is not an Annex-B/NAL codec and isn't handled here (hosts don't emit it on the native
-// path yet).
+// is not an Annex-B/NAL codec and isn't handled here — this client never advertises it in
+// the Hello, so a host never emits it at us.
 //
 // HOT PATH: both pumps run `formatDescription(fromIDR:codec:)` + `sampleBuffer(au:format:codec:)`
 // once per AU, so the conversion is built on `forEachNAL` — a zero-copy scan over the AU's bytes

clients/decky/main.py

@@ -669,7 +669,7 @@ class Plugin:
             # The client's own defaults (native display, host-default bitrate, auto pad).
             return {
                 "width": 0, "height": 0, "refresh_hz": 0, "bitrate_kbps": 0,
-                "gamepad": "auto", "compositor": "auto",
+                "codec": "auto", "gamepad": "auto", "compositor": "auto",
                 "inhibit_shortcuts": True, "mic_enabled": False,
             }
 

clients/decky/src/backend.ts

@@ -61,13 +61,14 @@ export interface RunnerInfo {
 }
 
 // The slice of the flatpak client's settings JSON this UI surfaces. The file can hold more
-// keys (codec, decoder, … set from the desktop client's own UI) — they round-trip untouched
+// keys (decoder, … set from the desktop client's own UI) — they round-trip untouched
 // because get_settings returns the whole parsed file and patches are object spreads.
 export interface StreamSettings {
   width: number; // 0 = native
   height: number; // 0 = native
   refresh_hz: number; // 0 = native
   bitrate_kbps: number; // 0 = host default
+  codec?: string; // "auto" | "hevc" | "h264" | "av1" — soft preference (absent in pre-codec files)
   gamepad: string; // "auto" | "xbox360" | "xboxone" | "dualsense" | "dualshock4" | "steamdeck"
   compositor: string; // "auto" | "kwin" | "wlroots" | "mutter" | "gamescope"
   inhibit_shortcuts: boolean;

clients/decky/src/settings.tsx

@@ -34,6 +34,15 @@ const GAMEPAD_LABELS: Record<string, string> = {
   dualshock4: "DualShock 4",
   steamdeck: "Steam Deck",
 };
+// Mirrors the desktop client's picker (ui_settings.rs CODECS) — a soft preference the host
+// falls back from when its GPU can't encode it.
+const CODECS = ["auto", "hevc", "h264", "av1"];
+const CODEC_LABELS: Record<string, string> = {
+  auto: "Automatic",
+  hevc: "HEVC (H.265)",
+  h264: "H.264 (AVC)",
+  av1: "AV1",
+};
 const COMPOSITORS = ["auto", "kwin", "wlroots", "mutter", "gamescope"];
 const COMPOSITOR_LABELS: Record<string, string> = {
   auto: "Automatic",
@@ -108,6 +117,21 @@ export const SettingsSection: FC = () => {
         valueSuffix=" Mbit/s"
         onChange={(v) => patch({ bitrate_kbps: v * 1000 })}
       />
+      <Field
+        label="Video codec"
+        description="Preferred stream codec — the host falls back when its GPU can't encode it"
+        childrenContainerWidth="max"
+      >
+        <RowActions>
+          <div style={selectShell}>
+            <Dropdown
+              rgOptions={CODECS.map((c) => ({ data: c, label: CODEC_LABELS[c] ?? c }))}
+              selectedOption={s.codec ?? "auto"}
+              onChange={(o) => patch({ codec: o.data as string })}
+            />
+          </div>
+        </RowActions>
+      </Field>
       <Field
         label="Gamepad type"
         description="Which virtual controller the host creates for your inputs"