feat(clients): codec preference on Windows/Apple/Android clients (Phase 2b)

Rounds out codec negotiation across the last three clients — each advertises what it can decode,
builds its decoder from the resolved Welcome.codec, and exposes a "Video codec" preference picker.

**Windows** (Rust, mirrors Linux): `decodable_codecs()` + `ffmpeg_codec_id()`; the D3D11VA and
software FFmpeg decoders (and the mid-session D3D11VA→software demotion) open the negotiated codec
instead of hardcoding HEVC; settings gain a `codec` field + reactor ComboBox; `--codec` CLI flag.

**Apple** (Swift/C-ABI): AnnexB is now codec-aware — a `VideoCodec` enum drives H.264 vs HEVC NAL
parsing / parameter-set extraction (`CMVideoFormatDescriptionCreateFromH264ParameterSets` for H.264,
no VPS) and AVCC repacking; `PunktfunkConnection` advertises H264|HEVC via `punktfunk_connect_ex7`,
reads `resolvedCodec` (`punktfunk_connection_codec`), and threads `videoCodec` into the stage-1/2
pipelines + `VideoDecoder`; SettingsView "Video codec" Picker (auto/HEVC/H.264). AV1 is left out
(hosts don't emit it on the native path, and it's not an AnnexB codec). Test call sites updated.

**Android** (Kotlin + Rust JNI): the JNI `nativeConnect` gains `preferredCodec`; the native decode
loop picks the AMediaCodec MIME (`video/hevc`|`video/avc`) from `connector.codec` and advertises
H264|HEVC; Settings `codec` field + Compose dropdown.

Core/host/probe/Linux clippy + tests green (unchanged from 2a). Windows/Apple/Android compile on
their platform CI (this Linux box can't build them — Windows toolchain / Xcode / the Android NDK's
opus-cmake toolchain). All follow the Linux client's validated pattern.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-02 00:29:38 +00:00
parent 6e41356919
commit b357934cb1
24 changed files with 328 additions and 74 deletions
@@ -255,6 +255,13 @@ public final class PunktfunkConnection {
/// PCM from `nextAudioPcm` is interleaved in the canonical wire order FL FR FC LFE RL RR SL SR.
public private(set) var resolvedAudioChannels: UInt8 = 2
/// The video codec the host resolved for this session (`Welcome.codec`, `PUNKTFUNK_CODEC_*`):
/// `2` = HEVC (default / older host), `1` = H.264, `4` = AV1. Build the decoder from THIS. The
/// resolved value honors the client's `preferredCodec` when the host could emit it.
public private(set) var resolvedCodec: UInt8 = 2 // PUNKTFUNK_CODEC_HEVC
/// The resolved codec as an `AnnexB.VideoCodec` (H.264 vs HEVC) drives the NAL parsing.
public var videoCodec: VideoCodec { VideoCodec(wire: resolvedCodec) }
/// Connect and start a session at the requested mode (the host creates a native virtual
/// output at exactly this size/refresh). Blocks up to `timeoutMs`.
///
@@ -285,6 +292,8 @@ public final class PunktfunkConnection {
bitrateKbps: UInt32 = 0,
videoCaps: UInt8 = 0,
audioChannels: UInt8 = 2,
videoCodecs: UInt8 = 0x02, // PUNKTFUNK_CODEC_HEVC the codecs this client can decode
preferredCodec: UInt8 = 0, // 0 = auto; else PUNKTFUNK_CODEC_* soft preference
launchID: String? = nil,
timeoutMs: UInt32 = 10_000
) throws {
@@ -300,16 +309,18 @@ public final class PunktfunkConnection {
withOptionalCString(launchID) { launch in
if let pin = pinSHA256 {
return pin.withUnsafeBytes { p in
punktfunk_connect_ex6(
punktfunk_connect_ex7(
cs, port, width, height, refreshHz, compositor.rawValue,
gamepad.rawValue, bitrateKbps, videoCaps, audioChannels, launch,
gamepad.rawValue, bitrateKbps, videoCaps, audioChannels,
videoCodecs, preferredCodec, launch,
p.bindMemory(to: UInt8.self).baseAddress, &observed,
cert, key, timeoutMs)
}
}
return punktfunk_connect_ex6(
return punktfunk_connect_ex7(
cs, port, width, height, refreshHz, compositor.rawValue,
gamepad.rawValue, bitrateKbps, videoCaps, audioChannels, launch,
gamepad.rawValue, bitrateKbps, videoCaps, audioChannels,
videoCodecs, preferredCodec, launch,
nil, &observed, cert, key, timeoutMs)
}
}
@@ -347,6 +358,9 @@ public final class PunktfunkConnection {
var ac: UInt8 = 2
_ = punktfunk_connection_audio_channels(handle, &ac)
resolvedAudioChannels = ac
var codec: UInt8 = 2 // PUNKTFUNK_CODEC_HEVC
_ = punktfunk_connection_codec(handle, &codec)
resolvedCodec = codec
}
/// A bandwidth speed-test measurement (see `startSpeedTest`). Partial until `done`.
@@ -620,6 +634,11 @@ public final class PunktfunkConnection {
/// the host then emits 4:4:4 only if it too opted in. `chromaFormat` reflects the real value.
public static let videoCap444: UInt8 = UInt8(PUNKTFUNK_VIDEO_CAP_444)
/// Codec bits for `videoCodecs` / `preferredCodec` and the value `resolvedCodec` returns.
public static let codecH264: UInt8 = UInt8(PUNKTFUNK_CODEC_H264)
public static let codecHEVC: UInt8 = UInt8(PUNKTFUNK_CODEC_HEVC)
public static let codecAV1: UInt8 = UInt8(PUNKTFUNK_CODEC_AV1)
/// Static HDR mastering metadata (SMPTE ST.2086 + content light level) the host sent for an HDR
/// session. Mirrors the wire/ABI `PunktfunkHdrMeta`; primaries are in ST.2086 **G, B, R** order,
/// 1/50000 units; mastering luminance in 0.0001 cd/m²; MaxCLL/MaxFALL in nits.