unom/punktfunk
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 18 Wiki Activity

feat(clients): codec preference on Windows/Apple/Android clients (Phase 2b)

Browse Source 
Rounds out codec negotiation across the last three clients — each advertises what it can decode,
builds its decoder from the resolved Welcome.codec, and exposes a "Video codec" preference picker.

**Windows** (Rust, mirrors Linux): `decodable_codecs()` + `ffmpeg_codec_id()`; the D3D11VA and
software FFmpeg decoders (and the mid-session D3D11VA→software demotion) open the negotiated codec
instead of hardcoding HEVC; settings gain a `codec` field + reactor ComboBox; `--codec` CLI flag.

**Apple** (Swift/C-ABI): AnnexB is now codec-aware — a `VideoCodec` enum drives H.264 vs HEVC NAL
parsing / parameter-set extraction (`CMVideoFormatDescriptionCreateFromH264ParameterSets` for H.264,
no VPS) and AVCC repacking; `PunktfunkConnection` advertises H264|HEVC via `punktfunk_connect_ex7`,
reads `resolvedCodec` (`punktfunk_connection_codec`), and threads `videoCodec` into the stage-1/2
pipelines + `VideoDecoder`; SettingsView "Video codec" Picker (auto/HEVC/H.264). AV1 is left out
(hosts don't emit it on the native path, and it's not an AnnexB codec). Test call sites updated.

**Android** (Kotlin + Rust JNI): the JNI `nativeConnect` gains `preferredCodec`; the native decode
loop picks the AMediaCodec MIME (`video/hevc`|`video/avc`) from `connector.codec` and advertises
H264|HEVC; Settings `codec` field + Compose dropdown.

Core/host/probe/Linux clippy + tests green (unchanged from 2a). Windows/Apple/Android compile on
their platform CI (this Linux box can't build them — Windows toolchain / Xcode / the Android NDK's
opus-cmake toolchain). All follow the Linux client's validated pattern.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Enrico Bühler
2026-07-02 00:29:38 +00:00
parent 6e41356919
commit b357934cb1
24 changed files with 328 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
clients/apple/Tests/PunktfunkKitTests/AnnexBTests.swift
+2 -2
View File
@@ -64,7 +64,7 @@ final class AnnexBTests: XCTestCase {
au.append(n)
}
let avcc = AnnexB.avcc(from: au)
let avcc = AnnexB.avcc(from: au, codec: .hevc)
// Only the IDR survives: 4-byte BE length, then the NAL bytes.
var expected = Data([0, 0, 0, UInt8(idr.count)])
expected.append(idr)
@@ -74,6 +74,6 @@ final class AnnexBTests: XCTestCase {
func testFormatDescriptionNilWithoutParameterSets() {
let idr = nal(type: 19, payload: [0xDD])
let au = Data(start4) + idr
XCTAssertNil(AnnexB.formatDescription(fromIDR: au))
XCTAssertNil(AnnexB.formatDescription(fromIDR: au, codec: .hevc))
}
}
clients/apple/Tests/PunktfunkKitTests/RemoteFirstLightTests.swift
+2 -2
View File
@@ -138,7 +138,7 @@ final class RemoteFirstLightTests: XCTestCase {
if firstPtsNs == 0 { firstPtsNs = au.ptsNs }
lastPtsNs = au.ptsNs
if let f = AnnexB.formatDescription(fromIDR: au.data) {
if let f = AnnexB.formatDescription(fromIDR: au.data, codec: .hevc) {
format = f
if decoder == nil {
let dims = CMVideoFormatDescriptionGetDimensions(f)
@@ -155,7 +155,7 @@ final class RemoteFirstLightTests: XCTestCase {
}
}
guard let f = format, let dec = decoder,
let sample = AnnexB.sampleBuffer(au: au, format: f)
let sample = AnnexB.sampleBuffer(au: au, format: f, codec: .hevc)
else { continue }
var gotPixels = false
clients/apple/Tests/PunktfunkKitTests/Stage444Tests.swift
+1 -1
View File
@@ -30,7 +30,7 @@ final class Stage444Tests: XCTestCase {
Stage444Probe.hwDecode444_8bit, "no hardware 4:4:4 decode on this device")
let data = Data(Probe444Blobs.au444_8bit)
let format = try XCTUnwrap(
AnnexB.formatDescription(fromIDR: data), "the 4:4:4 blob must yield a format description")
AnnexB.formatDescription(fromIDR: data, codec: .hevc), "the 4:4:4 blob must yield a format description")
let au = AccessUnit(data: data, ptsNs: 7_000_000, frameIndex: 0, flags: 0)
let box = FrameBox()
clients/apple/Tests/PunktfunkKitTests/VideoToolboxRoundTripTests.swift
+4 -4
View File
@@ -28,18 +28,18 @@ final class VideoToolboxRoundTripTests: XCTestCase {
// 1) Parameter-set extraction → format description.
let rebuilt = try XCTUnwrap(
AnnexB.formatDescription(fromIDR: annexB),
AnnexB.formatDescription(fromIDR: annexB, codec: .hevc),
"in-band VPS/SPS/PPS should yield a format description")
let dims = CMVideoFormatDescriptionGetDimensions(rebuilt)
XCTAssertEqual(Int(dims.width), width)
XCTAssertEqual(Int(dims.height), height)
// 2) Annex-B → AVCC re-pack must reproduce the encoder's own sample bytes.
XCTAssertEqual(AnnexB.avcc(from: annexB), avccSample)
XCTAssertEqual(AnnexB.avcc(from: annexB, codec: .hevc), avccSample)
// 3) Sample buffer → real decoder → pixels.
let au = AccessUnit(data: annexB, ptsNs: 1_000_000, frameIndex: 0, flags: 0)
let sample = try XCTUnwrap(AnnexB.sampleBuffer(au: au, format: rebuilt))
let sample = try XCTUnwrap(AnnexB.sampleBuffer(au: au, format: rebuilt, codec: .hevc))
var session: VTDecompressionSession?
XCTAssertEqual(
@@ -72,7 +72,7 @@ final class VideoToolboxRoundTripTests: XCTestCase {
func testVideoDecoderAsyncCallbackDeliversPixels() throws {
let (formatDesc, avccSample) = try encodeOneHEVCKeyframe()
let annexB = try annexBAU(formatDesc: formatDesc, avccSample: avccSample)
let format = try XCTUnwrap(AnnexB.formatDescription(fromIDR: annexB))
let format = try XCTUnwrap(AnnexB.formatDescription(fromIDR: annexB, codec: .hevc))
let au = AccessUnit(data: annexB, ptsNs: 42_000_000, frameIndex: 0, flags: 0)
let box = FrameBox()