From 3678c182d5002c5f5d7eb4f156ec111b2c4fa613 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Thu, 2 Jul 2026 00:29:38 +0000 Subject: [PATCH] feat(clients): codec preference on Windows/Apple/Android clients (Phase 2b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rounds out codec negotiation across the last three clients — each advertises what it can decode, builds its decoder from the resolved Welcome.codec, and exposes a "Video codec" preference picker. **Windows** (Rust, mirrors Linux): `decodable_codecs()` + `ffmpeg_codec_id()`; the D3D11VA and software FFmpeg decoders (and the mid-session D3D11VA→software demotion) open the negotiated codec instead of hardcoding HEVC; settings gain a `codec` field + reactor ComboBox; `--codec` CLI flag. **Apple** (Swift/C-ABI): AnnexB is now codec-aware — a `VideoCodec` enum drives H.264 vs HEVC NAL parsing / parameter-set extraction (`CMVideoFormatDescriptionCreateFromH264ParameterSets` for H.264, no VPS) and AVCC repacking; `PunktfunkConnection` advertises H264|HEVC via `punktfunk_connect_ex7`, reads `resolvedCodec` (`punktfunk_connection_codec`), and threads `videoCodec` into the stage-1/2 pipelines + `VideoDecoder`; SettingsView "Video codec" Picker (auto/HEVC/H.264). AV1 is left out (hosts don't emit it on the native path, and it's not an AnnexB codec). Test call sites updated. **Android** (Kotlin + Rust JNI): the JNI `nativeConnect` gains `preferredCodec`; the native decode loop picks the AMediaCodec MIME (`video/hevc`|`video/avc`) from `connector.codec` and advertises H264|HEVC; Settings `codec` field + Compose dropdown. Core/host/probe/Linux clippy + tests green (unchanged from 2a). Windows/Apple/Android compile on their platform CI (this Linux box can't build them — Windows toolchain / Xcode / the Android NDK's opus-cmake toolchain). All follow the Linux client's validated pattern. Co-Authored-By: Claude Opus 4.8 --- .../kotlin/io/unom/punktfunk/ConnectScreen.kt | 3 +- .../main/kotlin/io/unom/punktfunk/Settings.kt | 21 +++ .../io/unom/punktfunk/SettingsScreen.kt | 6 + .../io/unom/punktfunk/kit/NativeBridge.kt | 2 + clients/android/native/src/decode.rs | 13 +- clients/android/native/src/session.rs | 6 + .../Sources/PunktfunkClient/ContentView.swift | 12 ++ .../PunktfunkClient/SessionModel.swift | 8 +- .../PunktfunkClient/SettingsView.swift | 17 ++- .../apple/Sources/PunktfunkKit/AnnexB.swift | 122 +++++++++++++----- .../Sources/PunktfunkKit/DefaultsKeys.swift | 3 + .../PunktfunkKit/PunktfunkConnection.swift | 27 +++- .../Sources/PunktfunkKit/Stage2Pipeline.swift | 3 +- .../Sources/PunktfunkKit/StreamPump.swift | 4 +- .../Sources/PunktfunkKit/VideoDecoder.swift | 14 +- .../Tests/PunktfunkKitTests/AnnexBTests.swift | 4 +- .../RemoteFirstLightTests.swift | 4 +- .../PunktfunkKitTests/Stage444Tests.swift | 2 +- .../VideoToolboxRoundTripTests.swift | 8 +- clients/windows/src/app.rs | 25 ++++ clients/windows/src/main.rs | 7 + clients/windows/src/session.rs | 16 ++- clients/windows/src/trust.rs | 21 +++ clients/windows/src/video.rs | 54 ++++++-- 24 files changed, 328 insertions(+), 74 deletions(-) diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectScreen.kt index a380c10..ad110dd 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectScreen.kt @@ -187,7 +187,8 @@ fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) { targetHost, targetPort, w, h, hz, id.certPem, id.privateKeyPem, pinHex ?: "", settings.bitrateKbps, settings.compositor, gamepadPref, - hdrEnabled, settings.audioChannels, CONNECT_TIMEOUT_MS, + hdrEnabled, settings.audioChannels, settings.preferredCodec(), + CONNECT_TIMEOUT_MS, ) } connecting = false diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt index 01479be..1499d16 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt @@ -26,6 +26,9 @@ data class Settings( /** Requested audio channel count: 2 (stereo), 6 (5.1) or 8 (7.1). The host clamps to what it * can capture; the resolved count drives the decoder + AAudio layout. */ val audioChannels: Int = 2, + /** Preferred video codec: `"auto"` (host decides), `"hevc"`, or `"h264"`. A soft preference — the + * host emits it when it can, else falls back. AMediaCodec decodes whichever the host resolves. */ + val codec: String = "auto", val micEnabled: Boolean = false, /** Show the live stats overlay (FPS / throughput / latency) during a stream. */ val statsHudEnabled: Boolean = true, @@ -51,6 +54,7 @@ class SettingsStore(context: Context) { compositor = prefs.getInt(K_COMPOSITOR, 0), gamepad = prefs.getInt(K_GAMEPAD, 0), audioChannels = prefs.getInt(K_AUDIO_CH, 2), + codec = prefs.getString(K_CODEC, "auto") ?: "auto", micEnabled = prefs.getBoolean(K_MIC, false), statsHudEnabled = prefs.getBoolean(K_HUD, true), trackpadMode = prefs.getBoolean(K_TRACKPAD, true), @@ -66,6 +70,7 @@ class SettingsStore(context: Context) { .putInt(K_COMPOSITOR, s.compositor) .putInt(K_GAMEPAD, s.gamepad) .putInt(K_AUDIO_CH, s.audioChannels) + .putString(K_CODEC, s.codec) .putBoolean(K_MIC, s.micEnabled) .putBoolean(K_HUD, s.statsHudEnabled) .putBoolean(K_TRACKPAD, s.trackpadMode) @@ -81,6 +86,7 @@ class SettingsStore(context: Context) { const val K_COMPOSITOR = "compositor" const val K_GAMEPAD = "gamepad" const val K_AUDIO_CH = "audio_channels" + const val K_CODEC = "codec" const val K_MIC = "mic_enabled" const val K_HUD = "stats_hud_enabled" const val K_TRACKPAD = "trackpad_mode" @@ -156,6 +162,21 @@ val AUDIO_CHANNEL_OPTIONS = listOf( 8 to "7.1 Surround", ) +/** (stored value, label) for the preferred video codec. `"auto"` = host decides. */ +val CODEC_OPTIONS = listOf( + "auto" to "Automatic", + "hevc" to "HEVC (H.265)", + "h264" to "H.264 (AVC)", +) + +/** The [Settings.codec] string as a `quic::CODEC_*` preference byte (`0` = auto). H264=1, HEVC=2. */ +fun Settings.preferredCodec(): Int = when (codec) { + "h264" -> 1 + "hevc" -> 2 + "av1" -> 4 + else -> 0 +} + /** (kbps, label). `0` = host default. */ val BITRATE_OPTIONS = listOf( 0 to "Automatic", diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt index 5447fd5..f2bc874 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/SettingsScreen.kt @@ -95,6 +95,12 @@ fun SettingsScreen(initial: Settings, onChange: (Settings) -> Unit, onBack: () - selected = s.bitrateKbps, ) { kbps -> update(s.copy(bitrateKbps = kbps)) } + SettingDropdown( + label = "Video codec", + options = CODEC_OPTIONS, + selected = s.codec, + ) { c -> update(s.copy(codec = c)) } + // HDR is only meaningful on a panel that can present HDR10; on an SDR display the toggle // is disabled (and HDR is never advertised regardless) so the host doesn't send PQ the // panel would mis-tone-map. The capability is fixed for the device, so read it once. diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index 2cf3625..e4bb368 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -48,6 +48,8 @@ object NativeBridge { gamepadPref: Int, hdrEnabled: Boolean, audioChannels: Int, + /** Preferred video codec as a `quic::CODEC_*` bit (`0` = auto). Soft — the host falls back. */ + preferredCodec: Int, timeoutMs: Int, ): Long diff --git a/clients/android/native/src/decode.rs b/clients/android/native/src/decode.rs index f483956..7968ef0 100644 --- a/clients/android/native/src/decode.rs +++ b/clients/android/native/src/decode.rs @@ -27,16 +27,23 @@ pub fn run( ) { boost_thread_priority(); let mode = client.mode(); - let codec = match MediaCodec::from_decoder_type("video/hevc") { + // The MediaCodec MIME for the codec the host resolved (`Welcome.codec`): HEVC or H.264. AMediaCodec + // needs no out-of-band extradata — the in-band VPS/SPS/PPS on every IDR configure it either way. + let mime = match client.codec { + punktfunk_core::quic::CODEC_H264 => "video/avc", + _ => "video/hevc", + }; + let codec = match MediaCodec::from_decoder_type(mime) { Some(c) => c, None => { - log::error!("decode: no HEVC decoder on this device"); + log::error!("decode: no {mime} decoder on this device"); return; } }; + log::info!("decode: codec mime = {mime}"); let mut format = MediaFormat::new(); - format.set_str("mime", "video/hevc"); + format.set_str("mime", mime); format.set_i32("width", mode.width as i32); format.set_i32("height", mode.height as i32); // Generous input buffer so a large keyframe AU is never truncated. diff --git a/clients/android/native/src/session.rs b/clients/android/native/src/session.rs index c12f72e..7de2abf 100644 --- a/clients/android/native/src/session.rs +++ b/clients/android/native/src/session.rs @@ -167,6 +167,7 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo gamepad_pref: jint, hdr_enabled: jboolean, audio_channels: jint, + preferred_codec: jint, timeout_ms: jint, ) -> jlong { let host: String = match env.get_string(&host) { @@ -224,6 +225,11 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo // decoder + AAudio layout (read in `crate::audio::AudioPlayback::start`). Anything else // normalizes to stereo here. punktfunk_core::audio::normalize_channels(audio_channels.clamp(0, u8::MAX as jint) as u8), + // Codecs this device can decode — AMediaCodec decodes both HEVC and H.264 (AV1 isn't wired; + // hosts don't emit it on the native path yet). The host resolves the emitted codec from these + // + the soft `preferred_codec` and echoes it in `connector.codec`, which drives the mime below. + punktfunk_core::quic::CODEC_H264 | punktfunk_core::quic::CODEC_HEVC, + preferred_codec.clamp(0, u8::MAX as jint) as u8, None, // launch: default app pin, // Some → Crypto on host-fp mismatch identity, // owned (cert, key) PEM, or None (anonymous) diff --git a/clients/apple/Sources/PunktfunkClient/ContentView.swift b/clients/apple/Sources/PunktfunkClient/ContentView.swift index 52d42e4..e238d85 100644 --- a/clients/apple/Sources/PunktfunkClient/ContentView.swift +++ b/clients/apple/Sources/PunktfunkClient/ContentView.swift @@ -28,10 +28,20 @@ struct ContentView: View { @AppStorage(DefaultsKey.gamepadType) private var gamepadType = 0 @AppStorage(DefaultsKey.bitrateKbps) private var bitrateKbps = 0 @AppStorage(DefaultsKey.audioChannels) private var audioChannels = 2 + @AppStorage(DefaultsKey.codec) private var codec = "auto" @AppStorage(DefaultsKey.hdrEnabled) private var hdrEnabled = true @AppStorage(DefaultsKey.fullscreenWhileStreaming) private var fullscreenWhileStreaming = true @AppStorage(DefaultsKey.hudEnabled) private var hudEnabled = true @AppStorage(DefaultsKey.hudPlacement) private var hudPlacement = HUDPlacement.topTrailing.rawValue + /// The `codec` setting as a `PUNKTFUNK_CODEC_*` soft-preference byte (`0` = auto). + private var preferredCodecByte: UInt8 { + switch codec { + case "h264": return PunktfunkConnection.codecH264 + case "hevc": return PunktfunkConnection.codecHEVC + case "av1": return PunktfunkConnection.codecAV1 + default: return 0 + } + } @State private var showAddHost = false @State private var pairingTarget: StoredHost? /// A fresh `pair=required`/unknown host the user tapped: drives the choice between no-PIN @@ -378,6 +388,7 @@ struct ContentView: View { bitrateKbps: UInt32(clamping: bitrateKbps), audioChannels: UInt8(clamping: audioChannels), hdrEnabled: hdrEnabled, + preferredCodec: preferredCodecByte, launchID: launchID, allowTofu: allowTofu, requestAccess: requestAccess) @@ -521,6 +532,7 @@ struct ContentView: View { bitrateKbps: bitrate, audioChannels: UInt8(clamping: audioChannels), hdrEnabled: hdrEnabled, + preferredCodec: preferredCodecByte, autoTrust: true) } } diff --git a/clients/apple/Sources/PunktfunkClient/SessionModel.swift b/clients/apple/Sources/PunktfunkClient/SessionModel.swift index ea96df0..87a0725 100644 --- a/clients/apple/Sources/PunktfunkClient/SessionModel.swift +++ b/clients/apple/Sources/PunktfunkClient/SessionModel.swift @@ -108,6 +108,7 @@ final class SessionModel: ObservableObject { bitrateKbps: UInt32 = 0, audioChannels: UInt8 = 2, hdrEnabled: Bool = true, + preferredCodec: UInt8 = 0, launchID: String? = nil, allowTofu: Bool = false, autoTrust: Bool = false, @@ -155,12 +156,17 @@ final class SessionModel: ObservableObject { if want444, canDecode444 { videoCaps |= PunktfunkConnection.videoCap444 } + // This client's VideoToolbox path decodes H.264 and HEVC (AV1 isn't wired — hosts don't + // emit it on the native path yet). The host resolves the emitted codec from these + the + // soft `preferredCodec`; `resolvedCodec` reflects what it chose. + let videoCodecs = PunktfunkConnection.codecH264 | PunktfunkConnection.codecHEVC let result = Result { try PunktfunkConnection( host: host.address, port: host.port, width: width, height: height, refreshHz: hz, pinSHA256: pin, identity: identity, compositor: compositor, gamepad: gamepad, bitrateKbps: bitrateKbps, videoCaps: videoCaps, - audioChannels: audioChannels, launchID: launchID, + audioChannels: audioChannels, + videoCodecs: videoCodecs, preferredCodec: preferredCodec, launchID: launchID, // Delegated approval: the host holds this connect open until the operator approves // it (~180 s) — outwait that window so a slow approval still lands here. Normal // connects keep the snappy default. diff --git a/clients/apple/Sources/PunktfunkClient/SettingsView.swift b/clients/apple/Sources/PunktfunkClient/SettingsView.swift index e6aef36..a57fb21 100644 --- a/clients/apple/Sources/PunktfunkClient/SettingsView.swift +++ b/clients/apple/Sources/PunktfunkClient/SettingsView.swift @@ -30,6 +30,7 @@ struct SettingsView: View { @AppStorage(DefaultsKey.fullscreenWhileStreaming) private var fullscreenWhileStreaming = true @AppStorage(DefaultsKey.micEnabled) private var micEnabled = true @AppStorage(DefaultsKey.audioChannels) private var audioChannels = 2 + @AppStorage(DefaultsKey.codec) private var codec = "auto" @AppStorage(DefaultsKey.hudEnabled) private var hudEnabled = true @AppStorage(DefaultsKey.hudPlacement) private var hudPlacement = HUDPlacement.topTrailing.rawValue @ObservedObject private var gamepads = GamepadManager.shared @@ -671,16 +672,22 @@ struct SettingsView: View { @ViewBuilder private var hdrSection: some View { Section { + Picker("Video codec", selection: $codec) { + Text("Automatic").tag("auto") + Text("HEVC (H.265)").tag("hevc") + Text("H.264 (AVC)").tag("h264") + } Toggle("10-bit HDR", isOn: $hdrEnabled) Toggle("Full chroma (4:4:4)", isOn: $enable444) } header: { Text("Video quality") } footer: { - Text("HDR requests a 10-bit BT.2020 PQ (HDR10) stream — it only engages when the host is " - + "sending HDR content AND this display supports HDR. 4:4:4 requests full chroma " - + "(sharper text/UI, more bandwidth) — it only engages when this device can " - + "hardware-decode it AND the host opted in. Otherwise the stream stays 8-bit " - + "4:2:0 SDR. Applies from the next session.") + Text("Codec is a preference — the host falls back if it can't encode the one you pick " + + "(and 10-bit/4:4:4 are HEVC-only). HDR requests a 10-bit BT.2020 PQ (HDR10) stream — " + + "it only engages when the host is sending HDR content AND this display supports HDR. " + + "4:4:4 requests full chroma (sharper text/UI, more bandwidth) — it only engages when " + + "this device can hardware-decode it AND the host opted in. Otherwise the stream stays " + + "8-bit 4:2:0 SDR. Applies from the next session.") .font(.geist(12, relativeTo: .caption)) .foregroundStyle(.secondary) } diff --git a/clients/apple/Sources/PunktfunkKit/AnnexB.swift b/clients/apple/Sources/PunktfunkKit/AnnexB.swift index ed9436d..9dfdd09 100644 --- a/clients/apple/Sources/PunktfunkKit/AnnexB.swift +++ b/clients/apple/Sources/PunktfunkKit/AnnexB.swift @@ -10,6 +10,20 @@ import CoreMedia import Foundation +/// The video codec of the host's elementary stream — negotiated in the Welcome and read via +/// `punktfunk_connection_codec`. Both are Annex-B with in-band parameter sets on every IDR; they +/// differ only in NAL-header layout and which parameter sets exist (HEVC adds a VPS). AV1 is not an +/// Annex-B/NAL codec and isn't handled here (hosts don't emit it on the native path yet). +public enum VideoCodec: Equatable { + case h264 + case hevc + + /// Resolve from the wire `Welcome.codec` byte (`PUNKTFUNK_CODEC_*`; unknown → HEVC). + public init(wire: UInt8) { + self = wire == 0x01 ? .h264 : .hevc // 0x01 = PUNKTFUNK_CODEC_H264 + } +} + public enum AnnexB { /// Split an Annex-B stream into NAL units (start codes 00 00 01 / 00 00 00 01 stripped). /// All zeros immediately preceding a start code are dropped: they're either the @@ -47,40 +61,83 @@ public enum AnnexB { return (first >> 1) & 0x3F } - /// Build a format description from an IDR AU's in-band VPS(32)/SPS(33)/PPS(34). - /// Returns nil when the AU carries no parameter sets (non-IDR). - public static func formatDescription(fromIDR au: Data) -> CMVideoFormatDescription? { + /// H.264 NAL unit type (bits 0..4 of the first byte). + public static func h264NalType(_ nal: Data) -> UInt8 { + guard let first = nal.first else { return 0xFF } + return first & 0x1F + } + + /// True if this NAL is a parameter set for `codec` (dropped from AVCC; kept for the format desc). + /// HEVC: VPS 32 / SPS 33 / PPS 34. H.264: SPS 7 / PPS 8 (no VPS). + private static func isParameterSet(_ nal: Data, _ codec: VideoCodec) -> Bool { + switch codec { + case .hevc: let t = hevcNalType(nal); return t == 32 || t == 33 || t == 34 + case .h264: let t = h264NalType(nal); return t == 7 || t == 8 + } + } + + /// Build a format description from an IDR AU's in-band parameter sets (HEVC: VPS/SPS/PPS; + /// H.264: SPS/PPS). Returns nil when the AU carries no parameter sets (non-IDR). + public static func formatDescription(fromIDR au: Data, codec: VideoCodec) + -> CMVideoFormatDescription? + { + // Collect the parameter-set NALs in the order VideoToolbox wants them (HEVC: VPS,SPS,PPS; + // H.264: SPS,PPS). var vps: Data?, sps: Data?, pps: Data? for nal in nalUnits(in: au) { - switch hevcNalType(nal) { - case 32: vps = nal - case 33: sps = nal - case 34: pps = nal - default: break + switch codec { + case .hevc: + switch hevcNalType(nal) { + case 32: vps = nal + case 33: sps = nal + case 34: pps = nal + default: break + } + case .h264: + switch h264NalType(nal) { + case 7: sps = nal + case 8: pps = nal + default: break + } } } - guard let vps, let sps, let pps else { return nil } + guard let sps, let pps else { return nil } + let sets: [Data] = codec == .hevc ? [vps, sps, pps].compactMap { $0 } : [sps, pps] + guard codec == .h264 || sets.count == 3 else { return nil } // HEVC needs the VPS too var format: CMVideoFormatDescription? - let sets = [vps, sps, pps] - let status: OSStatus = sets[0].withUnsafeBytes { v in - sets[1].withUnsafeBytes { s in - sets[2].withUnsafeBytes { p in - let pointers: [UnsafePointer] = [ - v.bindMemory(to: UInt8.self).baseAddress!, - s.bindMemory(to: UInt8.self).baseAddress!, - p.bindMemory(to: UInt8.self).baseAddress!, - ] - let sizes = [vps.count, sps.count, pps.count] - return CMVideoFormatDescriptionCreateFromHEVCParameterSets( - allocator: kCFAllocatorDefault, - parameterSetCount: 3, - parameterSetPointers: pointers, - parameterSetSizes: sizes, - nalUnitHeaderLength: 4, - extensions: nil, - formatDescriptionOut: &format) - } + // Pin every parameter set's bytes for the duration of the create call, then hand + // VideoToolbox parallel pointer/size arrays. + var pointers: [UnsafePointer] = [] + var sizes: [Int] = [] + func withAll(_ i: Int, _ body: () -> Void) { + if i == sets.count { body(); return } + sets[i].withUnsafeBytes { raw in + pointers.append(raw.bindMemory(to: UInt8.self).baseAddress!) + sizes.append(sets[i].count) + withAll(i + 1, body) + } + } + var status: OSStatus = -1 + withAll(0) { + switch codec { + case .hevc: + status = CMVideoFormatDescriptionCreateFromHEVCParameterSets( + allocator: kCFAllocatorDefault, + parameterSetCount: pointers.count, + parameterSetPointers: pointers, + parameterSetSizes: sizes, + nalUnitHeaderLength: 4, + extensions: nil, + formatDescriptionOut: &format) + case .h264: + status = CMVideoFormatDescriptionCreateFromH264ParameterSets( + allocator: kCFAllocatorDefault, + parameterSetCount: pointers.count, + parameterSetPointers: pointers, + parameterSetSizes: sizes, + nalUnitHeaderLength: 4, + formatDescriptionOut: &format) } } return status == noErr ? format : nil @@ -88,11 +145,10 @@ public enum AnnexB { /// Re-pack an Annex-B AU as AVCC (4-byte big-endian length before each NAL), dropping /// the parameter-set NALs (they live in the format description). - public static func avcc(from au: Data) -> Data { + public static func avcc(from au: Data, codec: VideoCodec) -> Data { var out = Data(capacity: au.count + 16) for nal in nalUnits(in: au) { - let t = hevcNalType(nal) - if t == 32 || t == 33 || t == 34 { continue } // VPS/SPS/PPS + if isParameterSet(nal, codec) { continue } var len = UInt32(nal.count).bigEndian withUnsafeBytes(of: &len) { out.append(contentsOf: $0) } out.append(nal) @@ -102,9 +158,9 @@ public enum AnnexB { /// Wrap one AU as a decode-ready CMSampleBuffer. public static func sampleBuffer( - au: AccessUnit, format: CMVideoFormatDescription + au: AccessUnit, format: CMVideoFormatDescription, codec: VideoCodec ) -> CMSampleBuffer? { - let avccData = avcc(from: au.data) + let avccData = avcc(from: au.data, codec: codec) var blockBuffer: CMBlockBuffer? guard CMBlockBufferCreateWithMemoryBlock( allocator: kCFAllocatorDefault, memoryBlock: nil, diff --git a/clients/apple/Sources/PunktfunkKit/DefaultsKeys.swift b/clients/apple/Sources/PunktfunkKit/DefaultsKeys.swift index 9b85529..370ac57 100644 --- a/clients/apple/Sources/PunktfunkKit/DefaultsKeys.swift +++ b/clients/apple/Sources/PunktfunkKit/DefaultsKeys.swift @@ -18,6 +18,9 @@ public enum DefaultsKey { /// Requested audio channel count: 2 (stereo), 6 (5.1) or 8 (7.1). The host clamps to what it /// can capture; the resolved count drives the in-core decode + AVAudioEngine layout. public static let audioChannels = "punktfunk.audioChannels" + /// Preferred video codec: `"auto"` (host decides), `"hevc"`, or `"h264"`. A soft preference — + /// the host emits it when it can, else falls back. Drives the decoder via `Welcome.codec`. + public static let codec = "punktfunk.codec" public static let micEnabled = "punktfunk.micEnabled" public static let speakerUID = "punktfunk.speakerUID" public static let micUID = "punktfunk.micUID" diff --git a/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift b/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift index 564954d..172a5ed 100644 --- a/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift +++ b/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift @@ -255,6 +255,13 @@ public final class PunktfunkConnection { /// PCM from `nextAudioPcm` is interleaved in the canonical wire order FL FR FC LFE RL RR SL SR. public private(set) var resolvedAudioChannels: UInt8 = 2 + /// The video codec the host resolved for this session (`Welcome.codec`, `PUNKTFUNK_CODEC_*`): + /// `2` = HEVC (default / older host), `1` = H.264, `4` = AV1. Build the decoder from THIS. The + /// resolved value honors the client's `preferredCodec` when the host could emit it. + public private(set) var resolvedCodec: UInt8 = 2 // PUNKTFUNK_CODEC_HEVC + /// The resolved codec as an `AnnexB.VideoCodec` (H.264 vs HEVC) — drives the NAL parsing. + public var videoCodec: VideoCodec { VideoCodec(wire: resolvedCodec) } + /// Connect and start a session at the requested mode (the host creates a native virtual /// output at exactly this size/refresh). Blocks up to `timeoutMs`. /// @@ -285,6 +292,8 @@ public final class PunktfunkConnection { bitrateKbps: UInt32 = 0, videoCaps: UInt8 = 0, audioChannels: UInt8 = 2, + videoCodecs: UInt8 = 0x02, // PUNKTFUNK_CODEC_HEVC — the codecs this client can decode + preferredCodec: UInt8 = 0, // 0 = auto; else PUNKTFUNK_CODEC_* soft preference launchID: String? = nil, timeoutMs: UInt32 = 10_000 ) throws { @@ -300,16 +309,18 @@ public final class PunktfunkConnection { withOptionalCString(launchID) { launch in if let pin = pinSHA256 { return pin.withUnsafeBytes { p in - punktfunk_connect_ex6( + punktfunk_connect_ex7( cs, port, width, height, refreshHz, compositor.rawValue, - gamepad.rawValue, bitrateKbps, videoCaps, audioChannels, launch, + gamepad.rawValue, bitrateKbps, videoCaps, audioChannels, + videoCodecs, preferredCodec, launch, p.bindMemory(to: UInt8.self).baseAddress, &observed, cert, key, timeoutMs) } } - return punktfunk_connect_ex6( + return punktfunk_connect_ex7( cs, port, width, height, refreshHz, compositor.rawValue, - gamepad.rawValue, bitrateKbps, videoCaps, audioChannels, launch, + gamepad.rawValue, bitrateKbps, videoCaps, audioChannels, + videoCodecs, preferredCodec, launch, nil, &observed, cert, key, timeoutMs) } } @@ -347,6 +358,9 @@ public final class PunktfunkConnection { var ac: UInt8 = 2 _ = punktfunk_connection_audio_channels(handle, &ac) resolvedAudioChannels = ac + var codec: UInt8 = 2 // PUNKTFUNK_CODEC_HEVC + _ = punktfunk_connection_codec(handle, &codec) + resolvedCodec = codec } /// A bandwidth speed-test measurement (see `startSpeedTest`). Partial until `done`. @@ -620,6 +634,11 @@ public final class PunktfunkConnection { /// the host then emits 4:4:4 only if it too opted in. `chromaFormat` reflects the real value. public static let videoCap444: UInt8 = UInt8(PUNKTFUNK_VIDEO_CAP_444) + /// Codec bits for `videoCodecs` / `preferredCodec` and the value `resolvedCodec` returns. + public static let codecH264: UInt8 = UInt8(PUNKTFUNK_CODEC_H264) + public static let codecHEVC: UInt8 = UInt8(PUNKTFUNK_CODEC_HEVC) + public static let codecAV1: UInt8 = UInt8(PUNKTFUNK_CODEC_AV1) + /// Static HDR mastering metadata (SMPTE ST.2086 + content light level) the host sent for an HDR /// session. Mirrors the wire/ABI `PunktfunkHdrMeta`; primaries are in ST.2086 **G, B, R** order, /// 1/50000 units; mastering luminance in 0.0001 cd/m²; MaxCLL/MaxFALL in nits. diff --git a/clients/apple/Sources/PunktfunkKit/Stage2Pipeline.swift b/clients/apple/Sources/PunktfunkKit/Stage2Pipeline.swift index 5c8741d..436d4ee 100644 --- a/clients/apple/Sources/PunktfunkKit/Stage2Pipeline.swift +++ b/clients/apple/Sources/PunktfunkKit/Stage2Pipeline.swift @@ -119,6 +119,7 @@ public final class Stage2Pipeline { // chroma subsampling drives only the decode pixel format (orthogonal to HDR/depth); the HDR // config is the Welcome's latched value, which a mid-session flip then overrides per-frame. decoder.setChroma444(connection.isChroma444) + decoder.setCodec(connection.videoCodec) presenter.configure(hdr: connection.isHDR) let token = token @@ -159,7 +160,7 @@ public final class Stage2Pipeline { } guard let au = try connection.nextAU(timeoutMs: 100) else { continue } onFrame?(au) - if let f = AnnexB.formatDescription(fromIDR: au.data) { + if let f = AnnexB.formatDescription(fromIDR: au.data, codec: connection.videoCodec) { format = f // refreshed on every IDR (mode changes included) awaitingIDR = false // a fresh IDR re-anchored decode — recovery complete } diff --git a/clients/apple/Sources/PunktfunkKit/StreamPump.swift b/clients/apple/Sources/PunktfunkKit/StreamPump.swift index 4b41c30..cecae03 100644 --- a/clients/apple/Sources/PunktfunkKit/StreamPump.swift +++ b/clients/apple/Sources/PunktfunkKit/StreamPump.swift @@ -95,7 +95,7 @@ final class StreamPump { guard let au = try connection.nextAU(timeoutMs: 100) else { continue } onFrame?(au) - let idrFormat = AnnexB.formatDescription(fromIDR: au.data) + let idrFormat = AnnexB.formatDescription(fromIDR: au.data, codec: connection.videoCodec) if let f = idrFormat { format = f // refreshed on every IDR (mode changes included) if awaitingIDR { @@ -119,7 +119,7 @@ final class StreamPump { } wasFailed = failed guard let f = format, - let sample = AnnexB.sampleBuffer(au: au, format: f), + let sample = AnnexB.sampleBuffer(au: au, format: f, codec: connection.videoCodec), token.isLive // don't enqueue a stale frame after a restart else { continue } layer.enqueue(sample) diff --git a/clients/apple/Sources/PunktfunkKit/VideoDecoder.swift b/clients/apple/Sources/PunktfunkKit/VideoDecoder.swift index 127b3f9..e8e6761 100644 --- a/clients/apple/Sources/PunktfunkKit/VideoDecoder.swift +++ b/clients/apple/Sources/PunktfunkKit/VideoDecoder.swift @@ -54,6 +54,10 @@ public final class VideoDecoder: @unchecked Sendable { /// depth / HDR). Read inside `createSessionLocked` under `lock`. private var chroma444 = false + /// The negotiated codec (`connection.videoCodec`), set once at session start. Drives the AnnexB + /// NAL parsing (H.264 vs HEVC parameter sets). Read under `lock`. + private var codec: VideoCodec = .hevc + public init( onDecoded: @escaping @Sendable (ReadyFrame) -> Void, onDecodeError: @escaping @Sendable (OSStatus) -> Void = { _ in } @@ -73,6 +77,14 @@ public final class VideoDecoder: @unchecked Sendable { lock.unlock() } + /// Select the negotiated codec (H.264 vs HEVC). Call once at session start, before decoding, + /// from `connection.videoCodec`. Thread-safe. + public func setCodec(_ c: VideoCodec) { + lock.lock() + codec = c + lock.unlock() + } + /// Submit one AU for asynchronous decode, (re)creating the session if `format` changed. The /// caller resolves `format` from the IDR exactly as stage-1 does (`AnnexB.formatDescription`). /// Returns false if the session couldn't be created or the frame couldn't be submitted. @@ -93,7 +105,7 @@ public final class VideoDecoder: @unchecked Sendable { // invalidate the session between here and DecodeFrame. The VT output callback takes the // ring lock, not this one, so there's no re-entrancy. DecodeFrame is async — non-blocking. guard let session, - let sample = AnnexB.sampleBuffer(au: au, format: newFormat) + let sample = AnnexB.sampleBuffer(au: au, format: newFormat, codec: codec) else { lock.unlock(); return false } var infoOut = VTDecodeInfoFlags() let status = VTDecompressionSessionDecodeFrame( diff --git a/clients/apple/Tests/PunktfunkKitTests/AnnexBTests.swift b/clients/apple/Tests/PunktfunkKitTests/AnnexBTests.swift index 8bde705..e443b00 100644 --- a/clients/apple/Tests/PunktfunkKitTests/AnnexBTests.swift +++ b/clients/apple/Tests/PunktfunkKitTests/AnnexBTests.swift @@ -64,7 +64,7 @@ final class AnnexBTests: XCTestCase { au.append(n) } - let avcc = AnnexB.avcc(from: au) + let avcc = AnnexB.avcc(from: au, codec: .hevc) // Only the IDR survives: 4-byte BE length, then the NAL bytes. var expected = Data([0, 0, 0, UInt8(idr.count)]) expected.append(idr) @@ -74,6 +74,6 @@ final class AnnexBTests: XCTestCase { func testFormatDescriptionNilWithoutParameterSets() { let idr = nal(type: 19, payload: [0xDD]) let au = Data(start4) + idr - XCTAssertNil(AnnexB.formatDescription(fromIDR: au)) + XCTAssertNil(AnnexB.formatDescription(fromIDR: au, codec: .hevc)) } } diff --git a/clients/apple/Tests/PunktfunkKitTests/RemoteFirstLightTests.swift b/clients/apple/Tests/PunktfunkKitTests/RemoteFirstLightTests.swift index d7a354d..cb81960 100644 --- a/clients/apple/Tests/PunktfunkKitTests/RemoteFirstLightTests.swift +++ b/clients/apple/Tests/PunktfunkKitTests/RemoteFirstLightTests.swift @@ -138,7 +138,7 @@ final class RemoteFirstLightTests: XCTestCase { if firstPtsNs == 0 { firstPtsNs = au.ptsNs } lastPtsNs = au.ptsNs - if let f = AnnexB.formatDescription(fromIDR: au.data) { + if let f = AnnexB.formatDescription(fromIDR: au.data, codec: .hevc) { format = f if decoder == nil { let dims = CMVideoFormatDescriptionGetDimensions(f) @@ -155,7 +155,7 @@ final class RemoteFirstLightTests: XCTestCase { } } guard let f = format, let dec = decoder, - let sample = AnnexB.sampleBuffer(au: au, format: f) + let sample = AnnexB.sampleBuffer(au: au, format: f, codec: .hevc) else { continue } var gotPixels = false diff --git a/clients/apple/Tests/PunktfunkKitTests/Stage444Tests.swift b/clients/apple/Tests/PunktfunkKitTests/Stage444Tests.swift index e78d9f3..328152a 100644 --- a/clients/apple/Tests/PunktfunkKitTests/Stage444Tests.swift +++ b/clients/apple/Tests/PunktfunkKitTests/Stage444Tests.swift @@ -30,7 +30,7 @@ final class Stage444Tests: XCTestCase { Stage444Probe.hwDecode444_8bit, "no hardware 4:4:4 decode on this device") let data = Data(Probe444Blobs.au444_8bit) let format = try XCTUnwrap( - AnnexB.formatDescription(fromIDR: data), "the 4:4:4 blob must yield a format description") + AnnexB.formatDescription(fromIDR: data, codec: .hevc), "the 4:4:4 blob must yield a format description") let au = AccessUnit(data: data, ptsNs: 7_000_000, frameIndex: 0, flags: 0) let box = FrameBox() diff --git a/clients/apple/Tests/PunktfunkKitTests/VideoToolboxRoundTripTests.swift b/clients/apple/Tests/PunktfunkKitTests/VideoToolboxRoundTripTests.swift index d501617..c989943 100644 --- a/clients/apple/Tests/PunktfunkKitTests/VideoToolboxRoundTripTests.swift +++ b/clients/apple/Tests/PunktfunkKitTests/VideoToolboxRoundTripTests.swift @@ -28,18 +28,18 @@ final class VideoToolboxRoundTripTests: XCTestCase { // 1) Parameter-set extraction → format description. let rebuilt = try XCTUnwrap( - AnnexB.formatDescription(fromIDR: annexB), + AnnexB.formatDescription(fromIDR: annexB, codec: .hevc), "in-band VPS/SPS/PPS should yield a format description") let dims = CMVideoFormatDescriptionGetDimensions(rebuilt) XCTAssertEqual(Int(dims.width), width) XCTAssertEqual(Int(dims.height), height) // 2) Annex-B → AVCC re-pack must reproduce the encoder's own sample bytes. - XCTAssertEqual(AnnexB.avcc(from: annexB), avccSample) + XCTAssertEqual(AnnexB.avcc(from: annexB, codec: .hevc), avccSample) // 3) Sample buffer → real decoder → pixels. let au = AccessUnit(data: annexB, ptsNs: 1_000_000, frameIndex: 0, flags: 0) - let sample = try XCTUnwrap(AnnexB.sampleBuffer(au: au, format: rebuilt)) + let sample = try XCTUnwrap(AnnexB.sampleBuffer(au: au, format: rebuilt, codec: .hevc)) var session: VTDecompressionSession? XCTAssertEqual( @@ -72,7 +72,7 @@ final class VideoToolboxRoundTripTests: XCTestCase { func testVideoDecoderAsyncCallbackDeliversPixels() throws { let (formatDesc, avccSample) = try encodeOneHEVCKeyframe() let annexB = try annexBAU(formatDesc: formatDesc, avccSample: avccSample) - let format = try XCTUnwrap(AnnexB.formatDescription(fromIDR: annexB)) + let format = try XCTUnwrap(AnnexB.formatDescription(fromIDR: annexB, codec: .hevc)) let au = AccessUnit(data: annexB, ptsNs: 42_000_000, frameIndex: 0, flags: 0) let box = FrameBox() diff --git a/clients/windows/src/app.rs b/clients/windows/src/app.rs index b95629b..71713d8 100644 --- a/clients/windows/src/app.rs +++ b/clients/windows/src/app.rs @@ -44,6 +44,14 @@ const BITRATES_MBPS: &[u32] = &[0, 10, 20, 30, 50, 80, 150]; /// Audio channel presets: `(channel count, display label)`. The host clamps to what it can /// capture; the resolved count drives the decoder + WASAPI render layout. const AUDIO_CHANNELS: &[(u8, &str)] = &[(2, "Stereo"), (6, "5.1 Surround"), (8, "7.1 Surround")]; +/// Preferred-codec presets: `(stored value, display label)`. Soft — the host falls back if it can't +/// encode the chosen codec. +const CODECS: &[(&str, &str)] = &[ + ("auto", "Automatic"), + ("hevc", "HEVC (H.265)"), + ("h264", "H.264 (AVC)"), + ("av1", "AV1"), +]; /// punktfunk's own license (MIT OR Apache-2.0), shown on the Licenses screen. const APP_LICENSE: &str = concat!( @@ -681,6 +689,7 @@ fn connect_with( mic_enabled: s.mic_enabled, hdr_enabled: s.hdr_enabled, decoder: DecoderPref::from_name(&s.decoder), + preferred_codec: s.preferred_codec(), pin, identity: ctx.identity.clone(), connect_timeout: opts.connect_timeout, @@ -1039,6 +1048,21 @@ fn settings_page(ctx: &Arc, set_screen: &AsyncSetState) -> Eleme }) }; + let codec_i = CODECS.iter().position(|&(v, _)| v == s.codec).unwrap_or(0) as i32; + let codec_names: Vec = CODECS.iter().map(|&(_, l)| l.to_string()).collect(); + let codec_combo = { + let ctx = ctx.clone(); + ComboBox::new(codec_names) + .header("Video codec") + .selected_index(codec_i) + .on_selection_changed(move |i: i32| { + let (v, _) = CODECS[(i.max(0) as usize).min(CODECS.len() - 1)]; + let mut s = ctx.settings.lock().unwrap(); + s.codec = v.to_string(); + s.save(); + }) + }; + let br_i = BITRATES_MBPS .iter() .position(|&m| m * 1000 == s.bitrate_kbps) @@ -1149,6 +1173,7 @@ fn settings_page(ctx: &Arc, set_screen: &AsyncSetState) -> Eleme .font_size(12.0) .foreground(ThemeRef::SecondaryText), decoder_combo, + codec_combo, bitrate_combo, hdr_toggle, )) diff --git a/clients/windows/src/main.rs b/clients/windows/src/main.rs index da2f239..bb83d79 100644 --- a/clients/windows/src/main.rs +++ b/clients/windows/src/main.rs @@ -182,6 +182,13 @@ fn run_headless_cli(args: &[String], identity: (String, String)) { mic_enabled: flag("--mic"), hdr_enabled: !flag("--no-hdr"), decoder, + // `--codec h264|hevc|av1` sets the soft preference; default auto (host decides). + preferred_codec: match arg("--codec").as_deref() { + Some("h264") | Some("avc") => punktfunk_core::quic::CODEC_H264, + Some("hevc") | Some("h265") => punktfunk_core::quic::CODEC_HEVC, + Some("av1") => punktfunk_core::quic::CODEC_AV1, + _ => 0, + }, pin, identity, // Headless CLI uses the normal (short) handshake budget; the long request-access wait is a diff --git a/clients/windows/src/session.rs b/clients/windows/src/session.rs index e9e2052..a40cbca 100644 --- a/clients/windows/src/session.rs +++ b/clients/windows/src/session.rs @@ -31,6 +31,9 @@ pub struct SessionParams { pub hdr_enabled: bool, /// Which video decode backend to use (auto/hardware/software). pub decoder: DecoderPref, + /// The user's preferred video codec (a `quic::CODEC_*` bit, `0` = auto). Soft — the host honors + /// it when it can emit it, else falls back; the resolved codec drives the decoder. + pub preferred_codec: u8, /// Pinned host fingerprint; `None` = trust on first use (caller persists the observed one). pub pin: Option<[u8; 32]>, pub identity: (String, String), @@ -166,7 +169,9 @@ fn pump( 0 }, params.audio_channels, - None, // launch: the Windows client has no library picker yet + crate::video::decodable_codecs(), // codecs FFmpeg can decode (HEVC/H.264/AV1) + params.preferred_codec, // the user's soft codec preference (0 = auto) + None, // launch: the Windows client has no library picker yet params.pin, Some(params.identity), params.connect_timeout, @@ -195,7 +200,14 @@ fn pump( fingerprint: connector.host_fingerprint, }); - let mut decoder = match Decoder::new(params.decoder) { + // Build the decoder for the codec the host resolved (never assume HEVC). + let codec_id = crate::video::ffmpeg_codec_id(connector.codec); + tracing::info!( + ?codec_id, + welcome_codec = connector.codec, + "negotiated video codec" + ); + let mut decoder = match Decoder::new(params.decoder, codec_id) { Ok(d) => d, Err(e) => { let _ = ev_tx.send_blocking(SessionEvent::Ended(Some(format!("video decoder: {e}")))); diff --git a/clients/windows/src/trust.rs b/clients/windows/src/trust.rs index 5d578ae..a7c21db 100644 --- a/clients/windows/src/trust.rs +++ b/clients/windows/src/trust.rs @@ -138,6 +138,26 @@ pub struct Settings { pub hdr_enabled: bool, /// Video decode backend: `auto` (D3D11VA, fall back to software), `hardware`, or `software`. pub decoder: String, + /// Preferred video codec: `"auto"` (host decides), `"hevc"`, `"h264"`, or `"av1"`. A soft + /// preference — the host honors it when it can emit it, else falls back to the best shared codec. + #[serde(default = "default_codec")] + pub codec: String, +} + +fn default_codec() -> String { + "auto".into() +} + +impl Settings { + /// The `codec` setting as a `quic::CODEC_*` preference bit (`0` = auto). + pub fn preferred_codec(&self) -> u8 { + match self.codec.as_str() { + "h264" | "avc" => punktfunk_core::quic::CODEC_H264, + "hevc" | "h265" => punktfunk_core::quic::CODEC_HEVC, + "av1" => punktfunk_core::quic::CODEC_AV1, + _ => 0, + } + } } impl Default for Settings { @@ -154,6 +174,7 @@ impl Default for Settings { audio_channels: 2, hdr_enabled: true, decoder: "auto".into(), + codec: "auto".into(), } } } diff --git a/clients/windows/src/video.rs b/clients/windows/src/video.rs index 18a5b67..05657a1 100644 --- a/clients/windows/src/video.rs +++ b/clients/windows/src/video.rs @@ -124,17 +124,46 @@ enum Backend { pub struct Decoder { backend: Backend, + /// The negotiated codec, so a mid-session D3D11VA→software demotion rebuilds for the same codec. + codec_id: ffmpeg::codec::Id, +} + +/// Map a negotiated `quic` codec bit to the FFmpeg decoder id the client opens. +pub fn ffmpeg_codec_id(wire: u8) -> ffmpeg::codec::Id { + match wire { + punktfunk_core::quic::CODEC_H264 => ffmpeg::codec::Id::H264, + punktfunk_core::quic::CODEC_AV1 => ffmpeg::codec::Id::AV1, + _ => ffmpeg::codec::Id::HEVC, + } +} + +/// The `quic` codec bitfield this client can decode — whatever FFmpeg has a decoder for (HEVC/H.264 +/// always; AV1 when built in). Advertised to the host so it never emits a codec we can't decode. +pub fn decodable_codecs() -> u8 { + let _ = ffmpeg::init(); + let mut bits = 0u8; + for (id, bit) in [ + (ffmpeg::codec::Id::HEVC, punktfunk_core::quic::CODEC_HEVC), + (ffmpeg::codec::Id::H264, punktfunk_core::quic::CODEC_H264), + (ffmpeg::codec::Id::AV1, punktfunk_core::quic::CODEC_AV1), + ] { + if ffmpeg::decoder::find(id).is_some() { + bits |= bit; + } + } + bits } impl Decoder { - pub fn new(pref: DecoderPref) -> Result { + pub fn new(pref: DecoderPref, codec_id: ffmpeg::codec::Id) -> Result { ffmpeg::init().context("ffmpeg init")?; if pref != DecoderPref::Software { - match D3d11vaDecoder::new() { + match D3d11vaDecoder::new(codec_id) { Ok(d) => { - tracing::info!("D3D11VA hardware decode active (zero-copy)"); + tracing::info!(?codec_id, "D3D11VA hardware decode active (zero-copy)"); return Ok(Decoder { backend: Backend::D3d11va(d), + codec_id, }); } Err(e) => { @@ -146,7 +175,8 @@ impl Decoder { } } Ok(Decoder { - backend: Backend::Software(SoftwareDecoder::new()?), + backend: Backend::Software(SoftwareDecoder::new(codec_id)?), + codec_id, }) } @@ -164,7 +194,7 @@ impl Decoder { Ok(f) => Ok(f.map(DecodedFrame::Gpu)), Err(e) => { tracing::warn!(error = %e, "D3D11VA decode failed — falling back to software"); - self.backend = Backend::Software(SoftwareDecoder::new()?); + self.backend = Backend::Software(SoftwareDecoder::new(self.codec_id)?); Ok(None) } }, @@ -183,9 +213,9 @@ struct SoftwareDecoder { } impl SoftwareDecoder { - fn new() -> Result { - let codec = - ffmpeg::decoder::find(ffmpeg::codec::Id::HEVC).ok_or(anyhow!("no HEVC decoder"))?; + fn new(codec_id: ffmpeg::codec::Id) -> Result { + let codec = ffmpeg::decoder::find(codec_id) + .ok_or_else(|| anyhow!("no {codec_id:?} decoder in libavcodec"))?; let mut ctx = ffmpeg::codec::Context::new_with_codec(codec); unsafe { let raw = ctx.as_mut_ptr(); @@ -194,7 +224,7 @@ impl SoftwareDecoder { (*raw).thread_type = ffmpeg::ffi::FF_THREAD_SLICE; (*raw).thread_count = 0; // auto } - let decoder = ctx.decoder().video().context("open HEVC decoder")?; + let decoder = ctx.decoder().video().context("open video decoder")?; Ok(SoftwareDecoder { decoder, sws: None }) } @@ -359,7 +389,7 @@ struct D3d11vaDecoder { unsafe impl Send for D3d11vaDecoder {} impl D3d11vaDecoder { - fn new() -> Result { + fn new(codec_id: ffmpeg::codec::Id) -> Result { use ffmpeg::ffi; let shared = crate::gpu::shared().ok_or_else(|| anyhow!("no shared D3D11 device"))?; if !shared.hardware { @@ -387,11 +417,11 @@ impl D3d11vaDecoder { bail!("av_hwdevice_ctx_init: {}", ffmpeg::Error::from(r)); } - let codec = ffi::avcodec_find_decoder(ffi::AVCodecID::AV_CODEC_ID_HEVC); + let codec = ffi::avcodec_find_decoder(codec_id.into()); if codec.is_null() { let mut hw = hw_device; ffi::av_buffer_unref(&mut hw); - bail!("no HEVC decoder"); + bail!("no {codec_id:?} decoder"); } let ctx = ffi::avcodec_alloc_context3(codec); (*ctx).hw_device_ctx = ffi::av_buffer_ref(hw_device);