// Annex-B HEVC → CoreMedia plumbing. // // The punktfunk host emits Annex-B access units with in-band VPS/SPS/PPS on every IDR // (deliberately — the client needs no out-of-band extradata). VideoToolbox wants the AVCC // flavor instead: a CMVideoFormatDescription built from the parameter sets, and sample // buffers whose NALs are 4-byte-length-prefixed. This file converts between the two. // // SCAFFOLD: written on the Linux host, not yet compiled against Xcode. import CoreMedia import Foundation /// The video codec of the host's elementary stream — negotiated in the Welcome and read via /// `punktfunk_connection_codec`. Both are Annex-B with in-band parameter sets on every IDR; they /// differ only in NAL-header layout and which parameter sets exist (HEVC adds a VPS). AV1 is not an /// Annex-B/NAL codec and isn't handled here (hosts don't emit it on the native path yet). public enum VideoCodec: Equatable { case h264 case hevc /// Resolve from the wire `Welcome.codec` byte (`PUNKTFUNK_CODEC_*`; unknown → HEVC). public init(wire: UInt8) { self = wire == 0x01 ? .h264 : .hevc // 0x01 = PUNKTFUNK_CODEC_H264 } } public enum AnnexB { /// Split an Annex-B stream into NAL units (start codes 00 00 01 / 00 00 00 01 stripped). /// All zeros immediately preceding a start code are dropped: they're either the /// 4-byte-code prefix or `trailing_zero_8bits` padding, never NAL payload (emulation /// prevention keeps 00 00 0x out of conforming NAL bytes) — same policy as ffmpeg. public static func nalUnits(in data: Data) -> [Data] { var nals: [Data] = [] let bytes = [UInt8](data) var i = 0 var start = -1 while i + 2 < bytes.count { if bytes[i] == 0, bytes[i + 1] == 0, bytes[i + 2] == 1 { var codeStart = i while codeStart > 0, bytes[codeStart - 1] == 0 { codeStart -= 1 } if start >= 0, start < codeStart { nals.append(Data(bytes[start..= 0, start < bytes.count { nals.append(Data(bytes[start...])) } return nals } /// HEVC NAL unit type (bits 1..6 of the first byte). public static func hevcNalType(_ nal: Data) -> UInt8 { guard let first = nal.first else { return 0xFF } return (first >> 1) & 0x3F } /// H.264 NAL unit type (bits 0..4 of the first byte). public static func h264NalType(_ nal: Data) -> UInt8 { guard let first = nal.first else { return 0xFF } return first & 0x1F } /// True if this NAL is a parameter set for `codec` (dropped from AVCC; kept for the format desc). /// HEVC: VPS 32 / SPS 33 / PPS 34. H.264: SPS 7 / PPS 8 (no VPS). private static func isParameterSet(_ nal: Data, _ codec: VideoCodec) -> Bool { switch codec { case .hevc: let t = hevcNalType(nal); return t == 32 || t == 33 || t == 34 case .h264: let t = h264NalType(nal); return t == 7 || t == 8 } } /// Build a format description from an IDR AU's in-band parameter sets (HEVC: VPS/SPS/PPS; /// H.264: SPS/PPS). Returns nil when the AU carries no parameter sets (non-IDR). public static func formatDescription(fromIDR au: Data, codec: VideoCodec) -> CMVideoFormatDescription? { // Collect the parameter-set NALs in the order VideoToolbox wants them (HEVC: VPS,SPS,PPS; // H.264: SPS,PPS). var vps: Data?, sps: Data?, pps: Data? for nal in nalUnits(in: au) { switch codec { case .hevc: switch hevcNalType(nal) { case 32: vps = nal case 33: sps = nal case 34: pps = nal default: break } case .h264: switch h264NalType(nal) { case 7: sps = nal case 8: pps = nal default: break } } } guard let sps, let pps else { return nil } let sets: [Data] = codec == .hevc ? [vps, sps, pps].compactMap { $0 } : [sps, pps] guard codec == .h264 || sets.count == 3 else { return nil } // HEVC needs the VPS too var format: CMVideoFormatDescription? // Pin every parameter set's bytes for the duration of the create call, then hand // VideoToolbox parallel pointer/size arrays. var pointers: [UnsafePointer] = [] var sizes: [Int] = [] func withAll(_ i: Int, _ body: () -> Void) { if i == sets.count { body(); return } sets[i].withUnsafeBytes { raw in pointers.append(raw.bindMemory(to: UInt8.self).baseAddress!) sizes.append(sets[i].count) withAll(i + 1, body) } } var status: OSStatus = -1 withAll(0) { switch codec { case .hevc: status = CMVideoFormatDescriptionCreateFromHEVCParameterSets( allocator: kCFAllocatorDefault, parameterSetCount: pointers.count, parameterSetPointers: pointers, parameterSetSizes: sizes, nalUnitHeaderLength: 4, extensions: nil, formatDescriptionOut: &format) case .h264: status = CMVideoFormatDescriptionCreateFromH264ParameterSets( allocator: kCFAllocatorDefault, parameterSetCount: pointers.count, parameterSetPointers: pointers, parameterSetSizes: sizes, nalUnitHeaderLength: 4, formatDescriptionOut: &format) } } return status == noErr ? format : nil } /// Re-pack an Annex-B AU as AVCC (4-byte big-endian length before each NAL), dropping /// the parameter-set NALs (they live in the format description). public static func avcc(from au: Data, codec: VideoCodec) -> Data { var out = Data(capacity: au.count + 16) for nal in nalUnits(in: au) { if isParameterSet(nal, codec) { continue } var len = UInt32(nal.count).bigEndian withUnsafeBytes(of: &len) { out.append(contentsOf: $0) } out.append(nal) } return out } /// Wrap one AU as a decode-ready CMSampleBuffer. public static func sampleBuffer( au: AccessUnit, format: CMVideoFormatDescription, codec: VideoCodec ) -> CMSampleBuffer? { let avccData = avcc(from: au.data, codec: codec) var blockBuffer: CMBlockBuffer? guard CMBlockBufferCreateWithMemoryBlock( allocator: kCFAllocatorDefault, memoryBlock: nil, blockLength: avccData.count, blockAllocator: kCFAllocatorDefault, customBlockSource: nil, offsetToData: 0, dataLength: avccData.count, flags: 0, blockBufferOut: &blockBuffer) == noErr, let block = blockBuffer else { return nil } let copied = avccData.withUnsafeBytes { raw in CMBlockBufferReplaceDataBytes( with: raw.baseAddress!, blockBuffer: block, offsetIntoDestination: 0, dataLength: avccData.count) } guard copied == noErr else { return nil } var timing = CMSampleTimingInfo( duration: .invalid, presentationTimeStamp: CMTime(value: Int64(au.ptsNs), timescale: 1_000_000_000), decodeTimeStamp: .invalid) var sampleSize = avccData.count var sample: CMSampleBuffer? guard CMSampleBufferCreate( allocator: kCFAllocatorDefault, dataBuffer: block, dataReady: true, makeDataReadyCallback: nil, refcon: nil, formatDescription: format, sampleCount: 1, sampleTimingEntryCount: 1, sampleTimingArray: &timing, sampleSizeEntryCount: 1, sampleSizeArray: &sampleSize, sampleBufferOut: &sample) == noErr else { return nil } // Low-latency display: render on arrival, don't wait for a clock. if let attachments = CMSampleBufferGetSampleAttachmentsArray(sample!, createIfNecessary: true) { let dict = unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self) CFDictionarySetValue( dict, Unmanaged.passUnretained(kCMSampleAttachmentKey_DisplayImmediately).toOpaque(), Unmanaged.passUnretained(kCFBooleanTrue).toOpaque()) } return sample } }