feat(clients): HDR Steps 2-3 — apply mastering metadata + display capability-gate

Continues docs/hdr-pipeline-plan.md. Steps 0/1 + Step 2 (Windows/Android) already landed in 3526517; this is Step 2 (Apple) + Step 3 (all clients). Client-only — no core/host/ABI change (the 0xCE/next_hdr_meta/color_info surfaces shipped in Step 0). Step 2 — clients APPLY the host's HDR metadata (each remaps from the wire form: ST.2086 G,B,R order, mastering luminance in 0.0001 cd/m2): - Apple: connect via punktfunk_connect_ex5 (resurrects the previously-dead HDR pipeline); nextHdrMeta/colorInfo wrappers + HdrMeta SEI-blob builders; the pump drains nextHdrMeta -> VideoDecoder.setHdrMeta -> CVBufferSetAttachment of MasteringDisplayColorVolume (24B BE) + ContentLightLevelInfo (4B BE) on each HDR pixel buffer (correct for the itur_2100_PQ layer; CAEDRMetadata avoided as ambiguous there). Step 3 — capability-gate: advertise HDR caps ONLY when the display can present it, so an SDR display gets a proper BT.709 stream instead of PQ it would mis-tone-map; an HDR display self-tone-maps from the Step-1/2 mastering metadata. - Windows: present::display_supports_hdr() (DXGI any IDXGIOutput6 colour space == G2084), ANDed with the user HDR setting in session.rs; logs the SDR drop. - Apple: NSScreen.maximumExtendedDynamicRangeColorComponentValue>1 (macOS) / UIScreen.main.potentialEDRHeadroom>1 (iOS) in SessionModel. - Android: Settings.displaySupportsHdr (Display.getHdrCapabilities HDR10/HDR10+) passed through a new hdr_enabled jboolean on nativeConnect; session.rs gates the caps. Validation: Android native (incl. the jboolean gate) builds + clippy clean via cargo-ndk; fmt clean. Windows (MSVC), Apple (Swift) and the Kotlin side are CI/on-glass validated — not compilable on the Linux dev box. Deferred to the RTX box: mid-session Reconfigure SDR-downgrade on monitor move, and confirming the host emits SDR for an SDR client off an HDR desktop. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-21 09:46:58 +00:00
parent 3526517eb1
commit 551012bb43
12 changed files with 193 additions and 27 deletions
@@ -5,6 +5,12 @@ import Foundation
 import PunktfunkKit
 import SwiftUI

+#if canImport(AppKit)
+    import AppKit
+#elseif canImport(UIKit)
+    import UIKit
+#endif
+
 /// Pump-thread-side frame counters; a 1 Hz main-actor timer drains them into @Published
 /// values. NSLock instead of an actor — the writer is the (non-async) pump thread.
 final class FrameMeter: @unchecked Sendable {
@@ -93,6 +99,7 @@ final class SessionModel: ObservableObject {
                 compositor: PunktfunkConnection.Compositor = .auto,
                 gamepad: PunktfunkConnection.GamepadType = .auto,
                 bitrateKbps: UInt32 = 0,
+                 hdrEnabled: Bool = true,
                 launchID: String? = nil,
                 allowTofu: Bool = false,
                 autoTrust: Bool = false) {
@@ -101,17 +108,36 @@ final class SessionModel: ObservableObject {
        activeHost = host
        errorMessage = nil
        let pin = host.pinnedSHA256
+        // Capability gate (main-actor — screen APIs): only advertise HDR when this display can
+        // actually present it, so the host sends a proper SDR stream to an SDR display rather than
+        // BT.2020 PQ the panel would mis-tone-map. The display self-tone-maps HDR from the mastering
+        // metadata we apply (Step 2) when it IS HDR.
+        let displayHDR: Bool = {
+            #if os(macOS)
+                return (NSScreen.main?.maximumExtendedDynamicRangeColorComponentValue ?? 1.0) > 1.0
+            #else
+                return UIScreen.main.potentialEDRHeadroom > 1.0
+            #endif
+        }()
+        let hdrCapable = hdrEnabled && displayHDR
        Task.detached(priority: .userInitiated) {
            // PunktfunkConnection.init blocks on the QUIC handshake — keep it off the main
            // actor. The persistent identity is presented on every connect so a paired
            // host recognizes this Mac (nil = anonymous, fine for hosts without
            // --require-pairing; Keychain/generation failure must not block connecting).
            let identity = (try? ClientIdentityStore.shared.load())?.identity
+            // Advertise 10-bit + HDR10 when enabled: the host upgrades to a BT.2020 PQ Main10 stream
+            // only for actual HDR content (its own gate); the VideoToolbox/Metal present path is
+            // HDR-capable (P010 + itur_2100_PQ + EDR). 0 keeps the 8-bit BT.709 SDR stream.
+            let videoCaps: UInt8 = hdrCapable
+                ? (PunktfunkConnection.videoCap10Bit | PunktfunkConnection.videoCapHDR)
+                : 0
            let result = Result { try PunktfunkConnection(
                host: host.address, port: host.port,
                width: width, height: height, refreshHz: hz,
                pinSHA256: pin, identity: identity, compositor: compositor,
-                gamepad: gamepad, bitrateKbps: bitrateKbps, launchID: launchID) }
+                gamepad: gamepad, bitrateKbps: bitrateKbps, videoCaps: videoCaps,
+                launchID: launchID) }
            await MainActor.run { [weak self] in
                guard let self else { return }
                // The user may have abandoned this attempt (window closed, another host
@@ -532,6 +532,11 @@ public final class PunktfunkConnection {
        }
    }

+    /// Video-capability bit: the client can decode a 10-bit (Main10) HEVC stream.
+    public static let videoCap10Bit: UInt8 = UInt8(PUNKTFUNK_VIDEO_CAP_10BIT)
+    /// Video-capability bit: the client can present BT.2020 PQ HDR10 (implies 10-bit).
+    public static let videoCapHDR: UInt8 = UInt8(PUNKTFUNK_VIDEO_CAP_HDR)
+
    /// Static HDR mastering metadata (SMPTE ST.2086 + content light level) the host sent for an HDR
    /// session. Mirrors the wire/ABI `PunktfunkHdrMeta`; primaries are in ST.2086 **G, B, R** order,
    /// 1/50000 units; mastering luminance in 0.0001 cd/m²; MaxCLL/MaxFALL in nits.
@@ -128,6 +128,11 @@ public final class Stage2Pipeline {
                        lastFramesDropped = dropped
                        recovery.request()
                    }
+                    // Drain any HDR mastering-metadata update (0xCE) and hand it to the decoder, which
+                    // attaches it to subsequent HDR frames. Non-blocking; only HDR sessions emit these.
+                    if connection.isHDR, let meta = try? connection.nextHdrMeta(timeoutMs: 0) {
+                        decoder.setHdrMeta(meta)
+                    }
                    guard let au = try connection.nextAU(timeoutMs: 100) else { continue }
                    onFrame?(au)
                    if let f = AnnexB.formatDescription(fromIDR: au.data) {
@@ -49,6 +49,12 @@ public final class VideoDecoder: @unchecked Sendable {
    /// pump can re-gate on the next IDR.
    private let onDecodeError: @Sendable (OSStatus) -> Void

+    /// Latest source HDR mastering metadata (from `PunktfunkConnection.nextHdrMeta`), attached to
+    /// each decoded HDR pixel buffer so the compositor tone-maps from the real grade. Guarded by its
+    /// own lock — written by the pump thread, read on the VT decode callback.
+    private let metaLock = NSLock()
+    private var hdrMeta: PunktfunkConnection.HdrMeta?
+
    public init(
        onDecoded: @escaping @Sendable (ReadyFrame) -> Void,
        onDecodeError: @escaping @Sendable (OSStatus) -> Void = { _ in }
@@ -59,6 +65,14 @@ public final class VideoDecoder: @unchecked Sendable {

    deinit { teardown() }

+    /// Set the source HDR mastering metadata (drained from `PunktfunkConnection.nextHdrMeta`). It's
+    /// attached to subsequent decoded HDR pixel buffers. Thread-safe; cheap to call on each update.
+    public func setHdrMeta(_ meta: PunktfunkConnection.HdrMeta) {
+        metaLock.lock()
+        hdrMeta = meta
+        metaLock.unlock()
+    }
+
    /// Submit one AU for asynchronous decode, (re)creating the session if `format` changed. The
    /// caller resolves `format` from the IDR exactly as stage-1 does (`AnnexB.formatDescription`).
    /// Returns false if the session couldn't be created or the frame couldn't be submitted.
@@ -185,6 +199,22 @@ public final class VideoDecoder: @unchecked Sendable {
        let isHDR =
            CVPixelBufferGetPixelFormatType(imageBuffer)
            == kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange
+        // Attach the source's mastering display + content light level (ST.2086 / CEA-861.3) so the
+        // compositor tone-maps from the real grade rather than inferring from the PQ colourspace
+        // alone. The SEI byte payloads map 1:1 to these CVImageBuffer attachment keys.
+        if isHDR {
+            metaLock.lock()
+            let meta = hdrMeta
+            metaLock.unlock()
+            if let meta {
+                CVBufferSetAttachment(
+                    imageBuffer, kCVImageBufferMasteringDisplayColorVolumeKey,
+                    meta.masteringDisplayColorVolume() as CFData, .shouldPropagate)
+                CVBufferSetAttachment(
+                    imageBuffer, kCVImageBufferContentLightLevelInfoKey,
+                    meta.contentLightLevelInfo() as CFData, .shouldPropagate)
+            }
+        }
        onDecoded(
            ReadyFrame(ptsNs: ptsNs, decodedNs: decodedNs, pixelBuffer: imageBuffer, isHDR: isHDR))
    }