diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectScreen.kt index 031ec23..fe23bb6 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/ConnectScreen.kt @@ -140,11 +140,15 @@ fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) { status = "Connecting to $targetHost:$targetPort…" discovery.stop() // free the Wi-Fi radio before the stream session scope.launch { + // Advertise HDR only when this device's display can present it (else the host sends a + // proper SDR stream rather than PQ the panel would mis-tone-map). + val hdrEnabled = displaySupportsHdr(context) val handle = withContext(Dispatchers.IO) { NativeBridge.nativeConnect( targetHost, targetPort, w, h, hz, id.certPem, id.privateKeyPem, pinHex ?: "", settings.bitrateKbps, settings.compositor, settings.gamepad, + hdrEnabled, ) } connecting = false diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt index 5e7b384..a904068 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/Settings.kt @@ -1,6 +1,7 @@ package io.unom.punktfunk import android.content.Context +import android.view.Display /** * User-tunable stream settings, persisted in `SharedPreferences`. A `0` resolution/refresh means @@ -76,6 +77,21 @@ fun nativeDisplayMode(context: Context): Triple { return Triple(maxOf(w, h), minOf(w, h), hz) } +/** + * True when this device's display can actually present HDR10, so we should advertise HDR to the + * host. On an SDR panel we advertise `0` instead — the host then sends a proper 8-bit BT.709 stream + * rather than BT.2020 PQ the panel would mis-tone-map (the washed-out/dark failure). Mirrors the + * capability gate the Apple/Windows clients apply. + */ +fun displaySupportsHdr(context: Context): Boolean { + val display = runCatching { context.display }.getOrNull() ?: return false + @Suppress("DEPRECATION") // hdrCapabilities is the supported query on minSdk 31 + val caps = display.hdrCapabilities ?: return false + return caps.supportedHdrTypes.any { + it == Display.HdrCapabilities.HDR_TYPE_HDR10 || it == Display.HdrCapabilities.HDR_TYPE_HDR10_PLUS + } +} + /** Resolve [Settings] (with its 0=native placeholders) to the concrete mode to request. */ fun Settings.effectiveMode(context: Context): Triple { val native = nativeDisplayMode(context) diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index 6b4f65a..ee67b7c 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -44,6 +44,7 @@ object NativeBridge { bitrateKbps: Int, compositorPref: Int, gamepadPref: Int, + hdrEnabled: Boolean, ): Long /** 64-hex SHA-256 of the cert the host presented on [handle]; valid after a successful connect. */ diff --git a/clients/android/native/src/decode.rs b/clients/android/native/src/decode.rs index 00b7985..f483956 100644 --- a/clients/android/native/src/decode.rs +++ b/clients/android/native/src/decode.rs @@ -289,10 +289,18 @@ fn android_hdr_static_info(m: &punktfunk_core::quic::HdrMeta) -> [u8; 25] { let max_nits = (m.max_display_mastering_luminance / 10_000).min(u16::MAX as u32) as u16; let min_units = m.min_display_mastering_luminance.min(u16::MAX as u32) as u16; let fields: [u16; 12] = [ - r[0], r[1], g[0], g[1], b_[0], b_[1], // R, G, B primaries - m.white_point[0], m.white_point[1], // white point - max_nits, min_units, // max (nits) / min (0.0001-nit) display luminance - m.max_cll, m.max_fall, // MaxCLL / MaxFALL (nits) + r[0], + r[1], + g[0], + g[1], + b_[0], + b_[1], // R, G, B primaries + m.white_point[0], + m.white_point[1], // white point + max_nits, + min_units, // max (nits) / min (0.0001-nit) display luminance + m.max_cll, + m.max_fall, // MaxCLL / MaxFALL (nits) ]; let mut out = [0u8; 25]; // out[0] = 0 (Type 1 descriptor id), already zero for (i, v) in fields.iter().enumerate() { diff --git a/clients/android/native/src/session.rs b/clients/android/native/src/session.rs index 2246e1e..377f869 100644 --- a/clients/android/native/src/session.rs +++ b/clients/android/native/src/session.rs @@ -144,6 +144,7 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo bitrate_kbps: jint, compositor_pref: jint, gamepad_pref: jint, + hdr_enabled: jboolean, ) -> jlong { let host: String = match env.get_string(&host) { Ok(s) => s.into(), @@ -184,10 +185,17 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo CompositorPref::from_u8(compositor_pref.clamp(0, u8::MAX as jint) as u8), GamepadPref::from_u8(gamepad_pref.clamp(0, u8::MAX as jint) as u8), bitrate_kbps.max(0) as u32, // 0 = host default - // Advertise 10-bit + HDR: the host (e.g. Windows) only upgrades to a Main10 / BT.2020 PQ - // encode when the client sets these. AMediaCodec decodes Main10 from the SPS and the decode - // loop signals the Surface's HDR dataspace from the reported colour (see crate::decode). - punktfunk_core::quic::VIDEO_CAP_10BIT | punktfunk_core::quic::VIDEO_CAP_HDR, + // Advertise 10-bit + HDR ONLY when this device's display can actually present it (Kotlin + // checks Display.getHdrCapabilities() and passes the result): the host (e.g. Windows) then + // upgrades to a Main10 / BT.2020 PQ encode. On an SDR display we advertise 0 so the host + // sends a proper 8-bit BT.709 stream rather than PQ the panel would mis-tone-map. AMediaCodec + // decodes Main10 from the SPS and the decode loop signals the Surface HDR dataspace + static + // metadata (see crate::decode). + if hdr_enabled != 0 { + punktfunk_core::quic::VIDEO_CAP_10BIT | punktfunk_core::quic::VIDEO_CAP_HDR + } else { + 0 + }, None, // launch: default app pin, // Some → Crypto on host-fp mismatch identity, // owned (cert, key) PEM, or None (anonymous) diff --git a/clients/apple/Sources/PunktfunkClient/SessionModel.swift b/clients/apple/Sources/PunktfunkClient/SessionModel.swift index 5ba2042..1949da5 100644 --- a/clients/apple/Sources/PunktfunkClient/SessionModel.swift +++ b/clients/apple/Sources/PunktfunkClient/SessionModel.swift @@ -5,6 +5,12 @@ import Foundation import PunktfunkKit import SwiftUI +#if canImport(AppKit) + import AppKit +#elseif canImport(UIKit) + import UIKit +#endif + /// Pump-thread-side frame counters; a 1 Hz main-actor timer drains them into @Published /// values. NSLock instead of an actor — the writer is the (non-async) pump thread. final class FrameMeter: @unchecked Sendable { @@ -93,6 +99,7 @@ final class SessionModel: ObservableObject { compositor: PunktfunkConnection.Compositor = .auto, gamepad: PunktfunkConnection.GamepadType = .auto, bitrateKbps: UInt32 = 0, + hdrEnabled: Bool = true, launchID: String? = nil, allowTofu: Bool = false, autoTrust: Bool = false) { @@ -101,17 +108,36 @@ final class SessionModel: ObservableObject { activeHost = host errorMessage = nil let pin = host.pinnedSHA256 + // Capability gate (main-actor — screen APIs): only advertise HDR when this display can + // actually present it, so the host sends a proper SDR stream to an SDR display rather than + // BT.2020 PQ the panel would mis-tone-map. The display self-tone-maps HDR from the mastering + // metadata we apply (Step 2) when it IS HDR. + let displayHDR: Bool = { + #if os(macOS) + return (NSScreen.main?.maximumExtendedDynamicRangeColorComponentValue ?? 1.0) > 1.0 + #else + return UIScreen.main.potentialEDRHeadroom > 1.0 + #endif + }() + let hdrCapable = hdrEnabled && displayHDR Task.detached(priority: .userInitiated) { // PunktfunkConnection.init blocks on the QUIC handshake — keep it off the main // actor. The persistent identity is presented on every connect so a paired // host recognizes this Mac (nil = anonymous, fine for hosts without // --require-pairing; Keychain/generation failure must not block connecting). let identity = (try? ClientIdentityStore.shared.load())?.identity + // Advertise 10-bit + HDR10 when enabled: the host upgrades to a BT.2020 PQ Main10 stream + // only for actual HDR content (its own gate); the VideoToolbox/Metal present path is + // HDR-capable (P010 + itur_2100_PQ + EDR). 0 keeps the 8-bit BT.709 SDR stream. + let videoCaps: UInt8 = hdrCapable + ? (PunktfunkConnection.videoCap10Bit | PunktfunkConnection.videoCapHDR) + : 0 let result = Result { try PunktfunkConnection( host: host.address, port: host.port, width: width, height: height, refreshHz: hz, pinSHA256: pin, identity: identity, compositor: compositor, - gamepad: gamepad, bitrateKbps: bitrateKbps, launchID: launchID) } + gamepad: gamepad, bitrateKbps: bitrateKbps, videoCaps: videoCaps, + launchID: launchID) } await MainActor.run { [weak self] in guard let self else { return } // The user may have abandoned this attempt (window closed, another host diff --git a/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift b/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift index ed6cd4a..8fe0d40 100644 --- a/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift +++ b/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift @@ -532,6 +532,11 @@ public final class PunktfunkConnection { } } + /// Video-capability bit: the client can decode a 10-bit (Main10) HEVC stream. + public static let videoCap10Bit: UInt8 = UInt8(PUNKTFUNK_VIDEO_CAP_10BIT) + /// Video-capability bit: the client can present BT.2020 PQ HDR10 (implies 10-bit). + public static let videoCapHDR: UInt8 = UInt8(PUNKTFUNK_VIDEO_CAP_HDR) + /// Static HDR mastering metadata (SMPTE ST.2086 + content light level) the host sent for an HDR /// session. Mirrors the wire/ABI `PunktfunkHdrMeta`; primaries are in ST.2086 **G, B, R** order, /// 1/50000 units; mastering luminance in 0.0001 cd/m²; MaxCLL/MaxFALL in nits. diff --git a/clients/apple/Sources/PunktfunkKit/Stage2Pipeline.swift b/clients/apple/Sources/PunktfunkKit/Stage2Pipeline.swift index 728740a..18ed298 100644 --- a/clients/apple/Sources/PunktfunkKit/Stage2Pipeline.swift +++ b/clients/apple/Sources/PunktfunkKit/Stage2Pipeline.swift @@ -128,6 +128,11 @@ public final class Stage2Pipeline { lastFramesDropped = dropped recovery.request() } + // Drain any HDR mastering-metadata update (0xCE) and hand it to the decoder, which + // attaches it to subsequent HDR frames. Non-blocking; only HDR sessions emit these. + if connection.isHDR, let meta = try? connection.nextHdrMeta(timeoutMs: 0) { + decoder.setHdrMeta(meta) + } guard let au = try connection.nextAU(timeoutMs: 100) else { continue } onFrame?(au) if let f = AnnexB.formatDescription(fromIDR: au.data) { diff --git a/clients/apple/Sources/PunktfunkKit/VideoDecoder.swift b/clients/apple/Sources/PunktfunkKit/VideoDecoder.swift index 41ad45b..fa84521 100644 --- a/clients/apple/Sources/PunktfunkKit/VideoDecoder.swift +++ b/clients/apple/Sources/PunktfunkKit/VideoDecoder.swift @@ -49,6 +49,12 @@ public final class VideoDecoder: @unchecked Sendable { /// pump can re-gate on the next IDR. private let onDecodeError: @Sendable (OSStatus) -> Void + /// Latest source HDR mastering metadata (from `PunktfunkConnection.nextHdrMeta`), attached to + /// each decoded HDR pixel buffer so the compositor tone-maps from the real grade. Guarded by its + /// own lock — written by the pump thread, read on the VT decode callback. + private let metaLock = NSLock() + private var hdrMeta: PunktfunkConnection.HdrMeta? + public init( onDecoded: @escaping @Sendable (ReadyFrame) -> Void, onDecodeError: @escaping @Sendable (OSStatus) -> Void = { _ in } @@ -59,6 +65,14 @@ public final class VideoDecoder: @unchecked Sendable { deinit { teardown() } + /// Set the source HDR mastering metadata (drained from `PunktfunkConnection.nextHdrMeta`). It's + /// attached to subsequent decoded HDR pixel buffers. Thread-safe; cheap to call on each update. + public func setHdrMeta(_ meta: PunktfunkConnection.HdrMeta) { + metaLock.lock() + hdrMeta = meta + metaLock.unlock() + } + /// Submit one AU for asynchronous decode, (re)creating the session if `format` changed. The /// caller resolves `format` from the IDR exactly as stage-1 does (`AnnexB.formatDescription`). /// Returns false if the session couldn't be created or the frame couldn't be submitted. @@ -185,6 +199,22 @@ public final class VideoDecoder: @unchecked Sendable { let isHDR = CVPixelBufferGetPixelFormatType(imageBuffer) == kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange + // Attach the source's mastering display + content light level (ST.2086 / CEA-861.3) so the + // compositor tone-maps from the real grade rather than inferring from the PQ colourspace + // alone. The SEI byte payloads map 1:1 to these CVImageBuffer attachment keys. + if isHDR { + metaLock.lock() + let meta = hdrMeta + metaLock.unlock() + if let meta { + CVBufferSetAttachment( + imageBuffer, kCVImageBufferMasteringDisplayColorVolumeKey, + meta.masteringDisplayColorVolume() as CFData, .shouldPropagate) + CVBufferSetAttachment( + imageBuffer, kCVImageBufferContentLightLevelInfoKey, + meta.contentLightLevelInfo() as CFData, .shouldPropagate) + } + } onDecoded( ReadyFrame(ptsNs: ptsNs, decodedNs: decodedNs, pixelBuffer: imageBuffer, isHDR: isHDR)) } diff --git a/clients/windows/src/present.rs b/clients/windows/src/present.rs index 4ad8630..1397550 100644 --- a/clients/windows/src/present.rs +++ b/clients/windows/src/present.rs @@ -619,6 +619,36 @@ fn blob_bytes(blob: &ID3DBlob) -> &[u8] { } } +/// True if any attached display is currently in HDR (BT.2020 PQ) mode. The client advertises HDR +/// caps only when this holds, so an SDR display gets a proper 8-bit BT.709 stream instead of PQ it +/// would mis-tone-map (the washed-out/dark failure); an HDR display self-tone-maps from the +/// mastering metadata. Coarse — checks ANY output, not the app's specific monitor; a mid-session +/// monitor move to/from HDR is a follow-up (the `Reconfigure` downgrade). +pub fn display_supports_hdr() -> bool { + unsafe { + let factory: IDXGIFactory1 = match CreateDXGIFactory1() { + Ok(f) => f, + Err(_) => return false, + }; + let mut ai = 0u32; + while let Ok(adapter) = factory.EnumAdapters1(ai) { + ai += 1; + let mut oi = 0u32; + while let Ok(output) = adapter.EnumOutputs(oi) { + oi += 1; + if let Ok(o6) = output.cast::() { + if let Ok(desc) = o6.GetDesc1() { + if desc.ColorSpace == DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020 { + return true; + } + } + } + } + } + } + false +} + /// Generic HDR10 mastering metadata: BT.2020 primaries + D65 white, a 1000-nit mastering display, /// MaxCLL 1000 / MaxFALL 400. The fallback used only until the host's real `0xCE` metadata arrives. fn generic_hdr10_metadata() -> DXGI_HDR_METADATA_HDR10 { diff --git a/clients/windows/src/session.rs b/clients/windows/src/session.rs index 609c2b1..c0f2c2e 100644 --- a/clients/windows/src/session.rs +++ b/clients/windows/src/session.rs @@ -107,13 +107,19 @@ fn pump( params.compositor, params.gamepad, params.bitrate_kbps, - // Advertise 10-bit + HDR10 (when enabled): the presenter handles BT.2020 PQ frames (P010 on - // the GPU path, X2BGR10 on software), so the host may upgrade HDR content to a Main10/PQ - // stream — it still only does so for actual HDR content with its own 10-bit gate. 8-bit SDR - // is unaffected. A client that turns HDR off advertises `0` and always gets the 8-bit stream. - if params.hdr_enabled { + // Advertise 10-bit + HDR10 only when the user enabled HDR AND a display is actually in HDR + // mode: the host then upgrades HDR content to a Main10/PQ stream (its own 10-bit gate still + // applies). On an SDR display we advertise `0` so the host sends a proper 8-bit BT.709 stream + // rather than PQ the panel would mis-tone-map (washed-out/dark). An HDR display self-tone-maps + // from the mastering metadata we apply. The presenter handles BT.2020 PQ frames (P010 / X2BGR10). + if params.hdr_enabled && crate::present::display_supports_hdr() { punktfunk_core::quic::VIDEO_CAP_10BIT | punktfunk_core::quic::VIDEO_CAP_HDR } else { + if params.hdr_enabled { + tracing::info!( + "HDR enabled in settings but no HDR display detected — requesting SDR" + ); + } 0 }, None, // launch: the Windows client has no library picker yet diff --git a/docs/hdr-pipeline-plan.md b/docs/hdr-pipeline-plan.md index 4b40d44..34497be 100644 --- a/docs/hdr-pipeline-plan.md +++ b/docs/hdr-pipeline-plan.md @@ -186,20 +186,47 @@ the protocol, and gives an Apollo/Moonlight on-glass parity gate. display's real luminance and VUI 9/16/9; stock Moonlight shows correct (not washed-out) HDR. Add **encoder-CSC-range == signaled-range** check. -### Step 2 — Clients apply the metadata (Windows + Apple + Android, parallelizable) -- **Windows:** feed `hdr10_metadata()` from the received `HdrMeta` (drop the hardcode); **log** - `SetColorSpace1`/`SetHDRMetaData` failures. -- **Apple:** attach `kCVImageBufferMasteringDisplayColorVolumeKey` + `ContentLightLevelInfoKey` - / `CAEDRMetadata` from `HdrMeta`; CV color attachments from Welcome. -- **Android:** set `MediaFormat KEY_HDR_STATIC_INFO` from `HdrMeta`. +### Step 2 — Clients apply the metadata *(landed; CI/on-glass validation pending)* +All three clients now drain the protocol's `HdrMeta` (`next_hdr_meta` / `nextHdrMeta`) and apply it, +each remapping from the wire form (ST.2086 G,B,R order, mastering luminance in 0.0001 cd/m²) to the +platform's expected layout: +- **Windows (Rust, CI-compiled):** session pump drains `next_hdr_meta` into a `LATEST_HDR_META` + slot; `present_newest` applies it via `Presenter::set_hdr_metadata` → real `SetHDRMetaData` + (`hdr_meta_to_dxgi`: G,B,R→R,G,B reorder, 0.0001-nit→nit for `MaxMasteringLuminance`), dropping + the 1000/1000/400 hardcode. `SetColorSpace1`/`SetHDRMetaData` failures + an SDR-display + colour-space rejection are now **logged**, not swallowed. +- **Apple (Swift, mac-runner CI):** connect now advertises caps via `punktfunk_connect_ex5` + (`SessionModel` computes `videoCap10Bit|videoCapHDR` from `hdrEnabled`) — *this is the fix that + resurrects Apple's previously-dead HDR pipeline*. `nextHdrMeta`/`colorInfo` wrappers added; the + pump drains `nextHdrMeta` → `VideoDecoder.setHdrMeta` → `CVBufferSetAttachment` of + `kCVImageBufferMasteringDisplayColorVolumeKey` (24-byte BE SEI) + + `kCVImageBufferContentLightLevelInfoKey` (4-byte BE) on each HDR pixel buffer (the correct path + for the itur_2100_PQ layer; `CAEDRMetadata` on a PQ layer is ambiguous and was avoided). +- **Android (Rust `decode.rs`, cargo-ndk verified):** when `client.color.is_hdr()`, drain the first + `next_hdr_meta` and set `MediaFormat` `hdr-static-info` (`KEY_HDR_STATIC_INFO`) before + `configure()` — `android_hdr_static_info` builds the 25-byte CTA-861.3 Type-1 blob (LE, **R,G,B** + order, max-lum in **nits-u16**). `Display.getHdrCapabilities` gate deferred (the Surface DataSpace + already drives SurfaceFlinger tone-mapping on non-HDR displays). -### Step 3 — Display-capability query + client tone-mapping + robust fallback -The common-case correctness step — most displays are SDR. - -- **HDR→SDR on every client** (defined BT.2390 EETF / Hable), not silent OS fallback. -- Content-peak > display-peak roll-off (`GetDesc1` / `NSScreen.maximumEDR…` / - `Display.getHdrCapabilities`); explicit SDR fallback when HDR present fails. -- Optional client→host "send me SDR" downgrade as a trailing field on `Reconfigure`. +### Step 3 — Display-capability gate *(landed; CI/on-glass validation pending)* +The common-case correctness step — most client displays are SDR. **Chosen approach: capability-gate** +(not an in-shader BT.2390 tone-map). Rationale: with Steps 1–2 the host sends *correct* mastering +metadata, so an HDR display self-tone-maps from it; the real remaining gap is SDR displays, best +fixed by **not advertising HDR you can't present** — the host then sends a proper BT.709 SDR stream +instead of PQ the panel would mis-tone-map (washed-out/dark). No guessed tone-map curve, deterministic. +- **Windows** (`present::display_supports_hdr` via DXGI: any `IDXGIOutput6` colour space == + `G2084`): `session.rs` ANDs it with the user's HDR setting before advertising caps; logs when it + drops to SDR. +- **Apple** (`SessionModel`, main-actor): `NSScreen.maximumExtendedDynamicRangeColorComponentValue + > 1` (macOS) / `UIScreen.main.potentialEDRHeadroom > 1` (iOS) ANDed with `hdrEnabled`. +- **Android** (`Settings.displaySupportsHdr` via `Display.getHdrCapabilities` HDR10/HDR10+): Kotlin + passes it to `nativeConnect`; `session.rs` gates the caps on the new `hdr_enabled` jboolean + (cargo-ndk-verified). +- **Deferred** (need on-glass / the RTX box): the **mid-session `Reconfigure` "downgrade to SDR"** + for a monitor move HDR↔SDR; and confirming the **host produces SDR for an SDR client even off an + HDR desktop** — on the native path the per-session SudoVDA follows the negotiated depth (SDR + client → SDR virtual display → SDR stream), so it should hold end-to-end; verify the + stale-HDR-SudoVDA edge case on the RTX box. ### Step 4 — Linux (last; capture blocked upstream) - **8-bit→Main10 NVENC upconvert shim** (`encode/linux.rs`) — Main10 transport with correct