feat(clients): HDR Steps 2-3 — apply mastering metadata + display capability-gate
Continues docs/hdr-pipeline-plan.md. Steps 0/1 + Step 2 (Windows/Android) already landed in 3526517; this is Step 2 (Apple) + Step 3 (all clients). Client-only — no core/host/ABI change (the 0xCE/next_hdr_meta/color_info surfaces shipped in Step 0). Step 2 — clients APPLY the host's HDR metadata (each remaps from the wire form: ST.2086 G,B,R order, mastering luminance in 0.0001 cd/m2): - Apple: connect via punktfunk_connect_ex5 (resurrects the previously-dead HDR pipeline); nextHdrMeta/colorInfo wrappers + HdrMeta SEI-blob builders; the pump drains nextHdrMeta -> VideoDecoder.setHdrMeta -> CVBufferSetAttachment of MasteringDisplayColorVolume (24B BE) + ContentLightLevelInfo (4B BE) on each HDR pixel buffer (correct for the itur_2100_PQ layer; CAEDRMetadata avoided as ambiguous there). Step 3 — capability-gate: advertise HDR caps ONLY when the display can present it, so an SDR display gets a proper BT.709 stream instead of PQ it would mis-tone-map; an HDR display self-tone-maps from the Step-1/2 mastering metadata. - Windows: present::display_supports_hdr() (DXGI any IDXGIOutput6 colour space == G2084), ANDed with the user HDR setting in session.rs; logs the SDR drop. - Apple: NSScreen.maximumExtendedDynamicRangeColorComponentValue>1 (macOS) / UIScreen.main.potentialEDRHeadroom>1 (iOS) in SessionModel. - Android: Settings.displaySupportsHdr (Display.getHdrCapabilities HDR10/HDR10+) passed through a new hdr_enabled jboolean on nativeConnect; session.rs gates the caps. Validation: Android native (incl. the jboolean gate) builds + clippy clean via cargo-ndk; fmt clean. Windows (MSVC), Apple (Swift) and the Kotlin side are CI/on-glass validated — not compilable on the Linux dev box. Deferred to the RTX box: mid-session Reconfigure SDR-downgrade on monitor move, and confirming the host emits SDR for an SDR client off an HDR desktop. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -140,11 +140,15 @@ fun ConnectScreen(settings: Settings, onConnected: (Long) -> Unit) {
|
||||
status = "Connecting to $targetHost:$targetPort…"
|
||||
discovery.stop() // free the Wi-Fi radio before the stream session
|
||||
scope.launch {
|
||||
// Advertise HDR only when this device's display can present it (else the host sends a
|
||||
// proper SDR stream rather than PQ the panel would mis-tone-map).
|
||||
val hdrEnabled = displaySupportsHdr(context)
|
||||
val handle = withContext(Dispatchers.IO) {
|
||||
NativeBridge.nativeConnect(
|
||||
targetHost, targetPort, w, h, hz,
|
||||
id.certPem, id.privateKeyPem, pinHex ?: "",
|
||||
settings.bitrateKbps, settings.compositor, settings.gamepad,
|
||||
hdrEnabled,
|
||||
)
|
||||
}
|
||||
connecting = false
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package io.unom.punktfunk
|
||||
|
||||
import android.content.Context
|
||||
import android.view.Display
|
||||
|
||||
/**
|
||||
* User-tunable stream settings, persisted in `SharedPreferences`. A `0` resolution/refresh means
|
||||
@@ -76,6 +77,21 @@ fun nativeDisplayMode(context: Context): Triple<Int, Int, Int> {
|
||||
return Triple(maxOf(w, h), minOf(w, h), hz)
|
||||
}
|
||||
|
||||
/**
|
||||
* True when this device's display can actually present HDR10, so we should advertise HDR to the
|
||||
* host. On an SDR panel we advertise `0` instead — the host then sends a proper 8-bit BT.709 stream
|
||||
* rather than BT.2020 PQ the panel would mis-tone-map (the washed-out/dark failure). Mirrors the
|
||||
* capability gate the Apple/Windows clients apply.
|
||||
*/
|
||||
fun displaySupportsHdr(context: Context): Boolean {
|
||||
val display = runCatching { context.display }.getOrNull() ?: return false
|
||||
@Suppress("DEPRECATION") // hdrCapabilities is the supported query on minSdk 31
|
||||
val caps = display.hdrCapabilities ?: return false
|
||||
return caps.supportedHdrTypes.any {
|
||||
it == Display.HdrCapabilities.HDR_TYPE_HDR10 || it == Display.HdrCapabilities.HDR_TYPE_HDR10_PLUS
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve [Settings] (with its 0=native placeholders) to the concrete mode to request. */
|
||||
fun Settings.effectiveMode(context: Context): Triple<Int, Int, Int> {
|
||||
val native = nativeDisplayMode(context)
|
||||
|
||||
@@ -44,6 +44,7 @@ object NativeBridge {
|
||||
bitrateKbps: Int,
|
||||
compositorPref: Int,
|
||||
gamepadPref: Int,
|
||||
hdrEnabled: Boolean,
|
||||
): Long
|
||||
|
||||
/** 64-hex SHA-256 of the cert the host presented on [handle]; valid after a successful connect. */
|
||||
|
||||
@@ -289,10 +289,18 @@ fn android_hdr_static_info(m: &punktfunk_core::quic::HdrMeta) -> [u8; 25] {
|
||||
let max_nits = (m.max_display_mastering_luminance / 10_000).min(u16::MAX as u32) as u16;
|
||||
let min_units = m.min_display_mastering_luminance.min(u16::MAX as u32) as u16;
|
||||
let fields: [u16; 12] = [
|
||||
r[0], r[1], g[0], g[1], b_[0], b_[1], // R, G, B primaries
|
||||
m.white_point[0], m.white_point[1], // white point
|
||||
max_nits, min_units, // max (nits) / min (0.0001-nit) display luminance
|
||||
m.max_cll, m.max_fall, // MaxCLL / MaxFALL (nits)
|
||||
r[0],
|
||||
r[1],
|
||||
g[0],
|
||||
g[1],
|
||||
b_[0],
|
||||
b_[1], // R, G, B primaries
|
||||
m.white_point[0],
|
||||
m.white_point[1], // white point
|
||||
max_nits,
|
||||
min_units, // max (nits) / min (0.0001-nit) display luminance
|
||||
m.max_cll,
|
||||
m.max_fall, // MaxCLL / MaxFALL (nits)
|
||||
];
|
||||
let mut out = [0u8; 25]; // out[0] = 0 (Type 1 descriptor id), already zero
|
||||
for (i, v) in fields.iter().enumerate() {
|
||||
|
||||
@@ -144,6 +144,7 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo
|
||||
bitrate_kbps: jint,
|
||||
compositor_pref: jint,
|
||||
gamepad_pref: jint,
|
||||
hdr_enabled: jboolean,
|
||||
) -> jlong {
|
||||
let host: String = match env.get_string(&host) {
|
||||
Ok(s) => s.into(),
|
||||
@@ -184,10 +185,17 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo
|
||||
CompositorPref::from_u8(compositor_pref.clamp(0, u8::MAX as jint) as u8),
|
||||
GamepadPref::from_u8(gamepad_pref.clamp(0, u8::MAX as jint) as u8),
|
||||
bitrate_kbps.max(0) as u32, // 0 = host default
|
||||
// Advertise 10-bit + HDR: the host (e.g. Windows) only upgrades to a Main10 / BT.2020 PQ
|
||||
// encode when the client sets these. AMediaCodec decodes Main10 from the SPS and the decode
|
||||
// loop signals the Surface's HDR dataspace from the reported colour (see crate::decode).
|
||||
punktfunk_core::quic::VIDEO_CAP_10BIT | punktfunk_core::quic::VIDEO_CAP_HDR,
|
||||
// Advertise 10-bit + HDR ONLY when this device's display can actually present it (Kotlin
|
||||
// checks Display.getHdrCapabilities() and passes the result): the host (e.g. Windows) then
|
||||
// upgrades to a Main10 / BT.2020 PQ encode. On an SDR display we advertise 0 so the host
|
||||
// sends a proper 8-bit BT.709 stream rather than PQ the panel would mis-tone-map. AMediaCodec
|
||||
// decodes Main10 from the SPS and the decode loop signals the Surface HDR dataspace + static
|
||||
// metadata (see crate::decode).
|
||||
if hdr_enabled != 0 {
|
||||
punktfunk_core::quic::VIDEO_CAP_10BIT | punktfunk_core::quic::VIDEO_CAP_HDR
|
||||
} else {
|
||||
0
|
||||
},
|
||||
None, // launch: default app
|
||||
pin, // Some → Crypto on host-fp mismatch
|
||||
identity, // owned (cert, key) PEM, or None (anonymous)
|
||||
|
||||
@@ -5,6 +5,12 @@ import Foundation
|
||||
import PunktfunkKit
|
||||
import SwiftUI
|
||||
|
||||
#if canImport(AppKit)
|
||||
import AppKit
|
||||
#elseif canImport(UIKit)
|
||||
import UIKit
|
||||
#endif
|
||||
|
||||
/// Pump-thread-side frame counters; a 1 Hz main-actor timer drains them into @Published
|
||||
/// values. NSLock instead of an actor — the writer is the (non-async) pump thread.
|
||||
final class FrameMeter: @unchecked Sendable {
|
||||
@@ -93,6 +99,7 @@ final class SessionModel: ObservableObject {
|
||||
compositor: PunktfunkConnection.Compositor = .auto,
|
||||
gamepad: PunktfunkConnection.GamepadType = .auto,
|
||||
bitrateKbps: UInt32 = 0,
|
||||
hdrEnabled: Bool = true,
|
||||
launchID: String? = nil,
|
||||
allowTofu: Bool = false,
|
||||
autoTrust: Bool = false) {
|
||||
@@ -101,17 +108,36 @@ final class SessionModel: ObservableObject {
|
||||
activeHost = host
|
||||
errorMessage = nil
|
||||
let pin = host.pinnedSHA256
|
||||
// Capability gate (main-actor — screen APIs): only advertise HDR when this display can
|
||||
// actually present it, so the host sends a proper SDR stream to an SDR display rather than
|
||||
// BT.2020 PQ the panel would mis-tone-map. The display self-tone-maps HDR from the mastering
|
||||
// metadata we apply (Step 2) when it IS HDR.
|
||||
let displayHDR: Bool = {
|
||||
#if os(macOS)
|
||||
return (NSScreen.main?.maximumExtendedDynamicRangeColorComponentValue ?? 1.0) > 1.0
|
||||
#else
|
||||
return UIScreen.main.potentialEDRHeadroom > 1.0
|
||||
#endif
|
||||
}()
|
||||
let hdrCapable = hdrEnabled && displayHDR
|
||||
Task.detached(priority: .userInitiated) {
|
||||
// PunktfunkConnection.init blocks on the QUIC handshake — keep it off the main
|
||||
// actor. The persistent identity is presented on every connect so a paired
|
||||
// host recognizes this Mac (nil = anonymous, fine for hosts without
|
||||
// --require-pairing; Keychain/generation failure must not block connecting).
|
||||
let identity = (try? ClientIdentityStore.shared.load())?.identity
|
||||
// Advertise 10-bit + HDR10 when enabled: the host upgrades to a BT.2020 PQ Main10 stream
|
||||
// only for actual HDR content (its own gate); the VideoToolbox/Metal present path is
|
||||
// HDR-capable (P010 + itur_2100_PQ + EDR). 0 keeps the 8-bit BT.709 SDR stream.
|
||||
let videoCaps: UInt8 = hdrCapable
|
||||
? (PunktfunkConnection.videoCap10Bit | PunktfunkConnection.videoCapHDR)
|
||||
: 0
|
||||
let result = Result { try PunktfunkConnection(
|
||||
host: host.address, port: host.port,
|
||||
width: width, height: height, refreshHz: hz,
|
||||
pinSHA256: pin, identity: identity, compositor: compositor,
|
||||
gamepad: gamepad, bitrateKbps: bitrateKbps, launchID: launchID) }
|
||||
gamepad: gamepad, bitrateKbps: bitrateKbps, videoCaps: videoCaps,
|
||||
launchID: launchID) }
|
||||
await MainActor.run { [weak self] in
|
||||
guard let self else { return }
|
||||
// The user may have abandoned this attempt (window closed, another host
|
||||
|
||||
@@ -532,6 +532,11 @@ public final class PunktfunkConnection {
|
||||
}
|
||||
}
|
||||
|
||||
/// Video-capability bit: the client can decode a 10-bit (Main10) HEVC stream.
|
||||
public static let videoCap10Bit: UInt8 = UInt8(PUNKTFUNK_VIDEO_CAP_10BIT)
|
||||
/// Video-capability bit: the client can present BT.2020 PQ HDR10 (implies 10-bit).
|
||||
public static let videoCapHDR: UInt8 = UInt8(PUNKTFUNK_VIDEO_CAP_HDR)
|
||||
|
||||
/// Static HDR mastering metadata (SMPTE ST.2086 + content light level) the host sent for an HDR
|
||||
/// session. Mirrors the wire/ABI `PunktfunkHdrMeta`; primaries are in ST.2086 **G, B, R** order,
|
||||
/// 1/50000 units; mastering luminance in 0.0001 cd/m²; MaxCLL/MaxFALL in nits.
|
||||
|
||||
@@ -128,6 +128,11 @@ public final class Stage2Pipeline {
|
||||
lastFramesDropped = dropped
|
||||
recovery.request()
|
||||
}
|
||||
// Drain any HDR mastering-metadata update (0xCE) and hand it to the decoder, which
|
||||
// attaches it to subsequent HDR frames. Non-blocking; only HDR sessions emit these.
|
||||
if connection.isHDR, let meta = try? connection.nextHdrMeta(timeoutMs: 0) {
|
||||
decoder.setHdrMeta(meta)
|
||||
}
|
||||
guard let au = try connection.nextAU(timeoutMs: 100) else { continue }
|
||||
onFrame?(au)
|
||||
if let f = AnnexB.formatDescription(fromIDR: au.data) {
|
||||
|
||||
@@ -49,6 +49,12 @@ public final class VideoDecoder: @unchecked Sendable {
|
||||
/// pump can re-gate on the next IDR.
|
||||
private let onDecodeError: @Sendable (OSStatus) -> Void
|
||||
|
||||
/// Latest source HDR mastering metadata (from `PunktfunkConnection.nextHdrMeta`), attached to
|
||||
/// each decoded HDR pixel buffer so the compositor tone-maps from the real grade. Guarded by its
|
||||
/// own lock — written by the pump thread, read on the VT decode callback.
|
||||
private let metaLock = NSLock()
|
||||
private var hdrMeta: PunktfunkConnection.HdrMeta?
|
||||
|
||||
public init(
|
||||
onDecoded: @escaping @Sendable (ReadyFrame) -> Void,
|
||||
onDecodeError: @escaping @Sendable (OSStatus) -> Void = { _ in }
|
||||
@@ -59,6 +65,14 @@ public final class VideoDecoder: @unchecked Sendable {
|
||||
|
||||
deinit { teardown() }
|
||||
|
||||
/// Set the source HDR mastering metadata (drained from `PunktfunkConnection.nextHdrMeta`). It's
|
||||
/// attached to subsequent decoded HDR pixel buffers. Thread-safe; cheap to call on each update.
|
||||
public func setHdrMeta(_ meta: PunktfunkConnection.HdrMeta) {
|
||||
metaLock.lock()
|
||||
hdrMeta = meta
|
||||
metaLock.unlock()
|
||||
}
|
||||
|
||||
/// Submit one AU for asynchronous decode, (re)creating the session if `format` changed. The
|
||||
/// caller resolves `format` from the IDR exactly as stage-1 does (`AnnexB.formatDescription`).
|
||||
/// Returns false if the session couldn't be created or the frame couldn't be submitted.
|
||||
@@ -185,6 +199,22 @@ public final class VideoDecoder: @unchecked Sendable {
|
||||
let isHDR =
|
||||
CVPixelBufferGetPixelFormatType(imageBuffer)
|
||||
== kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange
|
||||
// Attach the source's mastering display + content light level (ST.2086 / CEA-861.3) so the
|
||||
// compositor tone-maps from the real grade rather than inferring from the PQ colourspace
|
||||
// alone. The SEI byte payloads map 1:1 to these CVImageBuffer attachment keys.
|
||||
if isHDR {
|
||||
metaLock.lock()
|
||||
let meta = hdrMeta
|
||||
metaLock.unlock()
|
||||
if let meta {
|
||||
CVBufferSetAttachment(
|
||||
imageBuffer, kCVImageBufferMasteringDisplayColorVolumeKey,
|
||||
meta.masteringDisplayColorVolume() as CFData, .shouldPropagate)
|
||||
CVBufferSetAttachment(
|
||||
imageBuffer, kCVImageBufferContentLightLevelInfoKey,
|
||||
meta.contentLightLevelInfo() as CFData, .shouldPropagate)
|
||||
}
|
||||
}
|
||||
onDecoded(
|
||||
ReadyFrame(ptsNs: ptsNs, decodedNs: decodedNs, pixelBuffer: imageBuffer, isHDR: isHDR))
|
||||
}
|
||||
|
||||
@@ -619,6 +619,36 @@ fn blob_bytes(blob: &ID3DBlob) -> &[u8] {
|
||||
}
|
||||
}
|
||||
|
||||
/// True if any attached display is currently in HDR (BT.2020 PQ) mode. The client advertises HDR
|
||||
/// caps only when this holds, so an SDR display gets a proper 8-bit BT.709 stream instead of PQ it
|
||||
/// would mis-tone-map (the washed-out/dark failure); an HDR display self-tone-maps from the
|
||||
/// mastering metadata. Coarse — checks ANY output, not the app's specific monitor; a mid-session
|
||||
/// monitor move to/from HDR is a follow-up (the `Reconfigure` downgrade).
|
||||
pub fn display_supports_hdr() -> bool {
|
||||
unsafe {
|
||||
let factory: IDXGIFactory1 = match CreateDXGIFactory1() {
|
||||
Ok(f) => f,
|
||||
Err(_) => return false,
|
||||
};
|
||||
let mut ai = 0u32;
|
||||
while let Ok(adapter) = factory.EnumAdapters1(ai) {
|
||||
ai += 1;
|
||||
let mut oi = 0u32;
|
||||
while let Ok(output) = adapter.EnumOutputs(oi) {
|
||||
oi += 1;
|
||||
if let Ok(o6) = output.cast::<IDXGIOutput6>() {
|
||||
if let Ok(desc) = o6.GetDesc1() {
|
||||
if desc.ColorSpace == DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020 {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Generic HDR10 mastering metadata: BT.2020 primaries + D65 white, a 1000-nit mastering display,
|
||||
/// MaxCLL 1000 / MaxFALL 400. The fallback used only until the host's real `0xCE` metadata arrives.
|
||||
fn generic_hdr10_metadata() -> DXGI_HDR_METADATA_HDR10 {
|
||||
|
||||
@@ -107,13 +107,19 @@ fn pump(
|
||||
params.compositor,
|
||||
params.gamepad,
|
||||
params.bitrate_kbps,
|
||||
// Advertise 10-bit + HDR10 (when enabled): the presenter handles BT.2020 PQ frames (P010 on
|
||||
// the GPU path, X2BGR10 on software), so the host may upgrade HDR content to a Main10/PQ
|
||||
// stream — it still only does so for actual HDR content with its own 10-bit gate. 8-bit SDR
|
||||
// is unaffected. A client that turns HDR off advertises `0` and always gets the 8-bit stream.
|
||||
if params.hdr_enabled {
|
||||
// Advertise 10-bit + HDR10 only when the user enabled HDR AND a display is actually in HDR
|
||||
// mode: the host then upgrades HDR content to a Main10/PQ stream (its own 10-bit gate still
|
||||
// applies). On an SDR display we advertise `0` so the host sends a proper 8-bit BT.709 stream
|
||||
// rather than PQ the panel would mis-tone-map (washed-out/dark). An HDR display self-tone-maps
|
||||
// from the mastering metadata we apply. The presenter handles BT.2020 PQ frames (P010 / X2BGR10).
|
||||
if params.hdr_enabled && crate::present::display_supports_hdr() {
|
||||
punktfunk_core::quic::VIDEO_CAP_10BIT | punktfunk_core::quic::VIDEO_CAP_HDR
|
||||
} else {
|
||||
if params.hdr_enabled {
|
||||
tracing::info!(
|
||||
"HDR enabled in settings but no HDR display detected — requesting SDR"
|
||||
);
|
||||
}
|
||||
0
|
||||
},
|
||||
None, // launch: the Windows client has no library picker yet
|
||||
|
||||
Reference in New Issue
Block a user