enricobuehler
1fcf9e11ec
Clients derive Y'CbCr->RGB from the stream's SIGNALED matrix x range x depth via shared csc rows (Rust csc_rows + Swift CscRows) instead of hardcoded 709/2020 - a BT.601-signaled stream (a Linux host's RGB-input NVENC) no longer renders with a constant hue error. Host-side signaling made honest across NVENC/VAAPI/openh264/GameStream and the session plan's chroma/bit-depth. Decoded color-bar fixtures (601/709 x limited/full) pin the math in tests on both cores. Same presenter, tvOS HDR: tvOS has no Metal EDR API and a bare PQ colorspace tag composites UNTONE-MAPPED (the "overblown" Apple TV report), so HDR now splits on the display's live EDR headroom - PQ passthrough when the per-session AVDisplayManager mode switch landed (a real HDR10 output tone-maps itself), else an in-shader PQ->SDR tone-map (203-nit reference white, extended-Reinhard 1000-nit knee, 2020->709) into the proven SDR layer config. The 10-bit stream keeps its full decode depth either way. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
601 lines
34 KiB
Swift
601 lines
34 KiB
Swift
// Stage-2 presenter, present half: draw a decoded NV12 / P010 / 4:4:4 CVPixelBuffer into a CAMetalLayer
|
||
// drawable with a Y′CbCr→RGB shader. The hosting view's CADisplayLink still paces the pipeline once per
|
||
// vsync, but the actual `render` runs on Stage2Pipeline's dedicated RENDER THREAD (the link tick just
|
||
// signals it), so `nextDrawable()`'s blocking never lands on the main thread. See docs
|
||
// apple-stage2-presenter.md.
|
||
//
|
||
// Threading: created during view setup (main); `render`/`configure` run on the render thread — the
|
||
// layer's drawable/format/colour mutations all happen there (CAMetalLayer is designed for dedicated
|
||
// render threads; only the layer's GEOMETRY — frame/contentsScale — is touched from main, in
|
||
// SessionPresenter.layout, which also pushes the resulting pixel size here via `setDrawableTarget`
|
||
// so the render thread never reads layer geometry cross-thread). `setHdrMeta` (pump thread) and
|
||
// `setDrawableTarget` (main) only write lock-guarded staging state the render thread drains.
|
||
|
||
#if canImport(Metal) && canImport(QuartzCore)
|
||
import CoreGraphics
|
||
import CoreVideo
|
||
import Metal
|
||
import QuartzCore
|
||
import os
|
||
|
||
private let presenterLog = Logger(subsystem: "io.unom.punktfunk", category: "presenter")
|
||
|
||
/// HDR reference white (BT.2408 "HDR Reference White"): the absolute luminance, in nits, that the
|
||
/// PQ signal's diffuse white sits at. Passed to `CAEDRMetadata.hdr10(opticalOutputScale:)`, it anchors
|
||
/// 203-nit diffuse white at EDR 1.0 (the display's SDR-white level) and lets the system tone-map the
|
||
/// brighter highlights into the panel's headroom. This is the missing anchor that made the old HDR path
|
||
/// render "way too bright" (no `edrMetadata` → no reference-white anchoring); a LARGER value renders
|
||
/// dimmer. Matches the host's standard PQ reference white.
|
||
private let hdrReferenceWhiteNits: Float = 203.0
|
||
|
||
/// PUNKTFUNK_SDR_COLORSPACE=srgb — A/B hatch for the SDR layer's colour tag. Today the SDR layer
|
||
/// ships with `colorspace = nil`, which on macOS means NO colour matching: the BT.709/sRGB-encoded
|
||
/// stream is displayed with the panel's native primaries — mild oversaturation on every P3 Mac.
|
||
/// `srgb` tags the layer so CoreAnimation colour-matches it into the panel's gamut (the strictly
|
||
/// correct rendering). Kept OFF by default until the on-glass A/B confirms it (the nil path is the
|
||
/// long-proven look, and some users may prefer the vivid rendition); flip the default once verified.
|
||
private let sdrColorspaceOverride: CGColorSpace? = {
|
||
guard ProcessInfo.processInfo.environment["PUNKTFUNK_SDR_COLORSPACE"] == "srgb" else {
|
||
return nil
|
||
}
|
||
return CGColorSpace(name: CGColorSpace.sRGB)
|
||
}()
|
||
|
||
/// Runtime-compiled (no metallib build step needed in SwiftPM): a fullscreen triangle and Y′CbCr→RGB
|
||
/// fragment shaders whose conversion arrives as three constant rows computed per frame on the CPU
|
||
/// (`CscRows` — the Swift port of pf-client-core's `csc_rows`, from the decoded buffer's actual
|
||
/// signaling). One set of coefficients honors BT.601/709/2020 × full/limited × 8/10-bit instead of
|
||
/// the old hardcoded BT.709/BT.2020 matrices — a BT.601-signaled stream (a Linux host's RGB-input
|
||
/// NVENC) used to render with BT.709 coefficients, a constant hue error. uv.y is flipped (1 - p.y)
|
||
/// so the top-left-origin texture presents upright (NDC y is up). The HDR shader outputs PQ-encoded
|
||
/// R′G′B′ as-is — the CAMetalLayer's `itur_2100_PQ` colour space + `edrMetadata` tell the system
|
||
/// compositor the samples are PQ and how to tone-map them (no EOTF here, matching the host's
|
||
/// BT.2020 PQ emission).
|
||
private let shaderSource = """
|
||
#include <metal_stdlib>
|
||
using namespace metal;
|
||
|
||
struct VOut { float4 pos [[position]]; float2 uv; };
|
||
|
||
// The CPU-computed CSC rows (CscRows.swift, layout-matched): rgb[i] = dot(ri.xyz, yuv) + ri.w.
|
||
// Range expansion, the matrix, and the 10-bit MSB-packing factor are all folded in.
|
||
struct CscUniform { float4 r0; float4 r1; float4 r2; };
|
||
|
||
vertex VOut pf_vtx(uint vid [[vertex_id]]) {
|
||
float2 p = float2(float((vid << 1) & 2), float(vid & 2));
|
||
VOut o;
|
||
o.pos = float4(p * 2.0 - 1.0, 0.0, 1.0);
|
||
o.uv = float2(p.x, 1.0 - p.y);
|
||
return o;
|
||
}
|
||
|
||
// Bicubic (Catmull-Rom) sampling of the single-channel luma plane. The drawable is sized to the
|
||
// LAYER's pixels (see `render`), so this kernel performs the decoded→on-screen scale: when the
|
||
// window/view is bigger than the host's fixed mode a bilinear upscale looks soft; Catmull-Rom
|
||
// keeps edges crisp — matching AVSampleBufferDisplayLayer's (stage-1) scaler — and reduces to the
|
||
// exact texel at 1:1, so a native-resolution present stays pixel-exact.
|
||
// Nine bilinear taps (TheRealMJP's optimisation of the 16-tap kernel); `s` MUST be a linear
|
||
// sampler. Luma carries the perceived detail, so only it gets bicubic; chroma stays bilinear.
|
||
float catmullRomLuma(texture2d<float> tex, sampler s, float2 uv) {
|
||
float2 texSize = float2(tex.get_width(), tex.get_height());
|
||
float2 samplePos = uv * texSize;
|
||
float2 tc1 = floor(samplePos - 0.5) + 0.5;
|
||
float2 f = samplePos - tc1;
|
||
float2 w0 = f * (-0.5 + f * (1.0 - 0.5 * f));
|
||
float2 w1 = 1.0 + f * f * (-2.5 + 1.5 * f);
|
||
float2 w2 = f * (0.5 + f * (2.0 - 1.5 * f));
|
||
float2 w3 = f * f * (-0.5 + 0.5 * f);
|
||
float2 w12 = w1 + w2;
|
||
float2 off12 = w2 / w12;
|
||
float2 tc0 = (tc1 - 1.0) / texSize;
|
||
float2 tc3 = (tc1 + 2.0) / texSize;
|
||
float2 tc12 = (tc1 + off12) / texSize;
|
||
float r = 0.0;
|
||
r += tex.sample(s, float2(tc0.x, tc0.y)).r * (w0.x * w0.y);
|
||
r += tex.sample(s, float2(tc12.x, tc0.y)).r * (w12.x * w0.y);
|
||
r += tex.sample(s, float2(tc3.x, tc0.y)).r * (w3.x * w0.y);
|
||
r += tex.sample(s, float2(tc0.x, tc12.y)).r * (w0.x * w12.y);
|
||
r += tex.sample(s, float2(tc12.x, tc12.y)).r * (w12.x * w12.y);
|
||
r += tex.sample(s, float2(tc3.x, tc12.y)).r * (w3.x * w12.y);
|
||
r += tex.sample(s, float2(tc0.x, tc3.y)).r * (w0.x * w3.y);
|
||
r += tex.sample(s, float2(tc12.x, tc3.y)).r * (w12.x * w3.y);
|
||
r += tex.sample(s, float2(tc3.x, tc3.y)).r * (w3.x * w3.y);
|
||
return r;
|
||
}
|
||
|
||
// 4:2:0 chroma is left-cosited horizontally (H.273 chroma_loc type 0 — the MPEG convention the
|
||
// host encodes and VideoToolbox decodes as-is), but sampling the half-res plane at the luma UV
|
||
// assumes CENTER siting — a ~0.5-luma-px rightward chroma shift on hard colored edges. Offset the
|
||
// sample by +0.25 chroma texels to re-align (libplacebo/mpv's correction). Vertical siting for
|
||
// type 0 is centered, which plain sampling already matches. A full-size 4:4:4 plane has no
|
||
// subsampling to correct — the offset self-disables when the plane widths match.
|
||
float2 chromaUV(texture2d<float> lumaTex, texture2d<float> chromaTex, float2 uv) {
|
||
if (chromaTex.get_width() < lumaTex.get_width()) {
|
||
uv.x += 0.25 / float(chromaTex.get_width());
|
||
}
|
||
return uv;
|
||
}
|
||
|
||
// The shared sample + row-multiply: Y′CbCr (bicubic luma, siting-corrected bilinear chroma) →
|
||
// RGB via the per-frame rows. A full-size 4:4:4 chroma plane needs no change vs 4:2:0 (the siting
|
||
// offset self-disables). What the result MEANS depends on the stream: an SDR frame's rows yield
|
||
// gamma-encoded RGB, an HDR frame's rows yield PQ-encoded R′G′B′ — the fragment variants below
|
||
// differ only in what they do next.
|
||
float3 sampleRgb(texture2d<float> lumaTex, texture2d<float> chromaTex, float2 uv,
|
||
constant CscUniform& csc) {
|
||
constexpr sampler s(filter::linear, address::clamp_to_edge);
|
||
float3 yuv = float3(catmullRomLuma(lumaTex, s, uv),
|
||
chromaTex.sample(s, chromaUV(lumaTex, chromaTex, uv)).rg);
|
||
return saturate(float3(dot(csc.r0.xyz, yuv) + csc.r0.w,
|
||
dot(csc.r1.xyz, yuv) + csc.r1.w,
|
||
dot(csc.r2.xyz, yuv) + csc.r2.w));
|
||
}
|
||
|
||
// SDR: 8-bit NV12 / 4:4:4 → full-range RGB, transfer left baked (shown as-is, the proven SDR
|
||
// layer config).
|
||
fragment float4 pf_frag(VOut in [[stage_in]],
|
||
texture2d<float> lumaTex [[texture(0)]],
|
||
texture2d<float> chromaTex [[texture(1)]],
|
||
constant CscUniform& csc [[buffer(0)]]) {
|
||
return float4(sampleRgb(lumaTex, chromaTex, in.uv, csc), 1.0);
|
||
}
|
||
|
||
// HDR: 10-bit P010 / 4:4:4 (BT.2020, PQ-encoded Y′CbCr) → full-range PQ R′G′B′, output as-is —
|
||
// the CAMetalLayer's itur_2100_PQ colour space + edrMetadata tell the compositor the samples are
|
||
// PQ, so it does the PQ→display tone-map. No EOTF here. The rows fold in the exact 10-bit
|
||
// MSB-packing factor (the old hardcoded shader carried a documented ~0.1% /1024-vs-/1023 error).
|
||
fragment float4 pf_frag_hdr(VOut in [[stage_in]],
|
||
texture2d<float> lumaTex [[texture(0)]],
|
||
texture2d<float> chromaTex [[texture(1)]],
|
||
constant CscUniform& csc [[buffer(0)]]) {
|
||
return float4(sampleRgb(lumaTex, chromaTex, in.uv, csc), 1.0);
|
||
}
|
||
|
||
// HDR on tvOS when the display is composited WITHOUT HDR headroom (SDR output mode, or the user
|
||
// disabled Match Dynamic Range): no Metal EDR API exists there (CAEDRMetadata /
|
||
// wantsExtendedDynamicRangeContent are API_UNAVAILABLE(tvos)), and a bare PQ colour-space tag
|
||
// composites UNtone-mapped — the CAMetalLayer header says so outright — which showed as a badly
|
||
// overblown picture on Apple TV. So this variant finishes the job in-shader: PQ EOTF → linear
|
||
// light, 203-nit reference white (BT.2408) anchored at display white, extended-Reinhard highlight
|
||
// rolloff with a 1000-nit knee, BT.2020→BT.709 primaries, BT.709 OETF — into the proven SDR layer
|
||
// config. The 10-bit BT.2020 stream keeps its full decode depth; only the final presentation is
|
||
// display-referred SDR. (When the display IS in an HDR mode — requested per session via
|
||
// AVDisplayManager, see StreamViewIOS — tvOS presents pf_frag_hdr's PQ passthrough instead:
|
||
// in a genuine HDR10 output, PQ passthrough is the correct emission and the TV tone-maps.)
|
||
fragment float4 pf_frag_hdr_tv(VOut in [[stage_in]],
|
||
texture2d<float> lumaTex [[texture(0)]],
|
||
texture2d<float> chromaTex [[texture(1)]],
|
||
constant CscUniform& csc [[buffer(0)]]) {
|
||
// Y′CbCr → full-range PQ R′G′B′ via the per-frame rows (as pf_frag_hdr).
|
||
float3 pq = sampleRgb(lumaTex, chromaTex, in.uv, csc);
|
||
// ST 2084 EOTF: PQ code value → linear light, 1.0 = 10,000 nits.
|
||
const float m1 = 2610.0/16384.0;
|
||
const float m2 = 78.84375;
|
||
const float c1 = 3424.0/4096.0;
|
||
const float c2 = 18.8515625;
|
||
const float c3 = 18.6875;
|
||
float3 p = pow(pq, 1.0/m2);
|
||
float3 lin = pow(max(p - c1, 0.0) / (c2 - c3 * p), 1.0/m1);
|
||
// Scene-referred with diffuse white at 1.0 (the same 203-nit anchor the EDR path uses).
|
||
float3 t = lin * (10000.0/203.0);
|
||
// BT.2020 → BT.709 primaries while still linear; negatives are out-of-gamut, floor them.
|
||
float3 t709 = float3(
|
||
dot(t, float3( 1.6605, -0.5876, -0.0728)),
|
||
dot(t, float3(-0.1246, 1.1329, -0.0083)),
|
||
dot(t, float3(-0.0182, -0.1006, 1.1187)));
|
||
t709 = max(t709, 0.0);
|
||
// Extended Reinhard: 1.0 stays put, the 1000-nit knee lands at display white, above rolls off.
|
||
const float w = 1000.0/203.0;
|
||
float3 mapped = saturate(t709 * (1.0 + t709 / (w * w)) / (1.0 + t709));
|
||
// BT.709 OETF — the same encoding the SDR stream arrives in, so both paths present alike.
|
||
float3 e = select(1.099 * pow(mapped, 0.45) - 0.099, 4.5 * mapped, mapped < 0.018);
|
||
return float4(e, 1.0);
|
||
}
|
||
"""
|
||
|
||
public final class MetalVideoPresenter {
|
||
/// The layer the hosting view installs (as a sublayer) and sizes to its bounds.
|
||
public let layer: CAMetalLayer
|
||
|
||
private let device: MTLDevice
|
||
private let queue: MTLCommandQueue
|
||
/// SDR (BT.709 8-bit → bgra8) and HDR (BT.2020 PQ 10-bit → rgba16Float) pipelines. Selected per
|
||
/// frame in `render`; the layer is reconfigured to match when the session flips (HDR toggle).
|
||
private let pipelineSDR: MTLRenderPipelineState
|
||
private let pipelineHDR: MTLRenderPipelineState
|
||
/// tvOS only: the in-shader PQ→SDR tone-map fallback (pf_frag_hdr_tv → bgra8), used whenever
|
||
/// the display is composited without HDR headroom — see `setDisplayHeadroom`. nil elsewhere.
|
||
private let pipelineHDRToneMap: MTLRenderPipelineState?
|
||
private var textureCache: CVMetalTextureCache?
|
||
|
||
/// Current layer configuration — switched in `configure(hdr:)` when a frame's HDR-ness differs.
|
||
/// Render-thread confined once the pipeline runs (Stage2Pipeline.start's one pre-thread
|
||
/// `configure` call is ordered before the thread starts, so it doesn't race).
|
||
private var hdrActive = false
|
||
/// tvOS only: whether HDR frames currently present as PQ PASSTHROUGH (display has HDR headroom
|
||
/// — its own tone-map applies) vs the in-shader tone-map fallback. Render-thread confined;
|
||
/// derived from the staged display headroom at the top of every `render`.
|
||
private var hdrPassthroughActive = false
|
||
/// Last HDR mastering grade received via `setHdrMeta` (the host's 0xCE). Cached so a mid-session
|
||
/// SDR→HDR flip's `configureColor` re-applies the real grade instead of clobbering it back to the
|
||
/// bare reference-white anchor (an out-of-order race otherwise: `setHdrMeta` and the flip both write
|
||
/// `edrMetadata`). Render-thread confined (drained from `pendingHdrMeta` at the top of `render`).
|
||
private var lastHdrMeta: PunktfunkConnection.HdrMeta?
|
||
|
||
/// Cross-thread staging, all under `stagingLock`: the pump thread parks a fresh 0xCE grade in
|
||
/// `pendingHdrMeta` and the main thread parks the layout-derived drawable pixel size in
|
||
/// `drawableTarget`; the render thread drains both at the top of `render`, so every layer
|
||
/// format/colour mutation stays on the one thread that also calls `nextDrawable()`.
|
||
private let stagingLock = NSLock()
|
||
private var pendingHdrMeta: PunktfunkConnection.HdrMeta?
|
||
private var drawableTarget: CGSize = .zero
|
||
/// tvOS: the display's current EDR headroom (UIScreen.currentEDRHeadroom), pushed from the
|
||
/// main thread (SessionPresenter.layout + the mode-switch observers). > 1 ⇒ the display is
|
||
/// composited with HDR headroom, so HDR frames present as PQ passthrough; otherwise the
|
||
/// in-shader tone-map keeps the picture from blowing out. 1 (the default) is the safe start.
|
||
private var stagedDisplayHeadroom: CGFloat = 1.0
|
||
|
||
#if DEBUG
|
||
/// Last logged "decoded→drawable" signature, so the diagnostic logs only on a size/HDR change.
|
||
private var lastSizeSig = ""
|
||
#endif
|
||
|
||
/// nil if Metal is unavailable (no GPU / a headless CI) or a shader fails to compile — the caller
|
||
/// falls back to stage-1.
|
||
public static func make() -> MetalVideoPresenter? {
|
||
guard let device = MTLCreateSystemDefaultDevice(),
|
||
let queue = device.makeCommandQueue()
|
||
else { return nil }
|
||
let pipelineSDR: MTLRenderPipelineState
|
||
let pipelineHDR: MTLRenderPipelineState
|
||
let pipelineHDRToneMap: MTLRenderPipelineState?
|
||
do {
|
||
let library = try device.makeLibrary(source: shaderSource, options: nil)
|
||
let vtx = library.makeFunction(name: "pf_vtx")
|
||
let sdr = MTLRenderPipelineDescriptor()
|
||
sdr.vertexFunction = vtx
|
||
sdr.fragmentFunction = library.makeFunction(name: "pf_frag")
|
||
sdr.colorAttachments[0].pixelFormat = .bgra8Unorm
|
||
pipelineSDR = try device.makeRenderPipelineState(descriptor: sdr)
|
||
let hdr = MTLRenderPipelineDescriptor()
|
||
hdr.vertexFunction = vtx
|
||
hdr.fragmentFunction = library.makeFunction(name: "pf_frag_hdr")
|
||
hdr.colorAttachments[0].pixelFormat = .rgba16Float // PQ passthrough
|
||
pipelineHDR = try device.makeRenderPipelineState(descriptor: hdr)
|
||
#if os(tvOS)
|
||
// tvOS carries BOTH HDR pipelines: PQ passthrough when the display is composited
|
||
// with HDR headroom, the in-shader tone-map (→ the 8-bit SDR config) when it isn't.
|
||
// See setDisplayHeadroom / configureColor.
|
||
let tm = MTLRenderPipelineDescriptor()
|
||
tm.vertexFunction = vtx
|
||
tm.fragmentFunction = library.makeFunction(name: "pf_frag_hdr_tv")
|
||
tm.colorAttachments[0].pixelFormat = .bgra8Unorm
|
||
pipelineHDRToneMap = try device.makeRenderPipelineState(descriptor: tm)
|
||
#else
|
||
pipelineHDRToneMap = nil
|
||
#endif
|
||
} catch {
|
||
return nil
|
||
}
|
||
var cache: CVMetalTextureCache?
|
||
CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, device, nil, &cache)
|
||
guard let textureCache = cache else { return nil }
|
||
|
||
let layer = CAMetalLayer()
|
||
layer.device = device
|
||
layer.pixelFormat = .bgra8Unorm
|
||
layer.framebufferOnly = true
|
||
layer.isOpaque = true
|
||
#if os(macOS)
|
||
// displaySyncEnabled MUST stay false on macOS. It has flip-flopped, so the full history:
|
||
// sync ON was tried twice and starves the drawable pool both times — on macOS 26 a synced
|
||
// present only reaches glass when the WindowServer composites the window, and its FramePacing
|
||
// path does not treat our out-of-band image-queue presents as damage, so with a static scene
|
||
// the ONLY recurring damage is the 1 Hz stats HUD update: presents queue, all drawables stay
|
||
// held, `nextDrawable()` sleeps (sampled: ~70% of the render thread inside
|
||
// CAMetalLayerPrivateNextDrawableLocked → usleep), and the stream turns into a ~1 fps
|
||
// slideshow with normal-LOOKING stats (each rare frame is fresh, newest-wins ring). The
|
||
// 94fb7d1b fullscreen judder was the same starvation biting the then-main-thread render.
|
||
// With sync OFF the flip is immediate; the vsync alignment that sync was supposed to give
|
||
// (the HUD-off direct-scanout pacing fix) comes from scheduling the present at the display
|
||
// link's target time instead (`present(at:)` — see `render`).
|
||
layer.displaySyncEnabled = false
|
||
#endif
|
||
// The drawable is rendered at the LAYER's pixel size (set per-frame in `render`), so the
|
||
// shader — not the compositor — performs the decoded→on-screen scale (bicubic luma; the
|
||
// compositor's contentsGravity path is plain bilinear). The gravity stays aspect-fit as a
|
||
// transient fallback: during a live resize the compositor may composite a drawable from
|
||
// the previous layout before the next render catches up.
|
||
layer.contentsGravity = .resizeAspect
|
||
// Triple-buffer: more in-flight drawables before `nextDrawable()` (called on the display-link /
|
||
// MAIN thread) has to block waiting for one to free.
|
||
layer.maximumDrawableCount = 3
|
||
|
||
return MetalVideoPresenter(
|
||
device: device, queue: queue, pipelineSDR: pipelineSDR, pipelineHDR: pipelineHDR,
|
||
pipelineHDRToneMap: pipelineHDRToneMap, textureCache: textureCache, layer: layer)
|
||
}
|
||
|
||
private init(
|
||
device: MTLDevice, queue: MTLCommandQueue, pipelineSDR: MTLRenderPipelineState,
|
||
pipelineHDR: MTLRenderPipelineState, pipelineHDRToneMap: MTLRenderPipelineState?,
|
||
textureCache: CVMetalTextureCache, layer: CAMetalLayer
|
||
) {
|
||
self.device = device
|
||
self.queue = queue
|
||
self.pipelineSDR = pipelineSDR
|
||
self.pipelineHDR = pipelineHDR
|
||
self.pipelineHDRToneMap = pipelineHDRToneMap
|
||
self.textureCache = textureCache
|
||
self.layer = layer
|
||
}
|
||
|
||
/// Configure the layer + active pipeline for an SDR or HDR session. Called once at session start
|
||
/// (main, before the render thread exists) and again per-frame from `render` on the RENDER THREAD
|
||
/// (idempotent — the guard makes a same-state call a no-op), so a mid-session HDR toggle (the host
|
||
/// re-inits its encoder; the decoded `frame.isHDR` flips) reconfigures here automatically. HDR uses
|
||
/// an rgba16Float drawable + BT.2020 PQ colour space + EDR with a 203-nit reference-white anchor;
|
||
/// SDR uses the plain 8-bit sRGB path.
|
||
public func configure(hdr: Bool) {
|
||
#if os(tvOS)
|
||
// Reconfigure on an HDR flip AND on a passthrough↔tone-map flip: the display's headroom
|
||
// changes when the AVDisplayManager mode switch (requested at session start) completes —
|
||
// typically a second or two into the session.
|
||
stagingLock.lock()
|
||
let passthrough = stagedDisplayHeadroom > 1.0
|
||
stagingLock.unlock()
|
||
guard hdr != hdrActive || (hdr && passthrough != hdrPassthroughActive) else { return }
|
||
hdrActive = hdr
|
||
hdrPassthroughActive = passthrough
|
||
#else
|
||
guard hdr != hdrActive else { return }
|
||
hdrActive = hdr
|
||
#endif
|
||
configureColor(hdr: hdr)
|
||
}
|
||
|
||
/// tvOS: park the display's current EDR headroom (a MAIN-thread `UIScreen` read — pushed by
|
||
/// SessionPresenter.layout and the stream view's mode-switch observers). > 1 flips HDR frames
|
||
/// to PQ passthrough (the display's own tone-map applies); ≤ 1 keeps the in-shader tone-map.
|
||
/// Applied by the render thread on the next frame, like every other staged value here.
|
||
public func setDisplayHeadroom(_ headroom: CGFloat) {
|
||
stagingLock.lock()
|
||
stagedDisplayHeadroom = headroom
|
||
stagingLock.unlock()
|
||
}
|
||
|
||
/// Set the layer's pixel format + colour config for SDR or HDR. MAIN THREAD ONLY. EDR is requested
|
||
/// on macOS + iOS (the old `#if os(macOS)` guard left iOS EDR half-engaged). tvOS has NO EDR API
|
||
/// (`wantsExtendedDynamicRangeContent`/`edrMetadata`/`CAEDRMetadata` are all unavailable there) —
|
||
/// and a bare PQ colour-space tag composites UNtone-mapped (the "overblown HDR" Apple TV report),
|
||
/// so tvOS instead tone-maps PQ→SDR in the shader (pf_frag_hdr_tv) and keeps the SDR layer config.
|
||
private func configureColor(hdr: Bool) {
|
||
if hdr {
|
||
#if os(tvOS)
|
||
if hdrPassthroughActive {
|
||
// Display composited WITH HDR headroom (the session's AVDisplayManager request
|
||
// landed): emit PQ passthrough — in a real HDR10 output that's the correct
|
||
// emission, and the TV applies its own tone-map.
|
||
layer.pixelFormat = .rgba16Float
|
||
layer.colorspace = CGColorSpace(name: CGColorSpace.itur_2100_PQ)
|
||
} else {
|
||
// SDR-composited display: PQ would render untone-mapped (blown out) — the
|
||
// pf_frag_hdr_tv shader tone-maps to SDR instead.
|
||
layer.pixelFormat = .bgra8Unorm
|
||
layer.colorspace = nil
|
||
}
|
||
#else
|
||
layer.pixelFormat = .rgba16Float
|
||
layer.colorspace = CGColorSpace(name: CGColorSpace.itur_2100_PQ)
|
||
layer.wantsExtendedDynamicRangeContent = true
|
||
// Anchor reference white. Re-apply the real grade if one already arrived (0xCE before the
|
||
// flip); otherwise the bare 203-nit anchor. Without this anchor the PQ signal is too bright.
|
||
layer.edrMetadata = makeEDR(lastHdrMeta)
|
||
#endif
|
||
} else {
|
||
// SDR: gamma-encoded BT.709 [0,1] in an 8-bit drawable. Default: nil colorspace = NO
|
||
// colour matching on macOS (the panel's native primaries — the long-proven look,
|
||
// slightly oversaturated on P3 panels); PUNKTFUNK_SDR_COLORSPACE=srgb tags the layer
|
||
// for correct colour matching instead (A/B pending — see sdrColorspaceOverride).
|
||
layer.pixelFormat = .bgra8Unorm
|
||
layer.colorspace = sdrColorspaceOverride
|
||
#if !os(tvOS)
|
||
layer.wantsExtendedDynamicRangeContent = false
|
||
layer.edrMetadata = nil
|
||
#endif
|
||
}
|
||
}
|
||
|
||
#if !os(tvOS)
|
||
private func makeEDR(_ meta: PunktfunkConnection.HdrMeta?) -> CAEDRMetadata {
|
||
CAEDRMetadata.hdr10(
|
||
displayInfo: meta?.masteringDisplayColorVolume(),
|
||
contentInfo: meta?.contentLightLevelInfo(),
|
||
opticalOutputScale: hdrReferenceWhiteNits)
|
||
}
|
||
#endif
|
||
|
||
/// Update the HDR mastering metadata (drained from the host's 0xCE datagram) to refine the system
|
||
/// tone-map from the real grade. Called from the PUMP thread — the grade is only PARKED here (lock-
|
||
/// guarded); the render thread applies it at the top of the next `render`, keeping every layer
|
||
/// colour mutation on the one thread that also vends drawables.
|
||
public func setHdrMeta(_ meta: PunktfunkConnection.HdrMeta) {
|
||
stagingLock.lock()
|
||
pendingHdrMeta = meta
|
||
stagingLock.unlock()
|
||
}
|
||
|
||
/// Park the drawable pixel size the shader should render at: the metal layer's laid-out frame ×
|
||
/// contentsScale, both owned by the MAIN thread (SessionPresenter.layout pushes it on every layout/
|
||
/// backing change). The render thread reads this instead of the layer's geometry so it never
|
||
/// touches main-owned CALayer state. Zero until the first layout → `render` falls back to the
|
||
/// decoded frame size.
|
||
public func setDrawableTarget(_ size: CGSize) {
|
||
stagingLock.lock()
|
||
drawableTarget = size
|
||
stagingLock.unlock()
|
||
}
|
||
|
||
/// Draw one decoded frame to the next drawable and present it. RENDER THREAD (Stage2Pipeline's;
|
||
/// `nextDrawable()` may block up to a frame — that wait belongs here, never on main). `isHDR`
|
||
/// selects the 10-bit BT.2020 PQ path vs the 8-bit BT.709 path and is reconciled with the
|
||
/// layer config via `configure`. Returns true on success; false when there's no drawable yet, a
|
||
/// texture couldn't be made, or Metal errored — the caller then doesn't stamp a present (and can
|
||
/// requeue the frame). `onPresented` fires once the drawable actually reached glass, with the
|
||
/// `CLOCK_REALTIME` instant from the drawable's `presentedTime` — or nil when the system reports
|
||
/// none (a dropped drawable). It runs on a Metal callback thread; keep the handler thread-safe.
|
||
///
|
||
/// `presentAtMediaTime` (a `CACurrentMediaTime`-basis host time — the display link's
|
||
/// `targetTimestamp`) schedules the flip ON the vsync instead of "as soon as the GPU finishes":
|
||
/// with the layer's own sync disabled (mandatory on macOS — see init) an immediate present hits
|
||
/// glass mid-refresh whenever the layer is direct-scanout promoted (fullscreen, no HUD), which
|
||
/// is the "frametimes are off with the stats HUD closed" report. nil presents immediately
|
||
/// (`PUNKTFUNK_PRESENT_MODE=immediate` — the pre-fix behavior, kept as a diagnostic A/B).
|
||
@discardableResult
|
||
public func render(
|
||
_ pixelBuffer: CVPixelBuffer, isHDR: Bool = false,
|
||
presentAtMediaTime: CFTimeInterval? = nil,
|
||
onPresented: ((Int64?) -> Void)? = nil
|
||
) -> Bool {
|
||
// Drain the cross-thread staging (see `stagingLock`): the layout-derived drawable size and
|
||
// any freshly-arrived HDR grade, both applied from this thread.
|
||
stagingLock.lock()
|
||
let targetFromLayout = drawableTarget
|
||
let newHdrMeta = pendingHdrMeta
|
||
pendingHdrMeta = nil
|
||
stagingLock.unlock()
|
||
|
||
// Reconcile the layer with the decoded frame's HDR-ness (handles a mid-session SDR↔HDR flip).
|
||
configure(hdr: isHDR)
|
||
if let newHdrMeta {
|
||
self.lastHdrMeta = newHdrMeta
|
||
// tvOS has no edrMetadata — the cached grade is still kept (a later HDR flip's
|
||
// configureColor is where it matters there). macOS/iOS refine the live tone-map now.
|
||
#if !os(tvOS)
|
||
if hdrActive { layer.edrMetadata = makeEDR(newHdrMeta) }
|
||
#endif
|
||
}
|
||
|
||
// P010/x444 store 10-bit luma/chroma in 16-bit samples → R16/RG16; NV12/444v is 8-bit → R8/RG8.
|
||
// Derived from the actual decoded buffer so a 4:4:4 (full chroma plane) frame just works.
|
||
let pf = CVPixelBufferGetPixelFormatType(pixelBuffer)
|
||
let tenBit =
|
||
pf == kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange
|
||
|| pf == kCVPixelFormatType_420YpCbCr10BiPlanarFullRange
|
||
|| pf == kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange
|
||
|| pf == kCVPixelFormatType_444YpCbCr10BiPlanarFullRange
|
||
// The frame's Y′CbCr→RGB rows, from its ACTUAL signaling (buffer attachments + pixel
|
||
// format) — a BT.601-signaled stream gets 601 coefficients, full-range gets full-range
|
||
// expansion; recomputed per frame because the host can flip colour in-band (SDR↔HDR).
|
||
var csc = CscRows.rows(
|
||
CscRows.signal(of: pixelBuffer), depth: tenBit ? 10 : 8, msbPacked: tenBit)
|
||
guard let textureCache,
|
||
let luma = makeTexture(
|
||
pixelBuffer, plane: 0, format: tenBit ? .r16Unorm : .r8Unorm, cache: textureCache),
|
||
let chroma = makeTexture(
|
||
pixelBuffer, plane: 1, format: tenBit ? .rg16Unorm : .rg8Unorm, cache: textureCache)
|
||
else { return false }
|
||
|
||
// Size the drawable to the LAYER's pixels (its laid-out frame × contentsScale, pushed here by
|
||
// SessionPresenter.layout via `setDrawableTarget` — not read off the layer, whose geometry the
|
||
// main thread owns) so the Catmull-Rom shader performs the decoded→on-screen scale in one pass:
|
||
// a native-mode session stays exactly 1:1 (the kernel reduces to the identity texel), and a
|
||
// window bigger than the host's mode gets bicubic luma instead of the compositor's bilinear.
|
||
// Before the first layout (zero target) fall back to the decoded size. drawableSize does NOT
|
||
// track bounds (defaults to 0), so set it BEFORE nextDrawable; re-set only on a change
|
||
// (layout / Reconfigure / HDR flip — and every frame of a live resize, which is fine).
|
||
let decodedSize = CGSize(
|
||
width: CVPixelBufferGetWidth(pixelBuffer), height: CVPixelBufferGetHeight(pixelBuffer))
|
||
let targetSize = (targetFromLayout.width > 0 && targetFromLayout.height > 0)
|
||
? targetFromLayout : decodedSize
|
||
if layer.drawableSize != targetSize { layer.drawableSize = targetSize }
|
||
#if DEBUG
|
||
logSizeIfChanged(decoded: decodedSize, drawable: targetSize)
|
||
#endif
|
||
guard let drawable = layer.nextDrawable(),
|
||
let commandBuffer = queue.makeCommandBuffer()
|
||
else { return false }
|
||
|
||
let pass = MTLRenderPassDescriptor()
|
||
pass.colorAttachments[0].texture = drawable.texture
|
||
pass.colorAttachments[0].loadAction = .clear
|
||
pass.colorAttachments[0].clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 1)
|
||
pass.colorAttachments[0].storeAction = .store
|
||
guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: pass) else {
|
||
return false
|
||
}
|
||
#if os(tvOS)
|
||
// HDR splits by the display's headroom (kept in step with the layer by `configure` above):
|
||
// PQ passthrough into an HDR-composited display, the tone-map shader otherwise.
|
||
let hdrPipeline = hdrPassthroughActive ? pipelineHDR : (pipelineHDRToneMap ?? pipelineHDR)
|
||
encoder.setRenderPipelineState(hdrActive ? hdrPipeline : pipelineSDR)
|
||
#else
|
||
encoder.setRenderPipelineState(hdrActive ? pipelineHDR : pipelineSDR)
|
||
#endif
|
||
encoder.setFragmentTexture(CVMetalTextureGetTexture(luma), index: 0)
|
||
encoder.setFragmentTexture(CVMetalTextureGetTexture(chroma), index: 1)
|
||
encoder.setFragmentBytes(&csc, length: MemoryLayout<CscUniform>.stride, index: 0)
|
||
encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3)
|
||
encoder.endEncoding()
|
||
if let onPresented {
|
||
#if targetEnvironment(simulator)
|
||
// The simulator SDK exposes neither addPresentedHandler nor presentedTime — report
|
||
// nil so the caller stamps with its display-link estimate (the pre-presentedTime
|
||
// behavior; simulator numbers are indicative only anyway).
|
||
onPresented(nil)
|
||
#else
|
||
// Registered BEFORE present. presentedTime is CACurrentMediaTime-based; 0 means the
|
||
// system never put this drawable on glass (dropped) — report nil, the caller falls
|
||
// back to its display-link estimate.
|
||
drawable.addPresentedHandler { d in
|
||
onPresented(
|
||
d.presentedTime > 0
|
||
? Stage2Pipeline.realtimeNs(forDisplayLinkTimestamp: d.presentedTime)
|
||
: nil)
|
||
}
|
||
#endif
|
||
}
|
||
// Scheduled on the vsync when the pipeline gave us the link's target (see the doc comment);
|
||
// immediate otherwise. A target already in the past presents immediately — same thing.
|
||
if let presentAtMediaTime {
|
||
commandBuffer.present(drawable, atTime: presentAtMediaTime)
|
||
} else {
|
||
commandBuffer.present(drawable)
|
||
}
|
||
// Hold the CVMetalTextures + source pixel buffer (its IOSurface) alive until the GPU finishes
|
||
// sampling — releasing them at scope exit could free the backing mid-read.
|
||
commandBuffer.addCompletedHandler { _ in _ = (luma, chroma, pixelBuffer) }
|
||
commandBuffer.commit()
|
||
return true
|
||
}
|
||
|
||
/// Returns the CVMetalTexture (not just its MTLTexture) so the caller can keep it alive past the
|
||
/// draw — the MTLTexture is only valid while its CVMetalTexture is retained.
|
||
private func makeTexture(
|
||
_ pixelBuffer: CVPixelBuffer, plane: Int, format: MTLPixelFormat, cache: CVMetalTextureCache
|
||
) -> CVMetalTexture? {
|
||
let w = CVPixelBufferGetWidthOfPlane(pixelBuffer, plane)
|
||
let h = CVPixelBufferGetHeightOfPlane(pixelBuffer, plane)
|
||
var cvTexture: CVMetalTexture?
|
||
let status = CVMetalTextureCacheCreateTextureFromImage(
|
||
kCFAllocatorDefault, cache, pixelBuffer, nil, format, w, h, plane, &cvTexture)
|
||
guard status == kCVReturnSuccess, let cvTexture,
|
||
CVMetalTextureGetTexture(cvTexture) != nil
|
||
else { return nil }
|
||
return cvTexture
|
||
}
|
||
|
||
#if DEBUG
|
||
private func logSizeIfChanged(decoded: CGSize, drawable: CGSize) {
|
||
let sig = "\(Int(decoded.width))x\(Int(decoded.height))→\(Int(drawable.width))x\(Int(drawable.height))|hdr\(hdrActive ? 1 : 0)"
|
||
if sig != lastSizeSig {
|
||
lastSizeSig = sig
|
||
let msg =
|
||
"stage2: decoded \(Int(decoded.width))x\(Int(decoded.height)) → drawable \(Int(drawable.width))x\(Int(drawable.height)) hdr=\(hdrActive)"
|
||
presenterLog.info("\(msg, privacy: .public)")
|
||
}
|
||
}
|
||
#endif
|
||
}
|
||
#endif
|