fix(video): honor the signaled CSC matrix end-to-end + tvOS HDR presentation
Clients derive Y'CbCr->RGB from the stream's SIGNALED matrix x range x depth via shared csc rows (Rust csc_rows + Swift CscRows) instead of hardcoded 709/2020 - a BT.601-signaled stream (a Linux host's RGB-input NVENC) no longer renders with a constant hue error. Host-side signaling made honest across NVENC/VAAPI/openh264/GameStream and the session plan's chroma/bit-depth. Decoded color-bar fixtures (601/709 x limited/full) pin the math in tests on both cores. Same presenter, tvOS HDR: tvOS has no Metal EDR API and a bare PQ colorspace tag composites UNTONE-MAPPED (the "overblown" Apple TV report), so HDR now splits on the display's live EDR headroom - PQ passthrough when the per-session AVDisplayManager mode switch landed (a real HDR10 output tone-maps itself), else an in-shader PQ->SDR tone-map (203-nit reference white, extended-Reinhard 1000-nit knee, 2020->709) into the proven SDR layer config. The 10-bit stream keeps its full decode depth either way. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -119,11 +119,13 @@ pub struct ColorDesc {
|
||||
}
|
||||
|
||||
impl ColorDesc {
|
||||
/// Read the CICP fields off a raw decoded frame.
|
||||
/// Read the CICP fields off a raw decoded frame. Public: the Windows client's raw-FFI
|
||||
/// D3D11VA/software decoders build their per-frame `ColorDesc` with it too (same
|
||||
/// `ffmpeg-next` major, so the `AVFrame` type unifies across the workspace).
|
||||
///
|
||||
/// # Safety
|
||||
/// `frame` must point to a valid `AVFrame` (alive for the duration of the call).
|
||||
pub(crate) unsafe fn from_raw(frame: *const ffmpeg::ffi::AVFrame) -> ColorDesc {
|
||||
pub unsafe fn from_raw(frame: *const ffmpeg::ffi::AVFrame) -> ColorDesc {
|
||||
// SAFETY: caller guarantees a live AVFrame; these are plain enum field reads.
|
||||
unsafe {
|
||||
ColorDesc {
|
||||
@@ -141,6 +143,57 @@ impl ColorDesc {
|
||||
}
|
||||
}
|
||||
|
||||
/// The Y′CbCr→RGB conversion as three vec4 rows for a shader constant buffer / push-constant
|
||||
/// block: `rgb[i] = dot(r[i].xyz, yuv) + r[i].w` — bit-depth exact. The ONE coefficient
|
||||
/// implementation every presenter derives its CSC from (Vulkan push constants, the Windows
|
||||
/// client's D3D11 constant buffer), so a stream's signaled matrix/range is honored identically
|
||||
/// everywhere; the Apple client ports this function (and its tests) to Swift.
|
||||
///
|
||||
/// `depth` picks the limited-range code points (8-bit: 16/235/240 over 255; 10-bit:
|
||||
/// 64/940/960 over 1023 — NOT the same normalized values, the difference is ~half a
|
||||
/// code). `msb_packed` folds in the P010/X6 packing factor: 10 significant bits live in
|
||||
/// the MSBs of 16, so a UNORM16 sample reads `code·64/65535` — multiplying by
|
||||
/// `65535/65472` recovers exact `code/1023`.
|
||||
pub fn csc_rows(desc: ColorDesc, depth: u8, msb_packed: bool) -> [[f32; 4]; 3] {
|
||||
// BT.601 (5/6), BT.2020 (9/10); everything else — incl. unspecified — is the host's
|
||||
// BT.709 SDR default (mirrors the software path's swscale coefficient choice).
|
||||
let (kr, kb) = match desc.matrix {
|
||||
5 | 6 => (0.299, 0.114),
|
||||
9 | 10 => (0.2627, 0.0593),
|
||||
_ => (0.2126, 0.0722),
|
||||
};
|
||||
let kg = 1.0 - kr - kb;
|
||||
let max = f64::from((1u32 << depth) - 1); // 255 / 1023
|
||||
let step = f64::from(1u32 << (depth - 8)); // code points per 8-bit step: 1 / 4
|
||||
let pack = if msb_packed { 65535.0 / 65472.0 } else { 1.0 };
|
||||
let (sy, oy, sc) = if desc.full_range {
|
||||
(pack, 0.0f64, pack)
|
||||
} else {
|
||||
(
|
||||
pack * max / (219.0 * step),
|
||||
-(16.0 * step) / max,
|
||||
pack * max / (224.0 * step),
|
||||
)
|
||||
};
|
||||
// rgb = M * (yuv + off) = M*yuv + M*off — rows of M with the offset dot folded into
|
||||
// w. `yuv` is the SAMPLED (packed) value, so the offsets divide by the packing
|
||||
// factor to land on the same scale.
|
||||
let off = [oy / pack, -0.5 / pack, -0.5 / pack];
|
||||
let m = [
|
||||
[sy, 0.0, 2.0 * (1.0 - kr) * sc],
|
||||
[
|
||||
sy,
|
||||
-2.0 * (1.0 - kb) * kb / kg * sc,
|
||||
-2.0 * (1.0 - kr) * kr / kg * sc,
|
||||
],
|
||||
[sy, 2.0 * (1.0 - kb) * sc, 0.0],
|
||||
];
|
||||
core::array::from_fn(|r| {
|
||||
let w: f64 = (0..3).map(|c| m[r][c] * off[c]).sum();
|
||||
[m[r][0] as f32, m[r][1] as f32, m[r][2] as f32, w as f32]
|
||||
})
|
||||
}
|
||||
|
||||
/// RGBA pixels for `GdkMemoryTexture` (which takes a stride).
|
||||
pub struct CpuFrame {
|
||||
pub width: u32,
|
||||
@@ -1387,6 +1440,117 @@ unsafe extern "C" fn pick_vulkan(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn desc(matrix: u8, full_range: bool) -> ColorDesc {
|
||||
ColorDesc {
|
||||
primaries: 1,
|
||||
transfer: 1,
|
||||
matrix,
|
||||
full_range,
|
||||
}
|
||||
}
|
||||
|
||||
fn apply(rows: &[[f32; 4]; 3], yuv: [f32; 3]) -> [f32; 3] {
|
||||
core::array::from_fn(|r| {
|
||||
rows[r][0] * yuv[0] + rows[r][1] * yuv[1] + rows[r][2] * yuv[2] + rows[r][3]
|
||||
})
|
||||
}
|
||||
|
||||
/// 10-bit limited MSB-packed (P010/X6): reference white Y=940, black Y=64, neutral
|
||||
/// chroma 512 — sampled as UNORM16 of `code << 6`.
|
||||
#[test]
|
||||
fn bt2020_10bit_limited_white_black() {
|
||||
let rows = csc_rows(desc(9, false), 10, true);
|
||||
let s = |code: u32| ((code << 6) as f32) / 65535.0;
|
||||
let white = apply(&rows, [s(940), s(512), s(512)]);
|
||||
let black = apply(&rows, [s(64), s(512), s(512)]);
|
||||
for (w, b) in white.iter().zip(black) {
|
||||
assert!((w - 1.0).abs() < 0.002, "white {white:?}");
|
||||
assert!(b.abs() < 0.002, "black {black:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Reference white (Y=235, U=V=128 limited) → RGB 1.0; reference black (Y=16) → 0.0
|
||||
/// — the GL presenter's test, in row form.
|
||||
#[test]
|
||||
fn bt709_limited_white_black() {
|
||||
let rows = csc_rows(desc(1, false), 8, false);
|
||||
let white = apply(&rows, [235.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0]);
|
||||
let black = apply(&rows, [16.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0]);
|
||||
for (w, b) in white.iter().zip(black) {
|
||||
assert!((w - 1.0).abs() < 0.005, "white {white:?}");
|
||||
assert!(b.abs() < 0.005, "black {black:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Full-range identity points + the 601-vs-709 red excursion (guards the
|
||||
/// matrix-code dispatch), same as the GL presenter's test.
|
||||
#[test]
|
||||
fn full_range_and_red_excursion() {
|
||||
let rows = csc_rows(desc(5, true), 8, false);
|
||||
let white = apply(&rows, [1.0, 0.5, 0.5]);
|
||||
assert!(white.iter().all(|v| (v - 1.0).abs() < 1e-5), "{white:?}");
|
||||
let red = apply(&rows, [0.0, 0.5, 1.0]);
|
||||
assert!((red[0] - 2.0 * (1.0 - 0.299) * 0.5).abs() < 1e-4, "{red:?}");
|
||||
let rows709 = csc_rows(desc(1, true), 8, false);
|
||||
let red709 = apply(&rows709, [0.0, 0.5, 1.0]);
|
||||
assert!(
|
||||
(red709[0] - 2.0 * (1.0 - 0.2126) * 0.5).abs() < 1e-4,
|
||||
"{red709:?}"
|
||||
);
|
||||
assert!((red[0] - red709[0]).abs() > 0.05);
|
||||
}
|
||||
|
||||
/// The row form must agree with the GL presenter's column-major `yuv_to_rgb` on a
|
||||
/// grid of inputs — same math, different packing.
|
||||
#[test]
|
||||
fn rows_match_the_gl_matrix_form() {
|
||||
for (matrix, full) in [(1u8, false), (1, true), (5, false), (9, false), (9, true)] {
|
||||
let d = desc(matrix, full);
|
||||
let rows = csc_rows(d, 8, false);
|
||||
// Reimplementation of video_gl::yuv_to_rgb's application for comparison.
|
||||
let (kr, kb) = match matrix {
|
||||
5 | 6 => (0.299f32, 0.114f32),
|
||||
9 | 10 => (0.2627, 0.0593),
|
||||
_ => (0.2126, 0.0722),
|
||||
};
|
||||
let kg = 1.0 - kr - kb;
|
||||
let (sy, oy, sc) = if full {
|
||||
(1.0f32, 0.0f32, 1.0f32)
|
||||
} else {
|
||||
(255.0 / 219.0, -16.0 / 255.0, 255.0 / 224.0)
|
||||
};
|
||||
let mat = [
|
||||
sy,
|
||||
sy,
|
||||
sy,
|
||||
0.0,
|
||||
-2.0 * (1.0 - kb) * kb / kg * sc,
|
||||
2.0 * (1.0 - kb) * sc,
|
||||
2.0 * (1.0 - kr) * sc,
|
||||
-2.0 * (1.0 - kr) * kr / kg * sc,
|
||||
0.0,
|
||||
];
|
||||
let off = [oy, -0.5, -0.5];
|
||||
for yuv in [
|
||||
[0.1f32, 0.3, 0.7],
|
||||
[0.9, 0.5, 0.5],
|
||||
[0.5, 0.2, 0.8],
|
||||
[16.0 / 255.0, 0.5, 0.5],
|
||||
] {
|
||||
let v = [yuv[0] + off[0], yuv[1] + off[1], yuv[2] + off[2]];
|
||||
let gl: [f32; 3] =
|
||||
core::array::from_fn(|r| (0..3).map(|c| mat[c * 3 + r] * v[c]).sum());
|
||||
let ours = apply(&rows, yuv);
|
||||
for (a, b) in gl.iter().zip(ours) {
|
||||
assert!(
|
||||
(a - b).abs() < 1e-5,
|
||||
"{matrix}/{full}: gl {gl:?} rows {ours:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Lock the DRM FourCC magic numbers against typos — these are the exact values
|
||||
/// `<drm_fourcc.h>` defines, and a wrong one is what painted the Steam Deck green.
|
||||
#[test]
|
||||
@@ -1434,4 +1598,82 @@ mod tests {
|
||||
assert!(f.color.is_pq());
|
||||
assert_eq!((f.width, f.height), (64, 64));
|
||||
}
|
||||
|
||||
/// Golden colour fixtures: one 256×64 LOSSLESS x265 IDR of 8 fully-saturated colour bars per
|
||||
/// signaling variant (generated offline with ffmpeg/libx265; the RGB→YUV conversion matched
|
||||
/// to the VUI each fixture declares, so the original RGB is recoverable ±1 code). Decoding
|
||||
/// through the real CPU path (`SoftwareDecoder` → per-frame `ColorDesc` → swscale with the
|
||||
/// signaled matrix/range) must reproduce the bars — the end-to-end guard for the
|
||||
/// signaling-driven CSC across BT.601/709 × limited/full. A hardcoded-709 regression fails
|
||||
/// the 601 fixture by tens of code points; a range mix-up fails the full-range one.
|
||||
#[test]
|
||||
fn software_decode_reproduces_golden_bars() {
|
||||
const BARS: [(u8, u8, u8); 8] = [
|
||||
(255, 255, 255),
|
||||
(255, 255, 0),
|
||||
(0, 255, 255),
|
||||
(0, 255, 0),
|
||||
(255, 0, 255),
|
||||
(255, 0, 0),
|
||||
(0, 0, 255),
|
||||
(0, 0, 0),
|
||||
];
|
||||
let fixtures: [(&str, &[u8], ColorDesc); 3] = [
|
||||
(
|
||||
"601-limited",
|
||||
include_bytes!("../tests/bars-601-limited.h265"),
|
||||
ColorDesc {
|
||||
primaries: 1,
|
||||
transfer: 1,
|
||||
matrix: 5, // BT.470BG — what a Linux host's RGB-input NVENC signals
|
||||
full_range: false,
|
||||
},
|
||||
),
|
||||
(
|
||||
"709-limited",
|
||||
include_bytes!("../tests/bars-709-limited.h265"),
|
||||
ColorDesc {
|
||||
primaries: 1,
|
||||
transfer: 1,
|
||||
matrix: 1,
|
||||
full_range: false,
|
||||
},
|
||||
),
|
||||
(
|
||||
"709-full",
|
||||
include_bytes!("../tests/bars-709-full.h265"),
|
||||
ColorDesc {
|
||||
primaries: 1,
|
||||
transfer: 1,
|
||||
matrix: 1,
|
||||
full_range: true, // the PUNKTFUNK_444_FULLRANGE experiment's signaling
|
||||
},
|
||||
),
|
||||
];
|
||||
for (name, au, want_color) in fixtures {
|
||||
let mut dec = SoftwareDecoder::new(ffmpeg::codec::Id::HEVC).expect("hevc decoder");
|
||||
let mut got = dec.decode(au).expect("decode");
|
||||
if got.is_none() {
|
||||
dec.decoder.send_eof().ok();
|
||||
let mut frame = AvFrame::empty();
|
||||
if dec.decoder.receive_frame(&mut frame).is_ok() {
|
||||
got = Some(dec.convert_rgba(&frame).expect("convert"));
|
||||
}
|
||||
}
|
||||
let f = got.unwrap_or_else(|| panic!("{name}: no frame decoded"));
|
||||
assert_eq!(f.color, want_color, "{name}: signaling");
|
||||
assert_eq!((f.width, f.height), (256, 64), "{name}: dims");
|
||||
for (i, (r, g, b)) in BARS.iter().enumerate() {
|
||||
let (cx, cy) = (i * 32 + 16, 32usize);
|
||||
let o = cy * f.stride + cx * 4;
|
||||
let px = &f.rgba[o..o + 3];
|
||||
for (got, want) in px.iter().zip([r, g, b]) {
|
||||
assert!(
|
||||
got.abs_diff(*want) <= 3,
|
||||
"{name} bar {i}: got {px:?}, want ({r},{g},{b})"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,10 +52,12 @@ use windows::Win32::Graphics::Direct3D11::{
|
||||
D3D11_VPOV_DIMENSION_TEXTURE2D,
|
||||
};
|
||||
use windows::Win32::Graphics::Dxgi::Common::{
|
||||
DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709, DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P709,
|
||||
DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709, DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P2020,
|
||||
DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P601, DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P709,
|
||||
DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_LEFT_P2020, DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P2020,
|
||||
DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709, DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM,
|
||||
DXGI_FORMAT_NV12, DXGI_FORMAT_P010, DXGI_RATIONAL, DXGI_SAMPLE_DESC,
|
||||
DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P601, DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709,
|
||||
DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_NV12, DXGI_FORMAT_P010, DXGI_RATIONAL,
|
||||
DXGI_SAMPLE_DESC,
|
||||
};
|
||||
use windows::Win32::Graphics::Dxgi::{
|
||||
CreateDXGIFactory1, IDXGIAdapter1, IDXGIFactory1, IDXGIKeyedMutex, IDXGIResource1,
|
||||
@@ -629,9 +631,16 @@ impl D3d11vaDecoder {
|
||||
|
||||
// Colour spaces per frame (the host flips PQ in-band): YCbCr in, sRGB out — a PQ
|
||||
// stream is tone-mapped to SDR by the processor (module docs). CICP → DXGI enums.
|
||||
// BT.601 (5/6) matters in practice: a Linux host's RGB-input NVENC paths signal
|
||||
// BT470BG limited (NVENC's fixed internal RGB→YUV is BT.601 — ffmpeg force-writes
|
||||
// that VUI), and mapping it to P709 here was a constant hue error on those streams.
|
||||
// DXGI has no full-range G2084 YCbCr enum, so PQ is studio regardless of range.
|
||||
let in_cs = match (color.transfer, color.matrix, color.full_range) {
|
||||
(16, _, _) => DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_LEFT_P2020,
|
||||
(_, 9, _) => DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P2020,
|
||||
(_, 9 | 10, false) => DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P2020,
|
||||
(_, 9 | 10, true) => DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P2020,
|
||||
(_, 5 | 6, false) => DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P601,
|
||||
(_, 5 | 6, true) => DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P601,
|
||||
(_, _, true) => DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P709,
|
||||
_ => DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709,
|
||||
};
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -62,7 +62,17 @@ vec3 srgb_oetf(vec3 c) {
|
||||
}
|
||||
|
||||
void main() {
|
||||
vec3 yuv = vec3(texture(u_y, v_uv).r, texture(u_c, v_uv).rg);
|
||||
// 4:2:0 chroma is left-cosited (H.273 type 0 — the default inference when unsignaled, and
|
||||
// what the hosts produce), but sampling the half-res plane at the luma UV assumes CENTER
|
||||
// siting — a ~0.5-luma-px rightward chroma shift on hard colored edges. Offset +0.25 chroma
|
||||
// texels to re-align (the same correction the Apple/Windows clients apply). Self-disables
|
||||
// when the plane widths match (a full-size 4:4:4 chroma plane needs no correction).
|
||||
vec2 cuv = v_uv;
|
||||
int cw = textureSize(u_c, 0).x;
|
||||
if (cw < textureSize(u_y, 0).x) {
|
||||
cuv.x += 0.25 / float(cw);
|
||||
}
|
||||
vec3 yuv = vec3(texture(u_y, v_uv).r, texture(u_c, cuv).rg);
|
||||
vec3 rgb = vec3(
|
||||
dot(pc.r0.xyz, yuv) + pc.r0.w,
|
||||
dot(pc.r1.xyz, yuv) + pc.r1.w,
|
||||
|
||||
Binary file not shown.
@@ -9,55 +9,11 @@
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use ash::vk;
|
||||
use pf_client_core::video::ColorDesc;
|
||||
|
||||
/// The push-constant block's matrix half: three vec4 rows,
|
||||
/// `rgb[i] = dot(r[i].xyz, yuv) + r[i].w` — bit-depth exact.
|
||||
///
|
||||
/// `depth` picks the limited-range code points (8-bit: 16/235/240 over 255; 10-bit:
|
||||
/// 64/940/960 over 1023 — NOT the same normalized values, the difference is ~half a
|
||||
/// code). `msb_packed` folds in the P010/X6 packing factor: 10 significant bits live in
|
||||
/// the MSBs of 16, so a UNORM16 sample reads `code·64/65535` — multiplying by
|
||||
/// `65535/65472` recovers exact `code/1023`.
|
||||
pub fn csc_rows(desc: ColorDesc, depth: u8, msb_packed: bool) -> [[f32; 4]; 3] {
|
||||
// BT.601 (5/6), BT.2020 (9/10); everything else — incl. unspecified — is the host's
|
||||
// BT.709 SDR default (mirrors the software path's swscale coefficient choice).
|
||||
let (kr, kb) = match desc.matrix {
|
||||
5 | 6 => (0.299, 0.114),
|
||||
9 | 10 => (0.2627, 0.0593),
|
||||
_ => (0.2126, 0.0722),
|
||||
};
|
||||
let kg = 1.0 - kr - kb;
|
||||
let max = f64::from((1u32 << depth) - 1); // 255 / 1023
|
||||
let step = f64::from(1u32 << (depth - 8)); // code points per 8-bit step: 1 / 4
|
||||
let pack = if msb_packed { 65535.0 / 65472.0 } else { 1.0 };
|
||||
let (sy, oy, sc) = if desc.full_range {
|
||||
(pack, 0.0f64, pack)
|
||||
} else {
|
||||
(
|
||||
pack * max / (219.0 * step),
|
||||
-(16.0 * step) / max,
|
||||
pack * max / (224.0 * step),
|
||||
)
|
||||
};
|
||||
// rgb = M * (yuv + off) = M*yuv + M*off — rows of M with the offset dot folded into
|
||||
// w. `yuv` is the SAMPLED (packed) value, so the offsets divide by the packing
|
||||
// factor to land on the same scale.
|
||||
let off = [oy / pack, -0.5 / pack, -0.5 / pack];
|
||||
let m = [
|
||||
[sy, 0.0, 2.0 * (1.0 - kr) * sc],
|
||||
[
|
||||
sy,
|
||||
-2.0 * (1.0 - kb) * kb / kg * sc,
|
||||
-2.0 * (1.0 - kr) * kr / kg * sc,
|
||||
],
|
||||
[sy, 2.0 * (1.0 - kb) * sc, 0.0],
|
||||
];
|
||||
core::array::from_fn(|r| {
|
||||
let w: f64 = (0..3).map(|c| m[r][c] * off[c]).sum();
|
||||
[m[r][0] as f32, m[r][1] as f32, m[r][2] as f32, w as f32]
|
||||
})
|
||||
}
|
||||
// The coefficient math lives in pf-client-core next to `ColorDesc` (one tested
|
||||
// implementation shared with the Windows client's D3D11 constant buffer and mirrored by the
|
||||
// Apple client's Swift port); re-exported here so presenter callers keep their import path.
|
||||
pub use pf_client_core::video::csc_rows;
|
||||
|
||||
/// The pass objects (everything except the per-video-size framebuffer, which lives with
|
||||
/// the video image). Destroyed explicitly via [`CscPass::destroy`] from the presenter's
|
||||
@@ -337,118 +293,3 @@ pub(crate) fn build_fullscreen_pipeline(
|
||||
Ok(pipeline?[0])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn desc(matrix: u8, full_range: bool) -> ColorDesc {
|
||||
ColorDesc {
|
||||
primaries: 1,
|
||||
transfer: 1,
|
||||
matrix,
|
||||
full_range,
|
||||
}
|
||||
}
|
||||
|
||||
fn apply(rows: &[[f32; 4]; 3], yuv: [f32; 3]) -> [f32; 3] {
|
||||
core::array::from_fn(|r| {
|
||||
rows[r][0] * yuv[0] + rows[r][1] * yuv[1] + rows[r][2] * yuv[2] + rows[r][3]
|
||||
})
|
||||
}
|
||||
|
||||
/// 10-bit limited MSB-packed (P010/X6): reference white Y=940, black Y=64, neutral
|
||||
/// chroma 512 — sampled as UNORM16 of `code << 6`.
|
||||
#[test]
|
||||
fn bt2020_10bit_limited_white_black() {
|
||||
let rows = csc_rows(desc(9, false), 10, true);
|
||||
let s = |code: u32| ((code << 6) as f32) / 65535.0;
|
||||
let white = apply(&rows, [s(940), s(512), s(512)]);
|
||||
let black = apply(&rows, [s(64), s(512), s(512)]);
|
||||
for (w, b) in white.iter().zip(black) {
|
||||
assert!((w - 1.0).abs() < 0.002, "white {white:?}");
|
||||
assert!(b.abs() < 0.002, "black {black:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Reference white (Y=235, U=V=128 limited) → RGB 1.0; reference black (Y=16) → 0.0
|
||||
/// — the GL presenter's test, in row form.
|
||||
#[test]
|
||||
fn bt709_limited_white_black() {
|
||||
let rows = csc_rows(desc(1, false), 8, false);
|
||||
let white = apply(&rows, [235.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0]);
|
||||
let black = apply(&rows, [16.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0]);
|
||||
for (w, b) in white.iter().zip(black) {
|
||||
assert!((w - 1.0).abs() < 0.005, "white {white:?}");
|
||||
assert!(b.abs() < 0.005, "black {black:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Full-range identity points + the 601-vs-709 red excursion (guards the
|
||||
/// matrix-code dispatch), same as the GL presenter's test.
|
||||
#[test]
|
||||
fn full_range_and_red_excursion() {
|
||||
let rows = csc_rows(desc(5, true), 8, false);
|
||||
let white = apply(&rows, [1.0, 0.5, 0.5]);
|
||||
assert!(white.iter().all(|v| (v - 1.0).abs() < 1e-5), "{white:?}");
|
||||
let red = apply(&rows, [0.0, 0.5, 1.0]);
|
||||
assert!((red[0] - 2.0 * (1.0 - 0.299) * 0.5).abs() < 1e-4, "{red:?}");
|
||||
let rows709 = csc_rows(desc(1, true), 8, false);
|
||||
let red709 = apply(&rows709, [0.0, 0.5, 1.0]);
|
||||
assert!(
|
||||
(red709[0] - 2.0 * (1.0 - 0.2126) * 0.5).abs() < 1e-4,
|
||||
"{red709:?}"
|
||||
);
|
||||
assert!((red[0] - red709[0]).abs() > 0.05);
|
||||
}
|
||||
|
||||
/// The row form must agree with the GL presenter's column-major `yuv_to_rgb` on a
|
||||
/// grid of inputs — same math, different packing.
|
||||
#[test]
|
||||
fn rows_match_the_gl_matrix_form() {
|
||||
for (matrix, full) in [(1u8, false), (1, true), (5, false), (9, false), (9, true)] {
|
||||
let d = desc(matrix, full);
|
||||
let rows = csc_rows(d, 8, false);
|
||||
// Reimplementation of video_gl::yuv_to_rgb's application for comparison.
|
||||
let (kr, kb) = match matrix {
|
||||
5 | 6 => (0.299f32, 0.114f32),
|
||||
9 | 10 => (0.2627, 0.0593),
|
||||
_ => (0.2126, 0.0722),
|
||||
};
|
||||
let kg = 1.0 - kr - kb;
|
||||
let (sy, oy, sc) = if full {
|
||||
(1.0f32, 0.0f32, 1.0f32)
|
||||
} else {
|
||||
(255.0 / 219.0, -16.0 / 255.0, 255.0 / 224.0)
|
||||
};
|
||||
let mat = [
|
||||
sy,
|
||||
sy,
|
||||
sy,
|
||||
0.0,
|
||||
-2.0 * (1.0 - kb) * kb / kg * sc,
|
||||
2.0 * (1.0 - kb) * sc,
|
||||
2.0 * (1.0 - kr) * sc,
|
||||
-2.0 * (1.0 - kr) * kr / kg * sc,
|
||||
0.0,
|
||||
];
|
||||
let off = [oy, -0.5, -0.5];
|
||||
for yuv in [
|
||||
[0.1f32, 0.3, 0.7],
|
||||
[0.9, 0.5, 0.5],
|
||||
[0.5, 0.2, 0.8],
|
||||
[16.0 / 255.0, 0.5, 0.5],
|
||||
] {
|
||||
let v = [yuv[0] + off[0], yuv[1] + off[1], yuv[2] + off[2]];
|
||||
let gl: [f32; 3] =
|
||||
core::array::from_fn(|r| (0..3).map(|c| mat[c * 3 + r] * v[c]).sum());
|
||||
let ours = apply(&rows, yuv);
|
||||
for (a, b) in gl.iter().zip(ours) {
|
||||
assert!(
|
||||
(a - b).abs() < 1e-5,
|
||||
"{matrix}/{full}: gl {gl:?} rows {ours:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,10 +62,13 @@ pub struct OutputFormat {
|
||||
/// HDR: the capturer converts to 10-bit (IDD-push FP16 → `P010`, or `Rgb10a2` for a 4:4:4 source).
|
||||
/// `false` = 8-bit SDR.
|
||||
pub hdr: bool,
|
||||
/// Full-chroma 4:4:4 session: the capturer must keep full chroma — deliver packed **RGB**
|
||||
/// (`Bgra` / `Rgb10a2`), NOT the subsampled `Nv12`/`P010` the Windows video-engine path produces by
|
||||
/// default — because 4:4:4 can only be recovered from a full-chroma source. NVENC then does the
|
||||
/// RGB→YUV444 CSC at encode (chroma_format_idc=3). `false` on every 4:2:0 session.
|
||||
/// Full-chroma 4:4:4 session: the capturer must keep full chroma. On Windows the IDD-push
|
||||
/// capturer hands the **BGRA** slot through (skipping the subsampling BGRA→NV12
|
||||
/// VideoConverter) so NVENC ingests full-chroma RGB and CSCs to 4:4:4 itself — measured
|
||||
/// on-glass (RTX 5070 Ti): ARGB + `chromaFormatIDC=3` yields TRUE 4:4:4 and the conversion
|
||||
/// follows the configured VUI matrix (BT.709 limited since the VUI is always written). On
|
||||
/// Linux it forces the CPU RGB path the encoder swscales to `YUV444P`. `false` on every
|
||||
/// 4:2:0 session.
|
||||
pub chroma_444: bool,
|
||||
}
|
||||
|
||||
@@ -404,10 +407,11 @@ pub fn capture_virtual_output(
|
||||
// Duplication, no WGC helper). A FRESH monitor + ring is created per session: a REUSED monitor's
|
||||
// swap-chain dies after ~2 sessions and can't be revived. The ring is always FP16 when the display
|
||||
// is HDR (the driver composes the IDD in FP16); `want.hdr` proactively enables advanced color and
|
||||
// selects the per-frame conversion (FP16 → P010 vs BGRA → NV12). `IddPushCapturer` takes the
|
||||
// keepalive (it owns the virtual display). There is NO fallback (DDA + the WGC relay were removed):
|
||||
// if it can't open or the driver doesn't attach, the session fails cleanly and the client reconnects.
|
||||
idd_push::IddPushCapturer::open(target, pref, want.hdr, keep)
|
||||
// selects the per-frame conversion (FP16 → P010 vs BGRA → NV12, or BGRA → AYUV for a
|
||||
// `want.chroma_444` SDR session). `IddPushCapturer` takes the keepalive (it owns the virtual
|
||||
// display). There is NO fallback (DDA + the WGC relay were removed): if it can't open or the
|
||||
// driver doesn't attach, the session fails cleanly and the client reconnects.
|
||||
idd_push::IddPushCapturer::open(target, pref, want.hdr, want.chroma_444, keep)
|
||||
.map(|c| Box::new(c) as Box<dyn Capturer>)
|
||||
.map_err(|(e, _keep)| e.context("IDD-push capture open (no fallback)"))
|
||||
}
|
||||
@@ -422,9 +426,14 @@ pub(crate) fn capturer_supports_444() -> bool {
|
||||
}
|
||||
#[cfg(target_os = "windows")]
|
||||
pub(crate) fn capturer_supports_444() -> bool {
|
||||
// IDD-push 4:4:4 (full-chroma RGB from the FP16 ring) is the next step; until then the sole Windows
|
||||
// capturer delivers subsampled NV12/P010 only, so the host honestly negotiates 4:2:0.
|
||||
false
|
||||
// IDD-push delivers full-chroma BGRA for an SDR 4:4:4 session (skipping the NV12
|
||||
// VideoConverter) — but only the direct-NVENC backend ingests RGB and CSCs it to 4:4:4
|
||||
// (measured on-glass: true full chroma, matrix follows the configured VUI), so gate on it
|
||||
// (AMF can't 4:4:4 at all; the QSV/ffmpeg path has no RGB-input 4:4:4 wiring). An HDR
|
||||
// display can't be known here (the virtual display's mode settles after the Welcome); that
|
||||
// combination downgrades at capture time — the capturer emits P010 and the encoder's caps
|
||||
// cross-check reports the 4:2:0 truth (the in-band SPS keeps the client correct either way).
|
||||
crate::encode::windows_resolved_backend() == crate::encode::WindowsBackend::Nvenc
|
||||
}
|
||||
#[cfg(not(any(target_os = "linux", target_os = "windows")))]
|
||||
pub(crate) fn capturer_supports_444() -> bool {
|
||||
|
||||
@@ -464,21 +464,25 @@ float main(float4 pos : SV_POSITION, float2 uv : TEXCOORD0) : SV_TARGET {
|
||||
}
|
||||
";
|
||||
|
||||
/// P010 CHROMA pass PS — half-res, writes interleaved (Cb,Cr) to plane 1 (R16G16_UNORM RTV). Averages
|
||||
/// the 2x2 scRGB source footprint of this chroma sample (box filter) IN scRGB-linear space before the
|
||||
/// PQ encode, then forms Cb/Cr from the averaged-then-PQ-encoded RGB. `inv_src` = (1/srcW, 1/srcH).
|
||||
/// P010 CHROMA pass PS — half-res, writes interleaved (Cb,Cr) to plane 1 (R16G16_UNORM RTV).
|
||||
/// **Left-cosited** (H.273 chroma_loc type 0 — the default every decoder infers when
|
||||
/// chroma_loc_info is unsignaled, and what the clients' sampling corrections assume): the chroma
|
||||
/// sample sits ON the even luma column, vertically centered between its two rows — so the filter
|
||||
/// is the 2-row average of that ONE column, IN scRGB-linear space before the PQ encode, then
|
||||
/// Cb/Cr from the averaged-then-PQ-encoded RGB. (The old 2×2 box was CENTER-sited — a
|
||||
/// half-luma-pixel chroma shift against what decoders reconstruct; the narrow column decimation
|
||||
/// also keeps desktop text/edge chroma crisp, and block-uniform inputs stay exact for
|
||||
/// `hdr_p010_selftest`.) `inv_src` = (1/srcW, 1/srcH).
|
||||
const HDR_P010_UV_PS: &str = r"
|
||||
#include_common
|
||||
cbuffer C : register(b0) { float2 inv_src; float2 pad; };
|
||||
float2 main(float4 pos : SV_POSITION, float2 uv : TEXCOORD0) : SV_TARGET {
|
||||
// `uv` is the chroma-sample centre in [0,1]; the 4 co-sited luma texels sit at uv ± half a luma
|
||||
// texel in each axis. Average their scRGB (linear) values, then run the SAME PQ/CSC as the Y pass.
|
||||
// `uv` is the chroma RT texel centre = the middle of the 2x2 luma block; the left-cosited
|
||||
// target is the block's LEFT column, whose two texel centres sit at uv + (-h.x, ±h.y).
|
||||
float2 h = inv_src * 0.5;
|
||||
float3 a = max(tx.Sample(sm, uv + float2(-h.x, -h.y)).rgb, 0.0);
|
||||
float3 b = max(tx.Sample(sm, uv + float2( h.x, -h.y)).rgb, 0.0);
|
||||
float3 c = max(tx.Sample(sm, uv + float2(-h.x, h.y)).rgb, 0.0);
|
||||
float3 d = max(tx.Sample(sm, uv + float2( h.x, h.y)).rgb, 0.0);
|
||||
float3 scrgb = (a + b + c + d) * 0.25;
|
||||
float3 b = max(tx.Sample(sm, uv + float2(-h.x, h.y)).rgb, 0.0);
|
||||
float3 scrgb = (a + b) * 0.5;
|
||||
float3 nits = scrgb * 80.0;
|
||||
float3 lin2020 = mul(BT709_TO_BT2020, nits);
|
||||
float3 pq = pq_oetf(lin2020 / 10000.0);
|
||||
|
||||
@@ -669,6 +669,13 @@ pub struct IddPushCapturer {
|
||||
/// Windows mid-session. Drives the ring format (HDR → FP16 surfaces, SDR → BGRA) and the conversion.
|
||||
/// Polled in the capture loop; a change recreates the ring (see [`Self::recreate_ring`]).
|
||||
display_hdr: bool,
|
||||
/// The session negotiated full-chroma 4:4:4: while the display is SDR the BGRA slot passes
|
||||
/// THROUGH (a plain copy into the out ring, no NV12 VideoConverter) so NVENC gets full-chroma
|
||||
/// RGB and CSCs to 4:4:4 itself — measured on-glass: `chromaFormatIDC=3` + ARGB input yields
|
||||
/// TRUE 4:4:4 and the conversion follows the VUI matrix (BT.709 limited, always written).
|
||||
/// While the display is HDR this is overridden to the P010 path (no 10-bit 4:4:4 source):
|
||||
/// the stream honestly downgrades to 4:2:0 — the encoder's caps cross-check reports it.
|
||||
want_444: bool,
|
||||
/// Off-thread display-descriptor sampler (see [`DescriptorPoller`]) — the capture loop reads
|
||||
/// its snapshot instead of running CCD queries inline on the frame path.
|
||||
desc_poller: DescriptorPoller,
|
||||
@@ -824,9 +831,10 @@ impl IddPushCapturer {
|
||||
target: WinCaptureTarget,
|
||||
preferred: Option<(u32, u32, u32)>,
|
||||
client_10bit: bool,
|
||||
want_444: bool,
|
||||
keepalive: Box<dyn Send>,
|
||||
) -> std::result::Result<Self, (anyhow::Error, Box<dyn Send>)> {
|
||||
match Self::open_inner(target, preferred, client_10bit) {
|
||||
match Self::open_inner(target, preferred, client_10bit, want_444) {
|
||||
Ok(mut me) => {
|
||||
me._keepalive = keepalive;
|
||||
Ok(me)
|
||||
@@ -839,6 +847,7 @@ impl IddPushCapturer {
|
||||
target: WinCaptureTarget,
|
||||
preferred: Option<(u32, u32, u32)>,
|
||||
client_10bit: bool,
|
||||
want_444: bool,
|
||||
) -> Result<Self> {
|
||||
// The ring MUST live on the adapter the driver's swap-chain renders on. Primary: the
|
||||
// selected render GPU — the same pick SET_RENDER_ADAPTER pinned the driver to at monitor
|
||||
@@ -853,7 +862,7 @@ impl IddPushCapturer {
|
||||
LowPart: (target.adapter_luid & 0xffff_ffff) as u32,
|
||||
HighPart: (target.adapter_luid >> 32) as i32,
|
||||
});
|
||||
match Self::open_on(target.clone(), preferred, client_10bit, luid) {
|
||||
match Self::open_on(target.clone(), preferred, client_10bit, want_444, luid) {
|
||||
Ok(me) => Ok(me),
|
||||
Err(e) => {
|
||||
// Self-heal a render-adapter mismatch ONCE: on TEX_FAIL the driver has reported the
|
||||
@@ -878,7 +887,7 @@ impl IddPushCapturer {
|
||||
"IDD push: ring/driver render-adapter mismatch — rebinding the ring to the \
|
||||
driver's reported adapter"
|
||||
);
|
||||
Self::open_on(target, preferred, client_10bit, drv)
|
||||
Self::open_on(target, preferred, client_10bit, want_444, drv)
|
||||
.context("IDD-push rebind to the driver's reported render adapter")
|
||||
}
|
||||
}
|
||||
@@ -888,6 +897,7 @@ impl IddPushCapturer {
|
||||
target: WinCaptureTarget,
|
||||
preferred: Option<(u32, u32, u32)>,
|
||||
client_10bit: bool,
|
||||
want_444: bool,
|
||||
luid: LUID,
|
||||
) -> Result<Self> {
|
||||
let (pw, ph, _hz) = preferred
|
||||
@@ -1042,6 +1052,7 @@ impl IddPushCapturer {
|
||||
mode = format!("{w}x{h}"),
|
||||
display_hdr,
|
||||
client_10bit,
|
||||
want_444,
|
||||
ring_fp16 = display_hdr,
|
||||
"IDD push(host): created sealed ring + delivered the channel; waiting for the driver \
|
||||
to attach + publish"
|
||||
@@ -1060,6 +1071,7 @@ impl IddPushCapturer {
|
||||
generation,
|
||||
client_10bit,
|
||||
display_hdr,
|
||||
want_444,
|
||||
desc_poller: DescriptorPoller::spawn(
|
||||
target.target_id,
|
||||
DisplayDescriptor {
|
||||
@@ -1219,15 +1231,24 @@ impl IddPushCapturer {
|
||||
}
|
||||
}
|
||||
|
||||
/// The output texture format + the [`PixelFormat`] NVENC encodes, driven SOLELY by the DISPLAY's HDR
|
||||
/// state (like the WGC path): HDR → `P010` (BT.2020 PQ 10-bit limited) → NVENC Main10, and the client
|
||||
/// auto-detects PQ from the HEVC VUI; SDR → `Nv12` (BT.709 8-bit limited). Both are native YUV so
|
||||
/// NVENC skips its internal RGB→YUV CSC on the contended SM (plan §5.A). We do NOT gate HDR on the
|
||||
/// client's advertised `VIDEO_CAP_10BIT` — clients under-report it (e.g. the Mac advertises 10-bit
|
||||
/// only when its OWN display is HDR), yet all decode Main10 + auto-switch, exactly as on the WGC path.
|
||||
/// The output texture format + the [`PixelFormat`] NVENC encodes, driven by the DISPLAY's HDR
|
||||
/// state (like the WGC path) plus the session's 4:4:4 negotiation: HDR → `P010` (BT.2020 PQ
|
||||
/// 10-bit limited) → NVENC Main10, and the client auto-detects PQ from the HEVC VUI; SDR →
|
||||
/// `Nv12` (BT.709 8-bit limited), or full-chroma `Bgra` passthrough on a 4:4:4 session (NVENC
|
||||
/// CSCs RGB→YUV444 itself, following the BT.709 VUI — the one path that deliberately pays the
|
||||
/// SM-side CSC, because the video processor can only produce subsampled output). We do NOT
|
||||
/// gate HDR on the client's advertised `VIDEO_CAP_10BIT` — clients under-report it (e.g. the
|
||||
/// Mac advertises 10-bit only when its OWN display is HDR), yet all decode Main10 +
|
||||
/// auto-switch, exactly as on the WGC path. HDR wins over 4:4:4 (there is no 10-bit
|
||||
/// full-chroma source): the stream downgrades to 4:2:0 with a warning.
|
||||
fn out_format(&self) -> (DXGI_FORMAT, PixelFormat) {
|
||||
if self.display_hdr {
|
||||
if self.want_444 {
|
||||
warn_444_hdr_downgrade_once();
|
||||
}
|
||||
(DXGI_FORMAT_P010, PixelFormat::P010)
|
||||
} else if self.want_444 {
|
||||
(DXGI_FORMAT_B8G8R8A8_UNORM, PixelFormat::Bgra)
|
||||
} else {
|
||||
(DXGI_FORMAT_NV12, PixelFormat::Nv12)
|
||||
}
|
||||
@@ -1397,6 +1418,7 @@ impl IddPushCapturer {
|
||||
|
||||
/// Build the per-mode YUV converter if not already built: a VIDEO-engine BGRA→NV12 processor on an
|
||||
/// SDR display, or the FP16→P010 shader on an HDR display. Both keep NVENC's RGB→YUV CSC off the SM.
|
||||
/// An SDR 4:4:4 session needs NO converter — the BGRA slot passes through (see `out_format`).
|
||||
fn ensure_converter(&mut self) -> Result<()> {
|
||||
if self.display_hdr {
|
||||
if self.hdr_p010_conv.is_none() {
|
||||
@@ -1405,6 +1427,8 @@ impl IddPushCapturer {
|
||||
// belong to, and `?` propagates any failure before the converter is stored.
|
||||
self.hdr_p010_conv = Some(unsafe { HdrP010Converter::new(&self.device)? });
|
||||
}
|
||||
} else if self.want_444 {
|
||||
// Full-chroma passthrough — no conversion resources to build.
|
||||
} else if self.video_conv.is_none() {
|
||||
// SAFETY: `VideoConverter::new` is `unsafe` (it sets up the D3D11 VIDEO processor); we pass live
|
||||
// borrows of `self.device` + its immediate `self.context` (single-threaded, this thread) plus
|
||||
@@ -1509,6 +1533,11 @@ impl IddPushCapturer {
|
||||
self.height,
|
||||
)?;
|
||||
}
|
||||
} else if self.want_444 {
|
||||
// SDR 4:4:4: pass the BGRA slot through untouched — NVENC ingests full-chroma
|
||||
// RGB and CSCs to YUV 4:4:4 itself (per the always-written BT.709 VUI). Plain
|
||||
// copy-engine move; the slot releases back to the driver immediately.
|
||||
self.context.CopyResource(&out, &s.tex);
|
||||
} else {
|
||||
// SDR: BGRA slot → NV12 on the VIDEO engine; NVENC takes native NV12, no SM-side CSC.
|
||||
if let Some(conv) = self.video_conv.as_ref() {
|
||||
@@ -1672,6 +1701,21 @@ impl Capturer for IddPushCapturer {
|
||||
}
|
||||
}
|
||||
|
||||
/// A 4:4:4 session while the display is HDR: there is no 10-bit full-chroma source (the FP16
|
||||
/// desktop needs the PQ tone curve, which the P010 shader provides at 4:2:0), so the stream
|
||||
/// honestly downgrades — the encoder's `chroma_444` caps cross-check reports it and the in-band
|
||||
/// SPS keeps the client decoding correctly. Once per process: the state can flap mid-session.
|
||||
fn warn_444_hdr_downgrade_once() {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
static ONCE: AtomicBool = AtomicBool::new(true);
|
||||
if ONCE.swap(false, Ordering::Relaxed) {
|
||||
tracing::warn!(
|
||||
"4:4:4 negotiated but the display is HDR — no 10-bit full-chroma source exists; \
|
||||
encoding HDR 4:2:0 (P010) instead (disable HDR on the virtual display for 4:4:4)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for IddPushCapturer {
|
||||
fn drop(&mut self) {
|
||||
self.slots.clear();
|
||||
|
||||
@@ -326,11 +326,19 @@ impl NvencEncoder {
|
||||
};
|
||||
}
|
||||
|
||||
// NV12 / 4:4:4 paths: we do the RGB→YUV conversion ourselves as BT.709 *limited* range
|
||||
// (swscale), so signal that in the bitstream VUI (colorspace/range/primaries/transfer) —
|
||||
// otherwise the client decoder assumes a default and the picture comes out washed-out /
|
||||
// wrong-contrast. The RGB-input 4:2:0 path leaves these unset (NVENC's internal CSC writes
|
||||
// its own VUI). Matches the Windows NV12 path's BT.709 limited-range signalling.
|
||||
// NV12 / 4:4:4 paths: we do the RGB→YUV conversion ourselves as BT.709 (swscale), so
|
||||
// signal that in the bitstream VUI (colorspace/range/primaries/transfer) — otherwise the
|
||||
// client decoder assumes a default and the picture comes out washed-out / wrong-contrast.
|
||||
// The RGB-input 4:2:0 path leaves these unset (NVENC's internal CSC writes its own VUI).
|
||||
// Matches the Windows NV12 path's BT.709 limited-range signalling.
|
||||
//
|
||||
// PUNKTFUNK_444_FULLRANGE=1 (experimental, 4:4:4-only): convert AND signal FULL range —
|
||||
// recovers the ~12% of code space limited-range quantization gives up, for the exact
|
||||
// text/UI chroma 4:4:4 exists for. Every punktfunk client honors the signaled range
|
||||
// (csc_rows / the Apple rows port); ship as default only if the on-glass A/B shows a
|
||||
// visible win. Linux-only: the Windows path's NVENC-internal CSC range is unmeasured.
|
||||
let full_range_444 = want_444
|
||||
&& std::env::var("PUNKTFUNK_444_FULLRANGE").is_ok_and(|v| v.trim() == "1");
|
||||
if matches!(format, PixelFormat::Nv12) || want_444 {
|
||||
// SAFETY: same `video` builder — `raw = video.as_mut_ptr()` is the non-null, properly-
|
||||
// aligned, sole-owned, not-yet-opened `AVCodecContext`. We set its four VUI colour enum
|
||||
@@ -339,7 +347,11 @@ impl NvencEncoder {
|
||||
unsafe {
|
||||
let raw = video.as_mut_ptr();
|
||||
(*raw).colorspace = ffi::AVColorSpace::AVCOL_SPC_BT709;
|
||||
(*raw).color_range = ffi::AVColorRange::AVCOL_RANGE_MPEG; // limited/studio
|
||||
(*raw).color_range = if full_range_444 {
|
||||
ffi::AVColorRange::AVCOL_RANGE_JPEG // full
|
||||
} else {
|
||||
ffi::AVColorRange::AVCOL_RANGE_MPEG // limited/studio
|
||||
};
|
||||
(*raw).color_primaries = ffi::AVColorPrimaries::AVCOL_PRI_BT709;
|
||||
(*raw).color_trc = ffi::AVColorTransferCharacteristic::AVCOL_TRC_BT709;
|
||||
}
|
||||
@@ -401,10 +413,12 @@ impl NvencEncoder {
|
||||
// SAFETY: `sws` is the non-null context from the call above (null-checked). The ITU-709
|
||||
// coefficient table from `sws_getCoefficients` is a process-lifetime libswscale static,
|
||||
// reused for src+dst matrices; `sws_setColorspaceDetails` only reads it and writes scalar
|
||||
// CSC settings into `sws` (limited-range dst: dstRange = 0). No Rust memory is passed.
|
||||
// CSC settings into `sws` (dstRange matches the VUI: 0 = limited, 1 = the
|
||||
// PUNKTFUNK_444_FULLRANGE experiment). No Rust memory is passed.
|
||||
unsafe {
|
||||
let cs709 = ffi::sws_getCoefficients(SWS_CS_ITU709);
|
||||
ffi::sws_setColorspaceDetails(sws, cs709, 1, cs709, 0, 0, 1 << 16, 1 << 16);
|
||||
let dst_range = i32::from(full_range_444);
|
||||
ffi::sws_setColorspaceDetails(sws, cs709, 1, cs709, dst_range, 0, 1 << 16, 1 << 16);
|
||||
}
|
||||
Some(sws)
|
||||
} else {
|
||||
|
||||
@@ -204,8 +204,9 @@ unsafe fn open_vaapi_encoder_mode(
|
||||
let raw = video.as_mut_ptr();
|
||||
(*raw).rc_buffer_size = vbv_bits as i32;
|
||||
(*raw).gop_size = i32::MAX; // no periodic IDR (forced-IDR via pict_type=I on RFI)
|
||||
// We hand the encoder BT.709 *limited* NV12 (swscale CSC, or scale_vaapi which preserves the
|
||||
// input range we tag), so signal that VUI — else the client decoder washes the picture out.
|
||||
// We hand the encoder BT.709 *limited* NV12 (swscale CSC on the CPU path; scale_vaapi pinned
|
||||
// to `out_color_matrix=bt709:out_range=limited` on the zero-copy path, with the full-range
|
||||
// RGB input tagged), so signal that VUI — else the client decoder washes the picture out.
|
||||
(*raw).colorspace = ffi::AVColorSpace::AVCOL_SPC_BT709;
|
||||
(*raw).color_range = ffi::AVColorRange::AVCOL_RANGE_MPEG;
|
||||
(*raw).color_primaries = ffi::AVColorPrimaries::AVCOL_PRI_BT709;
|
||||
@@ -718,6 +719,11 @@ impl DmabufInner {
|
||||
(*par).format = ffi::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as c_int;
|
||||
(*par).width = width as c_int;
|
||||
(*par).height = height as c_int;
|
||||
// Declare the link's colour up front (full-range RGB — the compositor's desktop) so
|
||||
// the per-frame tags in `submit` match the negotiated link instead of reading as a
|
||||
// mid-stream property change.
|
||||
(*par).color_space = ffi::AVColorSpace::AVCOL_SPC_RGB;
|
||||
(*par).color_range = ffi::AVColorRange::AVCOL_RANGE_JPEG;
|
||||
(*par).time_base = ffi::AVRational {
|
||||
num: 1,
|
||||
den: fps as c_int,
|
||||
@@ -751,7 +757,14 @@ impl DmabufInner {
|
||||
}
|
||||
init!(src, ptr::null(), "buffer");
|
||||
init!(hwmap, c"mode=read".as_ptr(), "hwmap");
|
||||
init!(scale, c"format=nv12".as_ptr(), "scale_vaapi");
|
||||
// Pin the VPP's output colour to what the encoder's VUI signals (BT.709 limited).
|
||||
// Without the explicit options the conversion matrix is whatever the driver defaults
|
||||
// to for an unspecified output (Mesa: BT.601) — a hue shift against the signaled VUI.
|
||||
init!(
|
||||
scale,
|
||||
c"format=nv12:out_color_matrix=bt709:out_range=limited".as_ptr(),
|
||||
"scale_vaapi"
|
||||
);
|
||||
init!(sink, ptr::null(), "buffersink");
|
||||
|
||||
let link = |a: *mut ffi::AVFilterContext, b: *mut ffi::AVFilterContext| -> c_int {
|
||||
@@ -879,6 +892,12 @@ impl DmabufInner {
|
||||
(*drm).format = ffi::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as c_int;
|
||||
(*drm).width = self.width as c_int;
|
||||
(*drm).height = self.height as c_int;
|
||||
// The dmabuf is the compositor's rendered desktop: full-range RGB. Tag the frame so
|
||||
// the VPP's colour negotiation sees the real input instead of "unspecified" (an
|
||||
// untagged input lets the driver pick its own default for the RGB→NV12 conversion —
|
||||
// Mesa's is BT.601, contradicting the BT.709-limited VUI the encoder signals).
|
||||
(*drm).color_range = ffi::AVColorRange::AVCOL_RANGE_JPEG;
|
||||
(*drm).colorspace = ffi::AVColorSpace::AVCOL_SPC_RGB;
|
||||
(*drm).hw_frames_ctx = ffi::av_buffer_ref(self.drm_frames);
|
||||
(*drm).data[0] = Box::into_raw(desc) as *mut u8;
|
||||
// Own the descriptor so it frees with the frame (the fd is owned by the DmabufFrame,
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
//! Software H.264 encoder (openh264) — the GPU-less encode path for the Windows host (and a
|
||||
//! fallback when NVENC is unavailable). Low-latency screen-content config: single-reference,
|
||||
//! no B-frames (Baseline), bitrate rate-control, in-band SPS/PPS each IDR, BT.709 limited range.
|
||||
//! no B-frames (Baseline), bitrate rate-control, in-band SPS/PPS each IDR.
|
||||
//! Synchronous: `submit` encodes immediately and stashes the AU for `poll` (no internal queue).
|
||||
//!
|
||||
//! The RGB→YUV conversion is OURS, BT.709 limited range: openh264 writes no colour description
|
||||
//! into the VUI (unspecified), so decoders fall back to their default — BT.709 limited on every
|
||||
//! punktfunk client — and the pixels must match that default. The crate's own `YUVBuffer`
|
||||
//! converter is BT.601 (0.2578/0.5039/0.0977 + 16), which decoded-as-709 is a constant hue
|
||||
//! error; that's why it is NOT used here.
|
||||
// Every `unsafe` block in this file carries a `// SAFETY:` proof; enforce it (unsafe-proof program).
|
||||
#![deny(clippy::undocumented_unsafe_blocks)]
|
||||
|
||||
@@ -12,19 +18,20 @@ use openh264::encoder::{
|
||||
BitRate, Complexity, Encoder as Oh264, EncoderConfig, FrameRate, FrameType, IntraFramePeriod,
|
||||
Profile, RateControlMode, SpsPpsStrategy, UsageType,
|
||||
};
|
||||
use openh264::formats::{BgraSliceU8, RgbSliceU8, YUVBuffer};
|
||||
use openh264::formats::YUVSlices;
|
||||
use openh264::OpenH264API;
|
||||
|
||||
pub struct OpenH264Encoder {
|
||||
enc: Oh264,
|
||||
yuv: YUVBuffer,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
src_format: PixelFormat,
|
||||
/// BGRA scratch for the 3-bpp (Bgr) and R/B-swapped (Rgba/Rgbx) formats openh264 can't wrap
|
||||
/// directly. Reused across frames.
|
||||
scratch: Vec<u8>,
|
||||
/// The converted I420 planes (our BT.709-limited CSC — see the module doc), reused across
|
||||
/// frames: full-res luma + quarter-res Cb/Cr, tightly packed (stride = width, width/2).
|
||||
y_plane: Vec<u8>,
|
||||
u_plane: Vec<u8>,
|
||||
v_plane: Vec<u8>,
|
||||
frame_idx: i64,
|
||||
force_kf: bool,
|
||||
/// At most one AU per submit (no lookahead), handed back by the next `poll`.
|
||||
@@ -33,7 +40,7 @@ pub struct OpenH264Encoder {
|
||||
|
||||
// openh264's Encoder holds a raw C handle (not auto-Send); it lives on the single encode thread.
|
||||
// SAFETY: `OpenH264Encoder` wraps `Oh264` (openh264's `Encoder`), which holds a raw C handle to the
|
||||
// openh264 `ISVCEncoder` and is not auto-`Send`; the other fields (`YUVBuffer`, `Vec`, scalars,
|
||||
// openh264 `ISVCEncoder` and is not auto-`Send`; the other fields (the plane `Vec`s, scalars,
|
||||
// `Option<EncodedFrame>`) are plain owned data. The session creates the encoder, calls
|
||||
// `submit`/`poll`/`flush`, and drops it all on one dedicated encode thread, never sharing it by
|
||||
// reference across threads, so the C handle is only ever touched from a single thread. Moving the
|
||||
@@ -62,51 +69,75 @@ impl OpenH264Encoder {
|
||||
.scene_change_detect(false) // no surprise IDRs (bitrate spikes / freeze)
|
||||
.adaptive_quantization(true)
|
||||
.complexity(Complexity::Low) // latency over BD-rate
|
||||
.profile(Profile::Baseline); // no B-frames; BT.709 limited is the crate default VUI
|
||||
.profile(Profile::Baseline); // no B-frames; the VUI carries no colour description
|
||||
let api = OpenH264API::from_source(); // statically-bundled build (default `source` feature)
|
||||
let enc = Oh264::with_api_config(api, cfg).context("openh264 Encoder::with_api_config")?;
|
||||
let yuv = YUVBuffer::new(width as usize, height as usize);
|
||||
let (w, h) = (width as usize, height as usize);
|
||||
tracing::info!(
|
||||
"openh264 software encoder: {width}x{height}@{fps} {} Mbps (Baseline, screen-content)",
|
||||
bps / 1_000_000
|
||||
);
|
||||
Ok(Self {
|
||||
enc,
|
||||
yuv,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
src_format: format,
|
||||
scratch: Vec::new(),
|
||||
y_plane: vec![0; w * h],
|
||||
u_plane: vec![0; (w / 2) * (h / 2)],
|
||||
v_plane: vec![0; (w / 2) * (h / 2)],
|
||||
frame_idx: 0,
|
||||
force_kf: false,
|
||||
pending: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Normalize a packed source buffer into the reused BGRA `scratch` ([B,G,R,A]). `rgb_order`
|
||||
/// = source is R,G,B (swap into B,G,R); otherwise source is already B,G,R.
|
||||
fn normalize_to_bgra(&mut self, src: &[u8], src_bpp: usize, rgb_order: bool) {
|
||||
/// Convert one packed full-range RGB frame into the I420 planes, BT.709 limited range.
|
||||
/// `bpp` is the source pixel stride; `ri`/`gi`/`bi` the channel byte offsets within a pixel.
|
||||
/// Luma per pixel; Cb/Cr from the 2×2 block's averaged RGB (the same box filter the crate's
|
||||
/// converter used, so only the matrix changed).
|
||||
fn convert_bt709(&mut self, src: &[u8], bpp: usize, ri: usize, gi: usize, bi: usize) {
|
||||
let w = self.width as usize;
|
||||
let h = self.height as usize;
|
||||
self.scratch.resize(w * h * 4, 0);
|
||||
for px in 0..(w * h) {
|
||||
let s = &src[px * src_bpp..px * src_bpp + 3];
|
||||
let d = &mut self.scratch[px * 4..px * 4 + 4];
|
||||
if rgb_order {
|
||||
d[0] = s[2];
|
||||
d[1] = s[1];
|
||||
d[2] = s[0];
|
||||
} else {
|
||||
d[0] = s[0];
|
||||
d[1] = s[1];
|
||||
d[2] = s[2];
|
||||
let cw = w / 2;
|
||||
for by in 0..h / 2 {
|
||||
for bx in 0..cw {
|
||||
let mut sum = (0f32, 0f32, 0f32);
|
||||
for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {
|
||||
let (px, py) = (bx * 2 + dx, by * 2 + dy);
|
||||
let s = &src[(py * w + px) * bpp..];
|
||||
let (r, g, b) = (f32::from(s[ri]), f32::from(s[gi]), f32::from(s[bi]));
|
||||
self.y_plane[py * w + px] = luma709(r, g, b);
|
||||
sum = (sum.0 + r, sum.1 + g, sum.2 + b);
|
||||
}
|
||||
let (cb, cr) = chroma709(sum.0 / 4.0, sum.1 / 4.0, sum.2 / 4.0);
|
||||
self.u_plane[by * cw + bx] = cb;
|
||||
self.v_plane[by * cw + bx] = cr;
|
||||
}
|
||||
d[3] = 0xff;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// BT.709 luma coefficients (Kg = 1 − Kr − Kb).
|
||||
const KR: f32 = 0.2126;
|
||||
const KB: f32 = 0.0722;
|
||||
const KG: f32 = 1.0 - KR - KB;
|
||||
|
||||
/// One full-range RGB pixel (0..=255 channels) → the BT.709 limited-range 8-bit luma code
|
||||
/// (16..=235). Kept in lockstep with the client-side inverse (`pf-client-core::video::csc_rows`).
|
||||
fn luma709(r: f32, g: f32, b: f32) -> u8 {
|
||||
let y = KR * r + KG * g + KB * b; // full-scale luma, 0..=255
|
||||
(16.0 + y * (219.0 / 255.0) + 0.5) as u8 // `as` saturates — no manual clamp needed
|
||||
}
|
||||
|
||||
/// (Averaged) full-range RGB → the BT.709 limited-range Cb/Cr codes (16..=240, neutral 128).
|
||||
fn chroma709(r: f32, g: f32, b: f32) -> (u8, u8) {
|
||||
let y = KR * r + KG * g + KB * b;
|
||||
let cb = 128.0 + (b - y) * (224.0 / 255.0) / (2.0 * (1.0 - KB));
|
||||
let cr = 128.0 + (r - y) * (224.0 / 255.0) / (2.0 * (1.0 - KR));
|
||||
((cb + 0.5) as u8, (cr + 0.5) as u8)
|
||||
}
|
||||
|
||||
impl Encoder for OpenH264Encoder {
|
||||
fn submit(&mut self, captured: &CapturedFrame) -> Result<()> {
|
||||
ensure!(
|
||||
@@ -139,21 +170,13 @@ impl Encoder for OpenH264Encoder {
|
||||
self.src_format
|
||||
);
|
||||
|
||||
match self.src_format {
|
||||
PixelFormat::Rgb => self
|
||||
.yuv
|
||||
.read_rgb(RgbSliceU8::new(&bytes[..w * h * 3], (w, h))),
|
||||
PixelFormat::Bgra | PixelFormat::Bgrx => self
|
||||
.yuv
|
||||
.read_rgb(BgraSliceU8::new(&bytes[..w * h * 4], (w, h))),
|
||||
PixelFormat::Rgba | PixelFormat::Rgbx => {
|
||||
self.normalize_to_bgra(bytes, 4, true);
|
||||
self.yuv.read_rgb(BgraSliceU8::new(&self.scratch, (w, h)));
|
||||
}
|
||||
PixelFormat::Bgr => {
|
||||
self.normalize_to_bgra(bytes, 3, false);
|
||||
self.yuv.read_rgb(BgraSliceU8::new(&self.scratch, (w, h)));
|
||||
}
|
||||
// Source pixel stride + R/G/B byte offsets within a pixel — one converter for every
|
||||
// packed-RGB layout the capturers emit (no BGRA normalization pass needed).
|
||||
let (bpp, ri, gi, bi) = match self.src_format {
|
||||
PixelFormat::Rgb => (3, 0, 1, 2),
|
||||
PixelFormat::Bgr => (3, 2, 1, 0),
|
||||
PixelFormat::Rgba | PixelFormat::Rgbx => (4, 0, 1, 2),
|
||||
PixelFormat::Bgra | PixelFormat::Bgrx => (4, 2, 1, 0),
|
||||
// 10-bit HDR comes only from the GPU NVENC path; the software 8-bit H.264 encoder
|
||||
// can't represent it (and never receives it — the capturer pairs Rgb10a2 with NVENC).
|
||||
PixelFormat::Rgb10a2 => {
|
||||
@@ -166,13 +189,19 @@ impl Encoder for OpenH264Encoder {
|
||||
"software encoder cannot encode YUV GPU textures (NV12/P010 → NVENC only)"
|
||||
)
|
||||
}
|
||||
}
|
||||
};
|
||||
self.convert_bt709(bytes, bpp, ri, gi, bi);
|
||||
|
||||
if self.force_kf {
|
||||
self.enc.force_intra_frame();
|
||||
self.force_kf = false;
|
||||
}
|
||||
let bs = self.enc.encode(&self.yuv).context("openh264 encode")?;
|
||||
let slices = YUVSlices::new(
|
||||
(&self.y_plane, &self.u_plane, &self.v_plane),
|
||||
(w, h),
|
||||
(w, w / 2, w / 2),
|
||||
);
|
||||
let bs = self.enc.encode(&slices).context("openh264 encode")?;
|
||||
let mut data = Vec::new();
|
||||
bs.write_vec(&mut data); // AnnexB start codes; SPS/PPS prepended on IDR
|
||||
if !data.is_empty() {
|
||||
@@ -225,6 +254,47 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::capture::{CapturedFrame, FramePayload, PixelFormat};
|
||||
|
||||
/// The BT.709 limited-range anchor points: reference white → (235,128,128), black →
|
||||
/// (16,128,128), pure red's Cr must hit the positive extreme 240 (it does exactly:
|
||||
/// 255(1−Kr)·(224/255)/(2(1−Kr)) = 112). ±1 code for float rounding.
|
||||
#[test]
|
||||
fn bt709_conversion_anchor_points() {
|
||||
assert_eq!(luma709(255.0, 255.0, 255.0), 235);
|
||||
assert_eq!(luma709(0.0, 0.0, 0.0), 16);
|
||||
assert_eq!(chroma709(255.0, 255.0, 255.0), (128, 128));
|
||||
assert_eq!(chroma709(0.0, 0.0, 0.0), (128, 128));
|
||||
let (cb, cr) = chroma709(255.0, 0.0, 0.0);
|
||||
assert_eq!(cr, 240, "pure red must reach the Cr extreme");
|
||||
assert!((101..=103).contains(&cb), "red Cb ~102, got {cb}");
|
||||
let (cb, _) = chroma709(0.0, 0.0, 255.0);
|
||||
assert_eq!(cb, 240, "pure blue must reach the Cb extreme");
|
||||
}
|
||||
|
||||
/// The 601-vs-709 luma split on pure green (Kg 0.587 vs 0.7152) — guards against anyone
|
||||
/// "simplifying" the coefficients back to the crate's BT.601 converter (the hue-shift bug
|
||||
/// this module's own conversion exists to prevent).
|
||||
#[test]
|
||||
fn bt709_is_not_bt601() {
|
||||
// BT.601 green luma: 16 + 219·0.587 = 144.5; BT.709: 16 + 219·0.7152 = 172.6.
|
||||
let y = luma709(0.0, 255.0, 0.0);
|
||||
assert!((172..=174).contains(&y), "709 green luma ~173, got {y}");
|
||||
}
|
||||
|
||||
/// A flat gray frame converts to neutral chroma and mid luma across every plane byte
|
||||
/// (exercises the block loop + plane sizing, not just the per-pixel math).
|
||||
#[test]
|
||||
fn converts_flat_gray_to_neutral_planes() {
|
||||
let (w, h) = (16u32, 8u32);
|
||||
let mut enc =
|
||||
OpenH264Encoder::open(PixelFormat::Bgrx, w, h, 60, 1_000_000).expect("open openh264");
|
||||
let bytes = vec![0x80u8; (w * h * 4) as usize];
|
||||
enc.convert_bt709(&bytes, 4, 2, 1, 0);
|
||||
// 16 + 128·(219/255) = 125.9 → 126.
|
||||
assert!(enc.y_plane.iter().all(|&y| y == 126), "{:?}", &enc.y_plane[..4]);
|
||||
assert!(enc.u_plane.iter().all(|&u| u == 128));
|
||||
assert!(enc.v_plane.iter().all(|&v| v == 128));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encodes_synthetic_frame_to_annexb_idr() {
|
||||
let (w, h, fps) = (1280u32, 720u32, 60u32);
|
||||
|
||||
@@ -708,11 +708,13 @@ impl NvencD3d11Encoder {
|
||||
// input — a subsampled NV12/P010 source can't reconstruct full chroma (so the capturer is
|
||||
// forced to RGB for a 4:4:4 session, and we guard on the input format here too).
|
||||
//
|
||||
// ON-GLASS TODO (RTX box): confirm ARGB + chromaFormatIDC=3 + FREXT yields a *true* 4:4:4
|
||||
// stream. NVENC's RGB→YUV CSC is documented to honor chromaFormatIDC (unlike libavcodec's
|
||||
// wrapper, which always subsamples RGB to 4:2:0 — hence the Linux path feeds planar YUV444
|
||||
// instead). If on-glass shows 4:2:0, the follow-up is a BGRA→AYUV shader feeding the native
|
||||
// `NV_ENC_BUFFER_FORMAT_AYUV` 4:4:4 input format.
|
||||
// ON-GLASS MEASURED (RTX 5070 Ti, driver 610.43, 2026-07-10 — `nvenc_444_on_glass_probe`
|
||||
// below + colour-bar analysis): ARGB + chromaFormatIDC=3 + FREXT yields a TRUE 4:4:4
|
||||
// stream (1-px chroma stripes survive, adjacent-column |dU| ≈ 138), and NVENC's internal
|
||||
// RGB→YUV conversion FOLLOWS THE CONFIGURED VUI MATRIX (bars match BT.709 within ±1 code
|
||||
// with our 709 VUI; the same driver produces exact BT.601 when libavcodec's nvenc wrapper
|
||||
// sets its BT470BG VUI on Linux). The always-written SDR VUI above therefore makes the
|
||||
// pixels and the signaling agree by construction — no AYUV shader needed.
|
||||
let rgb_input = matches!(
|
||||
self.buffer_fmt,
|
||||
nv::NV_ENC_BUFFER_FORMAT::NV_ENC_BUFFER_FORMAT_ARGB
|
||||
@@ -752,21 +754,33 @@ impl NvencD3d11Encoder {
|
||||
}
|
||||
}
|
||||
|
||||
// HDR colour signaling: BT.2020 primaries + SMPTE ST.2084 (PQ) transfer + BT.2020-NCL
|
||||
// matrix, limited (studio) range — NVENC's RGB→YUV default. HEVC/H.264 carry it in the VUI;
|
||||
// AV1 has NO VUI, so the SAME CICP code points go in the sequence-header colour config
|
||||
// (`colorPrimaries`/`transferCharacteristics`/`matrixCoefficients`/`colorRange`). Without
|
||||
// this a non-HEVC decoder assumes BT.709 SDR → washed-out / colour-shifted HDR.
|
||||
// Colour signaling, written UNCONDITIONALLY (was HDR-only): the capturer hands NVENC
|
||||
// pre-converted NV12 (BT.709 limited, the IDD VideoConverter) or P010 (BT.2020 PQ
|
||||
// limited, the FP16→P010 shader), so the stream must SAY so — an SDR stream with no
|
||||
// colour description decodes correctly only on clients whose "unspecified" default
|
||||
// happens to be BT.709 limited (ours are, but Moonlight/third-party/Android-vendor
|
||||
// decoders default 601 at sub-HD resolutions). HEVC/H.264 carry it in the VUI; AV1 has
|
||||
// NO VUI, so the SAME CICP code points go in the sequence-header colour config
|
||||
// (`colorPrimaries`/`transferCharacteristics`/`matrixCoefficients`/`colorRange`).
|
||||
//
|
||||
// This is the per-stream colour *description* only. The static mastering-display (ST.2086)
|
||||
// and content-light (MaxCLL/MaxFALL) metadata — HEVC SEI / AV1 METADATA OBUs — is a
|
||||
// separate follow-up, as is wiring AV1/H.264 to a true 10-bit (Main10) encode (only HEVC
|
||||
// sets Main10 above today).
|
||||
if self.hdr {
|
||||
let prim = nv::NV_ENC_VUI_COLOR_PRIMARIES::NV_ENC_VUI_COLOR_PRIMARIES_BT2020;
|
||||
let trc =
|
||||
nv::NV_ENC_VUI_TRANSFER_CHARACTERISTIC::NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE2084;
|
||||
let mat = nv::NV_ENC_VUI_MATRIX_COEFFS::NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL;
|
||||
{
|
||||
let (prim, trc, mat) = if self.hdr {
|
||||
(
|
||||
nv::NV_ENC_VUI_COLOR_PRIMARIES::NV_ENC_VUI_COLOR_PRIMARIES_BT2020,
|
||||
nv::NV_ENC_VUI_TRANSFER_CHARACTERISTIC::NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE2084,
|
||||
nv::NV_ENC_VUI_MATRIX_COEFFS::NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL,
|
||||
)
|
||||
} else {
|
||||
(
|
||||
nv::NV_ENC_VUI_COLOR_PRIMARIES::NV_ENC_VUI_COLOR_PRIMARIES_BT709,
|
||||
nv::NV_ENC_VUI_TRANSFER_CHARACTERISTIC::NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709,
|
||||
nv::NV_ENC_VUI_MATRIX_COEFFS::NV_ENC_VUI_MATRIX_COEFFS_BT709,
|
||||
)
|
||||
};
|
||||
match self.codec {
|
||||
Codec::H265 => {
|
||||
let vui = &mut cfg.encodeCodecConfig.hevcConfig.hevcVUIParameters;
|
||||
@@ -1160,6 +1174,24 @@ impl Encoder for NvencD3d11Encoder {
|
||||
}
|
||||
_ => nv::NV_ENC_BUFFER_FORMAT::NV_ENC_BUFFER_FORMAT_ARGB,
|
||||
};
|
||||
// 4:4:4 honesty: the FREXT/chromaFormatIDC=3 config engages only on an RGB input (a
|
||||
// subsampled NV12/P010 source can't reconstruct full chroma). If the capturer handed
|
||||
// native YUV despite a 4:4:4 negotiation, this session encodes 4:2:0 — clear the flag
|
||||
// NOW so `caps().chroma_444` (and punktfunk1's post-open cross-check) reports what
|
||||
// the stream really carries instead of silently claiming full chroma.
|
||||
if self.chroma_444
|
||||
&& !matches!(
|
||||
self.buffer_fmt,
|
||||
nv::NV_ENC_BUFFER_FORMAT::NV_ENC_BUFFER_FORMAT_ARGB
|
||||
| nv::NV_ENC_BUFFER_FORMAT::NV_ENC_BUFFER_FORMAT_ABGR10
|
||||
)
|
||||
{
|
||||
tracing::warn!(
|
||||
format = ?captured.format,
|
||||
"4:4:4 negotiated but the capturer delivered subsampled YUV — encoding 4:2:0"
|
||||
);
|
||||
self.chroma_444 = false;
|
||||
}
|
||||
let device = frame.device.clone();
|
||||
self.init_session(&device)?;
|
||||
self.init_device = dev_raw;
|
||||
@@ -1573,3 +1605,162 @@ pub fn probe_can_encode_444(codec: Codec) -> bool {
|
||||
ok
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::capture::{dxgi::D3d11Frame, CapturedFrame, FramePayload};
|
||||
use windows::Win32::Graphics::Direct3D11::{
|
||||
D3D11_BIND_RENDER_TARGET, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC,
|
||||
D3D11_USAGE_DEFAULT,
|
||||
};
|
||||
use windows::Win32::Graphics::Dxgi::Common::{
|
||||
DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_SAMPLE_DESC,
|
||||
};
|
||||
use windows::Win32::Graphics::Dxgi::{
|
||||
CreateDXGIFactory1, IDXGIFactory1, DXGI_ADAPTER_FLAG_SOFTWARE,
|
||||
};
|
||||
|
||||
/// The 8 fully-saturated colour bars the matrix analysis samples (RGB). Saturated primaries
|
||||
/// separate BT.601 from BT.709 by tens of code points (e.g. pure-green luma 145 vs 173).
|
||||
const BARS: [(u8, u8, u8); 8] = [
|
||||
(255, 255, 255), // white
|
||||
(255, 255, 0), // yellow
|
||||
(0, 255, 255), // cyan
|
||||
(0, 255, 0), // green
|
||||
(255, 0, 255), // magenta
|
||||
(255, 0, 0), // red
|
||||
(0, 0, 255), // blue
|
||||
(0, 0, 0), // black
|
||||
];
|
||||
|
||||
/// BGRA probe pattern: left half = the 8 colour bars (flat patches → matrix measurement),
|
||||
/// right half = alternating 1-px red/blue columns (the chroma-resolution litmus: true 4:4:4
|
||||
/// keeps adjacent columns' chroma distinct; an internally-subsampled encode blends them).
|
||||
fn probe_pattern(w: usize, h: usize) -> Vec<u8> {
|
||||
let mut px = vec![0u8; w * h * 4];
|
||||
let bar_w = (w / 2) / BARS.len();
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let (r, g, b) = if x < w / 2 {
|
||||
BARS[(x / bar_w).min(BARS.len() - 1)]
|
||||
} else if x % 2 == 0 {
|
||||
(255, 0, 0) // red column
|
||||
} else {
|
||||
(0, 0, 255) // blue column
|
||||
};
|
||||
let o = (y * w + x) * 4;
|
||||
px[o] = b;
|
||||
px[o + 1] = g;
|
||||
px[o + 2] = r;
|
||||
px[o + 3] = 255;
|
||||
}
|
||||
}
|
||||
px
|
||||
}
|
||||
|
||||
/// Encode 30 static pattern frames through the real NVENC session (ARGB input, the exact
|
||||
/// production configuration) at the given chroma and write the Annex-B stream to `path`.
|
||||
fn encode_pattern(chroma: ChromaFormat, path: &str) {
|
||||
const W: u32 = 1280;
|
||||
const H: u32 = 720;
|
||||
// SAFETY (test-only): straight-line D3D11/DXGI COM calls on one thread; every out-pointer
|
||||
// is checked before use; the texture/device outlive the encoder (dropped at scope end).
|
||||
unsafe {
|
||||
let factory: IDXGIFactory1 = CreateDXGIFactory1().expect("DXGI factory");
|
||||
let mut adapter = None;
|
||||
for i in 0.. {
|
||||
let Ok(a) = factory.EnumAdapters1(i) else { break };
|
||||
let desc = a.GetDesc1().expect("adapter desc");
|
||||
if desc.Flags & DXGI_ADAPTER_FLAG_SOFTWARE.0 as u32 == 0 {
|
||||
adapter = Some(a);
|
||||
break;
|
||||
}
|
||||
}
|
||||
let adapter = adapter.expect("no hardware DXGI adapter");
|
||||
let (device, _ctx) =
|
||||
crate::capture::dxgi::make_device(&adapter).expect("make_device");
|
||||
|
||||
let bytes = probe_pattern(W as usize, H as usize);
|
||||
let init = D3D11_SUBRESOURCE_DATA {
|
||||
pSysMem: bytes.as_ptr() as *const _,
|
||||
SysMemPitch: W * 4,
|
||||
SysMemSlicePitch: 0,
|
||||
};
|
||||
let desc = D3D11_TEXTURE2D_DESC {
|
||||
Width: W,
|
||||
Height: H,
|
||||
MipLevels: 1,
|
||||
ArraySize: 1,
|
||||
Format: DXGI_FORMAT_B8G8R8A8_UNORM,
|
||||
SampleDesc: DXGI_SAMPLE_DESC {
|
||||
Count: 1,
|
||||
Quality: 0,
|
||||
},
|
||||
Usage: D3D11_USAGE_DEFAULT,
|
||||
// NVENC registration requires RENDER_TARGET on D3D11 input textures.
|
||||
BindFlags: D3D11_BIND_RENDER_TARGET.0 as u32,
|
||||
CPUAccessFlags: 0,
|
||||
MiscFlags: 0,
|
||||
};
|
||||
let mut tex = None;
|
||||
device
|
||||
.CreateTexture2D(&desc, Some(&init), Some(&mut tex))
|
||||
.expect("pattern texture");
|
||||
let tex = tex.expect("null pattern texture");
|
||||
|
||||
let mut enc = NvencD3d11Encoder::open(
|
||||
Codec::H265,
|
||||
PixelFormat::Bgra,
|
||||
W,
|
||||
H,
|
||||
60,
|
||||
100_000_000, // high rate: the 1-px stripes must survive quantization
|
||||
8,
|
||||
chroma,
|
||||
)
|
||||
.expect("NVENC open");
|
||||
let mut out = Vec::new();
|
||||
for i in 0..30u64 {
|
||||
let frame = CapturedFrame {
|
||||
width: W,
|
||||
height: H,
|
||||
pts_ns: i * 16_666_667,
|
||||
format: PixelFormat::Bgra,
|
||||
payload: FramePayload::D3d11(D3d11Frame {
|
||||
texture: tex.clone(),
|
||||
device: device.clone(),
|
||||
}),
|
||||
};
|
||||
enc.submit(&frame).expect("submit");
|
||||
while let Some(au) = enc.poll().expect("poll") {
|
||||
out.extend_from_slice(&au.data);
|
||||
}
|
||||
}
|
||||
enc.flush().ok();
|
||||
while let Ok(Some(au)) = enc.poll() {
|
||||
out.extend_from_slice(&au.data);
|
||||
}
|
||||
assert!(!out.is_empty(), "no AUs produced");
|
||||
let caps444 = enc.caps().chroma_444;
|
||||
std::fs::write(path, &out).expect("write bitstream");
|
||||
println!(
|
||||
"wrote {path}: {} bytes, requested {chroma:?}, caps.chroma_444={caps444}",
|
||||
out.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// ON-GLASS (RTX box): the measurement gating the AYUV 4:4:4 work — encodes the probe
|
||||
/// pattern through the REAL ARGB-input NVENC session once with `chromaFormatIDC=3`/FREXT
|
||||
/// and once as plain 4:2:0, so offline analysis of the two bitstreams answers (1) whether
|
||||
/// the FREXT stream is truly full-chroma and (2) which matrix NVENC's internal RGB→YUV CSC
|
||||
/// used (BT.601 vs BT.709 — saturated bars differ by tens of code points). Run with:
|
||||
/// cargo test -p punktfunk-host --features nvenc -- --ignored nvenc_444_on_glass --nocapture
|
||||
#[test]
|
||||
#[ignore = "requires an NVIDIA GPU + driver — run manually on the RTX box"]
|
||||
fn nvenc_444_on_glass_probe() {
|
||||
encode_pattern(ChromaFormat::Yuv444, "C:\\Users\\Public\\nvenc444_probe.h265");
|
||||
encode_pattern(ChromaFormat::Yuv420, "C:\\Users\\Public\\nvenc420_probe.h265");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,8 +53,8 @@ pub const SCM_AV1_MAIN10: u32 = 0x0002_0000;
|
||||
/// host can actually deliver it ([`host_hdr_capable`]); it is never a static claim, because a non-HDR
|
||||
/// host (Linux, or a Windows host without the `PUNKTFUNK_10BIT` opt-in) must not invite a client into
|
||||
/// an HDR mode it can't produce. (The previous placeholder 3843 = 0xF03 wrongly claimed HEVC Main10 +
|
||||
/// 4:4:4 and *no* AV1.) 4:4:4 stays off entirely: stock Moonlight is 4:2:0 and the Windows IDD-push
|
||||
/// capturer can't yet deliver full-chroma frames (`crate::capture::capturer_supports_444`).
|
||||
/// 4:4:4 and *no* AV1.) 4:4:4 stays off entirely on GameStream: stock Moonlight is 4:2:0 —
|
||||
/// full-chroma is a punktfunk/1-native negotiation only (`crate::capture::capturer_supports_444`).
|
||||
pub const SERVER_CODEC_MODE_SUPPORT: u32 = SCM_H264 | SCM_HEVC | SCM_AV1_MAIN8;
|
||||
|
||||
/// Whether this host can deliver an **HDR** (HEVC Main10 / BT.2020 PQ) GameStream — the single gate
|
||||
|
||||
@@ -380,6 +380,38 @@ fn stream_config(map: &HashMap<String, String>) -> Option<StreamConfig> {
|
||||
"client requested HDR (dynamicRangeMode != 0) but host is not HDR-capable — streaming 8-bit SDR"
|
||||
);
|
||||
}
|
||||
// The client's requested CSC (moonlight-common-c SdpGenerator.c: `encoderCscMode =
|
||||
// (colorspace << 1) | fullRange` — colorspace 0=Rec601, 1=Rec709, 2=Rec2020). Moonlight
|
||||
// renderers configure their YUV→RGB from this REQUESTED value (not the bitstream VUI), so a
|
||||
// host that encodes something else shifts the client's colours. INSTRUMENTATION ONLY for
|
||||
// now: we always encode BT.709 limited for SDR (the IDD VideoConverter / VUI-driven NVENC)
|
||||
// and BT.2020 PQ for HDR — log what clients actually ask for so honoring `encoderCscMode`
|
||||
// can be scoped from field data rather than guessed. (Absent on very old clients.)
|
||||
if let Some(csc) = parse_u("x-nv-video[0].encoderCscMode") {
|
||||
let (space, range) = (
|
||||
match csc >> 1 {
|
||||
0 => "Rec601",
|
||||
1 => "Rec709",
|
||||
2 => "Rec2020",
|
||||
_ => "unknown",
|
||||
},
|
||||
if csc & 1 != 0 { "full" } else { "limited" },
|
||||
);
|
||||
let ours = if hdr { "Rec2020 limited (PQ)" } else { "Rec709 limited" };
|
||||
let matches_ours = (hdr && csc >> 1 == 2 || !hdr && csc >> 1 == 1) && csc & 1 == 0;
|
||||
if matches_ours {
|
||||
tracing::info!(csc, space, range, "GameStream client requested CSC — matches ours");
|
||||
} else {
|
||||
tracing::warn!(
|
||||
csc,
|
||||
requested = format!("{space} {range}"),
|
||||
encoding = ours,
|
||||
"GameStream client requested a CSC we don't encode — Moonlight renders by its \
|
||||
REQUEST, so its colours will be shifted (honoring encoderCscMode is a known \
|
||||
follow-up; report this log line)"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Parity floor the client asks for (protects small frames); clamp to a sane max.
|
||||
let min_fec = parse_u("x-nv-vqos[0].fec.minRequiredFecPackets")
|
||||
.unwrap_or(2)
|
||||
|
||||
@@ -979,6 +979,19 @@ async fn serve_session(
|
||||
"encode chroma"
|
||||
);
|
||||
|
||||
// Linux 4:4:4 rides the CPU swscale → 8-bit `YUV444P` path (see `encode/linux`) — there
|
||||
// is no 10-bit 4:4:4 input there, so a 10-bit-negotiated session would silently encode
|
||||
// 8-bit. Resolve the depth DOWN before the Welcome so the wire never overstates what the
|
||||
// stream carries. (Windows NVENC composes Main 4:4:4 10 from an RGB input, so it keeps
|
||||
// the resolved depth — this clamp is Linux-only.)
|
||||
#[cfg(target_os = "linux")]
|
||||
let bit_depth: u8 = if chroma.is_444() && bit_depth == 10 {
|
||||
tracing::info!("4:4:4 on the Linux path encodes 8-bit YUV444P — resolving bit depth 8");
|
||||
8
|
||||
} else {
|
||||
bit_depth
|
||||
};
|
||||
|
||||
// Reserve the data-plane UDP socket up front and HOLD it through streaming (no
|
||||
// bind→read→drop→rebind window a concurrent session could race for a fixed port). A fixed
|
||||
// `--data-port` yields `direct = true` (stream straight to the client's reported address,
|
||||
|
||||
@@ -132,6 +132,16 @@ impl SessionPlan {
|
||||
let gpu = {
|
||||
let force_cpu_for_nvenc_444 =
|
||||
self.chroma.is_444() && !crate::encode::linux_zero_copy_is_vaapi();
|
||||
if gpu && force_cpu_for_nvenc_444 {
|
||||
// Surface the trade loudly: this is the single biggest per-frame cost a 4:4:4
|
||||
// session adds (full-res CPU readback + swscale RGB→YUV444P every frame), and
|
||||
// it looks like an unexplained fps ceiling if you don't know it happened.
|
||||
tracing::warn!(
|
||||
"4:4:4 session on the NVENC path: zero-copy GPU capture DISABLED — every \
|
||||
frame is CPU RGB + swscale RGB→YUV444P; expect a lower fps ceiling than \
|
||||
4:2:0 at this mode"
|
||||
);
|
||||
}
|
||||
gpu && !force_cpu_for_nvenc_444
|
||||
};
|
||||
crate::capture::OutputFormat {
|
||||
|
||||
Reference in New Issue
Block a user