From 97928516a0b21e0b60e46fe993d2d35dd33b2cbe Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Sun, 9 Aug 2026 16:58:15 +0200 Subject: [PATCH] fix(pf-capture): every NVIDIA HDR stream had red and blue swapped gamescope's capture textures are mappable, hence linear-tiled, and NVIDIA does not implement linear-tiled STORAGE for A2R10G10B10_UNORM_PACK32. Upstream says it plainly in rendervulkan.cpp: "imageStore lands in XBGR order there, swapping R/B". So the composite writes XBGR bytes into a buffer still LABELLED XRGB2101010, and our patch's spa_format_to_drm() derives that label from the negotiated SPA format alone, never asking the hardware what it can actually write. The host then believed the label, correctly at every step: xRGB_210LE -> PixelFormat::X2Rgb10 -> NV_ENC_BUFFER_FORMAT_ARGB10. DRM XRGB2101010 really is "B in the low 10 bits" and NVENC ARGB10 really is "B in the lowest 10 bits"; the Windows twin (R10G10B10A2 -> ABGR10) is correct by the same rule. Every mapping audits clean because the label was right and only the CONTENT was wrong -- which is why this survived a full trace of both ends. Fix the preference host-side: offer xBGR_210LE FIRST. The first compatible consumer pod wins, so that is what a gamescope session lands on, and an XBGR2101010 texture is one NVIDIA writes in its own order -- label and content agree. It costs nothing elsewhere: A2B10G10R10_UNORM_PACK32 is the universally supported packed-10 format, it is what upstream's own fallback picks, and X2Bgr10 has a first-class encoder path (NVENC ABGR10, VAAPI X2BGR10LE). xRGB_210LE stays as the second pod so a producer offering only it can still negotiate HDR instead of dropping to the SDR downgrade. Doing it here rather than in the patch set is deliberate: the real fix is for spa_format_to_drm() to offer only what vulkan_get_rgb10_capture_format() reports, but that function landed after 3.16.25 and the pin is 3.16.25-7-g60561e2+pfhdr4 (0 "2101010" strings in the shipped binary), so the deployed gamescope cannot self-correct. This ships in the host binary with no gamescope rebuild. Field-confirmed on the RTX 5070 Ti Bazzite host with 0.26.0, and confirmed host-side rather than client-side by reproducing the identical swap from two unrelated clients (16" MacBook Pro and Mac Studio). SDR was never affected -- it takes no packed-10 path. Gate (pf-lxcheck2, linux/amd64): fmt clean, clippy --all-targets -D warnings clean, cargo test -p pf-capture 60 passed / 0 failed incl. the new hdr_offers_xbgr_before_xrgb order pin. --- crates/pf-capture/src/linux/pipewire.rs | 13 +++-- crates/pf-capture/src/linux/pw_pods.rs | 65 +++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/crates/pf-capture/src/linux/pipewire.rs b/crates/pf-capture/src/linux/pipewire.rs index b202d5ae..2a631136 100644 --- a/crates/pf-capture/src/linux/pipewire.rs +++ b/crates/pf-capture/src/linux/pipewire.rs @@ -4,6 +4,7 @@ use super::pw_cursor::{composite_cursor, update_cursor_meta, CursorState}; use super::pw_pods::{ build_cursor_meta_param, build_default_format_obj, build_dmabuf_buffers, build_dmabuf_format, build_hdr_dmabuf_format, build_mappable_buffers, build_shm_only_buffers, serialize_pod, + HDR_FORMAT_ORDER, }; use super::{CapturedFrame, DmabufFrame, FramePayload, PixelFormat, ZeroCopyPolicy}; use anyhow::{Context, Result}; @@ -1850,13 +1851,15 @@ pub fn pipewire_thread( // negotiation-timeout path latches the process-wide SDR downgrade if nothing matches. let format_pods: Vec> = if want_hdr { tracing::info!( - "HDR capture: offering xRGB_210LE/xBGR_210LE LINEAR dmabufs with MANDATORY \ + "HDR capture: offering xBGR_210LE/xRGB_210LE LINEAR dmabufs with MANDATORY \ BT.2020 + SMPTE-2084 (PQ) colorimetry (GNOME 50+ monitor stream)" ); - vec![ - build_hdr_dmabuf_format(VideoFormat::xRGB_210LE, preferred)?, - build_hdr_dmabuf_format(VideoFormat::xBGR_210LE, preferred)?, - ] + // ⚠ Order is the whole fix — see the NVIDIA note on `HDR_FORMAT_ORDER`. The first + // compatible consumer pod wins, so this is what a gamescope session actually lands on. + HDR_FORMAT_ORDER + .iter() + .map(|fmt| build_hdr_dmabuf_format(*fmt, preferred)) + .collect::>>()? } else if want_dmabuf { let mut pods = Vec::with_capacity(if prefer_native_nv12 { 2 } else { 1 }); if prefer_native_nv12 { diff --git a/crates/pf-capture/src/linux/pw_pods.rs b/crates/pf-capture/src/linux/pw_pods.rs index 777913a2..f6ef62c8 100644 --- a/crates/pf-capture/src/linux/pw_pods.rs +++ b/crates/pf-capture/src/linux/pw_pods.rs @@ -121,6 +121,38 @@ pub(super) fn build_dmabuf_format( /// SDR — the same outcome as not offering HDR. const SPA_VIDEO_TRANSFER_SMPTE2084: u32 = 14; +/// The two 10-bit PQ formats an HDR session offers, **in negotiation order**. The order is not a +/// style choice — on NVIDIA it is the difference between correct colour and red/blue swapped. +/// +/// `xBGR_210LE` (DRM `XBGR2101010`, Vulkan `A2B10G10R10_UNORM_PACK32`) comes FIRST because the +/// first compatible consumer pod wins, and it is the only one gamescope fills correctly on every +/// vendor: +/// +/// * `A2R10G10B10_UNORM_PACK32` **linear-tiled storage** is an optional Vulkan feature that +/// NVIDIA does not implement. gamescope's capture textures are mappable, hence linear, so on +/// NVIDIA its composite `imageStore` into that image lands in XBGR order — the bytes come out +/// byte-reversed while the buffer is still LABELLED `XRGB2101010`. +/// * The host believes the label: `xRGB_210LE → PixelFormat::X2Rgb10 →` +/// `NV_ENC_BUFFER_FORMAT_ARGB10`. Every mapping in that chain is individually correct, which is +/// exactly why the bug is invisible from this side — the *content* is what's wrong. +/// * Upstream gamescope hit the same wall and fixed it with `vulkan_get_rgb10_capture_format()`, +/// which probes `linearTilingFeatures` for STORAGE+SAMPLED and falls back to `XBGR2101010`. +/// That landed AFTER 3.16.25, so the pinned `punktfunk-gamescope` (3.16.25-7-g60561e2 +pfhdr4) +/// predates it and cannot self-correct — hence fixing the preference host-side, where it ships +/// in the host binary with no gamescope rebuild. +/// +/// Preferring xBGR costs nothing anywhere else: `A2B10G10R10_UNORM_PACK32` is the universally +/// supported packed-10 format (it is the standard HDR10 swapchain format), it is what upstream +/// falls back to, and `X2Bgr10` has a first-class encoder path (NVENC `ABGR10`, VAAPI +/// `X2BGR10LE`). `xRGB_210LE` stays as the second pod so a producer that somehow offers only it +/// can still negotiate HDR rather than falling off to the SDR downgrade. +/// +/// ⚠ The real fix belongs upstream in the patch set: `spa_format_to_drm()` should offer only the +/// format `vulkan_get_rgb10_capture_format()` reports. Until the gamescope pin moves past that +/// commit, THIS ORDER is what keeps NVIDIA HDR sessions correct — do not "tidy" it. +pub(super) const HDR_FORMAT_ORDER: [VideoFormat; 2] = + [VideoFormat::xBGR_210LE, VideoFormat::xRGB_210LE]; + pub(super) fn build_hdr_dmabuf_format( format: VideoFormat, preferred: Option<(u32, u32, u32)>, @@ -596,4 +628,37 @@ mod tests { // The minimum must not exceed what producers already serve, or the ask becomes a demand. const { assert!(POOL_MIN <= 2) }; } + + /// xBGR_210LE must be offered FIRST, and this is a correctness test, not a style one. + /// + /// The first compatible consumer pod wins the negotiation. Leading with `xRGB_210LE` makes an + /// NVIDIA gamescope session land on `XRGB2101010`, whose linear-tiled `A2R10G10B10` storage + /// NVIDIA does not support — gamescope's composite `imageStore` writes XBGR bytes under an + /// XRGB label and the whole stream comes out with red and blue swapped. Every format mapping + /// on the host side is individually correct, so nothing downstream can detect it. + /// + /// Field-confirmed 2026-08-09 on the RTX 5070 Ti Bazzite host with 0.26.0. See the + /// [`HDR_FORMAT_ORDER`] docs for the upstream fix this predates. + #[test] + fn hdr_offers_xbgr_before_xrgb() { + assert_eq!( + HDR_FORMAT_ORDER[0], + VideoFormat::xBGR_210LE, + "xBGR_210LE must be offered first — leading with xRGB_210LE swaps red and blue on \ + every NVIDIA gamescope HDR session" + ); + assert_eq!( + HDR_FORMAT_ORDER[1], + VideoFormat::xRGB_210LE, + "xRGB_210LE stays as the fallback pod so a producer offering only it can still \ + negotiate HDR instead of dropping to the SDR downgrade" + ); + // Both must still build: the order is a preference, never a removal. + for fmt in HDR_FORMAT_ORDER { + assert!( + !build_hdr_dmabuf_format(fmt, None).unwrap().is_empty(), + "{fmt:?} must still produce a format pod" + ); + } + } }