From ad63994cb9166e215aa5e50565d22bd978fbfbaf Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Mon, 10 Aug 2026 12:30:24 +0200 Subject: [PATCH] fix(pf-encode): the 10-bit probe was the last ffmpeg NVENC open on a direct-SDK host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `can_encode_10bit`'s Linux NVIDIA arm answered "can this GPU encode 10-bit?" by opening an ffmpeg `hevc_nvenc` encoder. On a host that then streams over the direct SDK, that is the LOG-3 field bug: one ffmpeg NVENC open in a direct-SDK process wedges every later open process-wide with `NV_ENC_ERR_INVALID_VERSION` until the host restarts. `can_encode_444` was moved off the ffmpeg probe for exactly this reason on 2026-07-27. The 10-bit one was deliberately left behind, on the reading that "Linux HDR genuinely rides the libav P010 path". `open_video` contradicts that: if cuda && nvenc_direct_enabled() { // no 10-bit exclusion … NvencCudaEncoder::open(…, bit_depth, …) A CUDA capture goes to the direct backend at whatever depth was resolved, and `is_ten_bit_input` already accepts the packed 10-bit RGB (`X2Bgr10`) that a gamescope HDR capture negotiates. So on a default NVIDIA host the probe was loading ffmpeg's NVENC client for a session that never uses it. Observed on home-nobara-1 2026-08-10, gamescope + RTX 5070 Ti, client HDR on: resolved session plan … bit_depth: 10, hdr: true pipewire format negotiated … xBGR_210LE mapped=Some(X2Bgr10) modifier=0 hdr=true encoder submit failed — encoder rebuilt in place … NV_ENC_ERR_INVALID_VERSION encoder did not recover after repeated in-place rebuilds — ending the video session and with `PUNKTFUNK_NVENC_DIRECT=0` (nothing mixes, libav serves everything) the same HDR session streams clean: 0 errors, bit_depth=10, hdr: true. The 10-bit cap now rides `nvenc_cuda::probe_support()`'s existing throwaway session — the same place the 4:4:4 cap already rides, queried per listed GUID with `NV_ENC_CAPS_SUPPORT_10BIT_ENCODE`, which is what the Windows NVENC arm has always done (`enc/windows/nvenc.rs`). Unanswered fails CLOSED: an 8-bit session beats a wedged one. A host that will really serve over libav (`PUNKTFUNK_NVENC_DIRECT=0`, or a build without `--features nvenc`) keeps the ffmpeg probe, where it validates the actual path and ffmpeg's client is loaded anyway. ⚠ NOT YET VALIDATED ON GLASS. Gates are green — clippy `-D warnings` with `--features nvenc,vulkan-encode,pyrowave` on linux/amd64, 67 pf-encode tests, fmt — but the end-to-end HDR run is still owed. This branch is 42 commits behind main and its build cannot complete a punktfunk/1 handshake on home-nobara-1 at all (it stalls between "audio channels resolved" and "encode bit depth" and times out at 10 s, on EVERY attempt). That stall is NOT this change: a control build with only the routing reverted stalls identically, and the released 0.27.0 RPM on the same box handshakes fine and reaches `bit_depth=10`. Rebase onto main before re-testing. --- crates/pf-encode/src/enc/linux/nvenc_cuda.rs | 43 ++++++++++++++++++++ crates/pf-encode/src/lib.rs | 36 +++++++++++++++- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/crates/pf-encode/src/enc/linux/nvenc_cuda.rs b/crates/pf-encode/src/enc/linux/nvenc_cuda.rs index 8aec5be5..da6c5cf9 100644 --- a/crates/pf-encode/src/enc/linux/nvenc_cuda.rs +++ b/crates/pf-encode/src/enc/linux/nvenc_cuda.rs @@ -194,6 +194,16 @@ pub(crate) struct ProbedSupport { /// full-chroma 4:4:4 HEVC. `false` when unanswered (fail CLOSED, unlike `codecs`: the honest /// downgrade is a 4:2:0 session, not a dead one). pub hevc_444: bool, + /// `NV_ENC_CAPS_SUPPORT_10BIT_ENCODE` per listed codec — HEVC Main10 / AV1 10-bit. Rides this + /// same session for exactly the reason 4:4:4 does: the alternative, `linux::probe_can_encode_10bit`, + /// answers by opening an ffmpeg `hevc_nvenc`, and one ffmpeg NVENC open in a direct-SDK process + /// is the LOG-3 field bug that wedges every later open process-wide with + /// `NV_ENC_ERR_INVALID_VERSION`. That probe was left on ffmpeg when the 4:4:4 one was moved off + /// it, on the reading that "Linux HDR rides the libav P010 path" — no longer true for a CUDA + /// capture, which `open_video` sends to the direct backend (`bit_depth` passes straight through, + /// and `is_ten_bit_input` accepts packed 10-bit RGB). `false` when unanswered — fail CLOSED, an + /// 8-bit session beats a wedged one. + pub ten_bit: crate::CodecSupport, } /// The cached [`probe_support_uncached`] answer — one throwaway session per process lifetime. @@ -234,6 +244,11 @@ fn probe_support_uncached() -> ProbedSupport { av1: false, }, hevc_444: false, + ten_bit: crate::CodecSupport { + h264: false, + h265: false, + av1: false, + }, }; let Ok(api) = try_api() else { return unknown; @@ -299,6 +314,33 @@ fn probe_support_uncached() -> ProbedSupport { .is_ok() && val != 0; } + // The 10-bit cap, per codec, on the SAME still-open session — same reason 4:4:4 rides it. + // Only queried against a listed GUID (a cap query for an absent codec is undefined). + let mut ten_bit = crate::CodecSupport { + h264: false, + h265: false, + av1: false, + }; + if listed { + for (guid, slot) in [ + (nv::NV_ENC_CODEC_HEVC_GUID, &mut ten_bit.h265), + (nv::NV_ENC_CODEC_AV1_GUID, &mut ten_bit.av1), + ] { + if !guids.contains(&guid) { + continue; + } + let mut param = nv::NV_ENC_CAPS_PARAM { + version: nv::NV_ENC_CAPS_PARAM_VER, + capsToQuery: nv::NV_ENC_CAPS::NV_ENC_CAPS_SUPPORT_10BIT_ENCODE, + reserved: [0; 62], + }; + let mut val: core::ffi::c_int = 0; + *slot = (api.get_encode_caps)(enc, guid, &mut param, &mut val) + .nv_ok() + .is_ok() + && val != 0; + } + } let _ = (api.destroy_encoder)(enc); if !listed { tracing::warn!( @@ -313,6 +355,7 @@ fn probe_support_uncached() -> ProbedSupport { av1: guids.contains(&nv::NV_ENC_CODEC_AV1_GUID), }, hevc_444, + ten_bit, } } } diff --git a/crates/pf-encode/src/lib.rs b/crates/pf-encode/src/lib.rs index adabe2e0..8c093b5c 100644 --- a/crates/pf-encode/src/lib.rs +++ b/crates/pf-encode/src/lib.rs @@ -1600,7 +1600,41 @@ pub fn can_encode_10bit(codec: Codec) -> bool { }; vulkan10 || vaapi::probe_can_encode_10bit(codec) } else { - linux::probe_can_encode_10bit(codec) + // NVIDIA. Same rule the 4:4:4 arm above already follows, and for the same field + // bug: on a direct-SDK host the answer comes from the driver's own + // `NV_ENC_CAPS_SUPPORT_10BIT_ENCODE` over the direct SDK, NOT from opening an + // ffmpeg `hevc_nvenc`. One ffmpeg NVENC open in a direct-SDK process wedges every + // later open process-wide with `NV_ENC_ERR_INVALID_VERSION` until the host + // restarts (LOG-3). This probe was the LAST ffmpeg-NVENC use left on a default + // host — it was kept on the reading that "Linux HDR rides the libav P010 path", + // which `open_video` contradicts: a CUDA payload goes to the direct backend at + // whatever `bit_depth` was resolved, and `is_ten_bit_input` accepts the packed + // 10-bit RGB (`X2Bgr10`) a gamescope HDR capture negotiates. Reproduced on + // home-nobara-1 2026-08-10: HDR on → probe opens ffmpeg → the live direct-SDK + // session's caps probe returns INVALID_VERSION → repeated in-place rebuilds → + // "encoder did not recover" and the session dies. With the ffmpeg probe out of the + // process the same HDR session streams clean. + // + // Only a host that will REALLY serve the session over libav keeps the ffmpeg probe + // (PUNKTFUNK_NVENC_DIRECT=0, or a build without `--features nvenc`): there it + // validates the actual path, and ffmpeg's NVENC client runs in that process anyway. + #[cfg(feature = "nvenc")] + { + if nvenc_direct_enabled() { + let t = nvenc_cuda::probe_support().ten_bit; + match codec { + Codec::H265 => t.h265, + Codec::Av1 => t.av1, + _ => false, + } + } else { + linux::probe_can_encode_10bit(codec) + } + } + #[cfg(not(feature = "nvenc"))] + { + linux::probe_can_encode_10bit(codec) + } } } #[cfg(target_os = "windows")]