From 188f55d3b1dcca15c941c7155029b5102875b839 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Mon, 27 Jul 2026 20:30:26 +0200 Subject: [PATCH] fix(encode/nvenc): the host advertises what the driver lists, not a superset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every NVIDIA host advertised a static H.264|HEVC|AV1 superset, so a 1st-gen Maxwell (GTX 960M, no HEVC/AV1 encode) offered HEVC — a client that believed it got ~15 s of blank video and a disconnect instead of a stream. Both OSes now ask the driver itself (nvEncGetEncodeGUIDs) on one throwaway direct-SDK session: Linux on the shared CUDA context, Windows on the selected render adapter, wired into host_wire_caps AND the GameStream serverinfo mask (which had been left on the superset for NVIDIA on both OSes). Fails open — an unanswerable probe keeps the historical superset, so it can only ever narrow the advertisement to codecs the GPU really encodes. The HEVC 4:4:4 answer rides the same session on Linux instead of opening a libav hevc_nvenc FREXT probe: that open is the prime suspect for the field bug where one probe wedges NVENC process-wide (NV_ENC_ERR_INVALID_VERSION on every later session until a host restart), and the direct backend re-checks the same caps bit at session open anyway. The ffmpeg probe remains only for hosts that really stream over libav (PUNKTFUNK_NVENC_DIRECT=0 or a build without the nvenc feature), where ffmpeg's NVENC client runs regardless. The 10-bit probe deliberately stays libav — Linux HDR rides the libav P010 path. On-hardware: .136 (RTX 5070 Ti) 14/14 nvenc tests in one process incl. the probe followed by real sessions and dirty teardown; .173 (Windows RTX) probe + 47 release lib tests. The Windows probe test documents the pre-existing MSVC debug-link failure (LNK2019 via the sdk crate's unused lazy loader) — run it with --release, the same reason windows-host.yml gates with clippy. Co-Authored-By: Claude Fable 5 (cherry picked from commit 0346ec8090568eb499e8cb7d735305b28471185e) --- crates/pf-encode/src/enc/linux/mod.rs | 6 + crates/pf-encode/src/enc/linux/nvenc_cuda.rs | 183 ++++++++++++++++++ crates/pf-encode/src/enc/windows/nvenc.rs | 155 ++++++++++++--- crates/pf-encode/src/lib.rs | 115 +++++++++-- .../src/gamestream/serverinfo.rs | 17 +- 5 files changed, 428 insertions(+), 48 deletions(-) diff --git a/crates/pf-encode/src/enc/linux/mod.rs b/crates/pf-encode/src/enc/linux/mod.rs index fd04bd30..0814f86f 100644 --- a/crates/pf-encode/src/enc/linux/mod.rs +++ b/crates/pf-encode/src/enc/linux/mod.rs @@ -984,6 +984,12 @@ impl Drop for QuietLibavLog { /// takes for a live 4:4:4 stream — and reports whether it succeeded. HEVC-only; the result is cached /// by the caller ([`crate::can_encode_444`]). A GPU/driver/ffmpeg without RExt 4:4:4 fails /// the open here, so the host resolves the session to 4:2:0 before the Welcome (honest downgrade). +/// +/// ⚠️ Only consulted when libav will really serve the session (`PUNKTFUNK_NVENC_DIRECT=0`, or a +/// build without `--features nvenc`). A direct-SDK host answers from the driver's caps bit instead +/// (`nvenc_cuda::probe_support`) — running THIS probe there mixes ffmpeg's NVENC client into a +/// direct-SDK process, which is the LOG-3 field bug: one successful `hevc_nvenc` FREXT open+close +/// wedged every later NVENC open process-wide (`NV_ENC_ERR_INVALID_VERSION`) until a host restart. pub fn probe_can_encode_444(codec: Codec) -> bool { if codec != Codec::H265 { return false; diff --git a/crates/pf-encode/src/enc/linux/nvenc_cuda.rs b/crates/pf-encode/src/enc/linux/nvenc_cuda.rs index 22663768..77569b44 100644 --- a/crates/pf-encode/src/enc/linux/nvenc_cuda.rs +++ b/crates/pf-encode/src/enc/linux/nvenc_cuda.rs @@ -107,6 +107,12 @@ struct EncodeApi { *mut nv::NV_ENC_CAPS_PARAM, *mut core::ffi::c_int, ) -> nv::NVENCSTATUS, + // The two entry points behind [`probe_support`] — the driver's own list of encode GUIDs + // this chip exposes. Mandatory like every other entry: both have existed since NVENC 1.0, so a + // driver missing them is broken in ways the rest of this table would not survive either. + get_encode_guid_count: unsafe extern "C" fn(*mut c_void, *mut u32) -> nv::NVENCSTATUS, + get_encode_guids: + unsafe extern "C" fn(*mut c_void, *mut nv::GUID, u32, *mut u32) -> nv::NVENCSTATUS, get_encode_preset_config_ex: unsafe extern "C" fn( *mut c_void, nv::GUID, @@ -168,6 +174,141 @@ fn api() -> &'static EncodeApi { try_api().expect("NVENC call before a successful try_api() gate") } +/// Everything the host advertisement asks of this GPU's NVENC, answered by the driver itself on +/// ONE throwaway session: the encode-GUID list (which codecs exist at all) and the HEVC 4:4:4 cap. +#[derive(Clone, Copy)] +pub(crate) struct ProbedSupport { + /// Which codecs this chip's NVENC encodes (`nvEncGetEncodeGUIDs`). All-`false` = the probe + /// could not answer — [`crate::CodecSupport::wire_mask`] turns that into `None` so the caller + /// keeps the static superset (fail open). + pub codecs: crate::CodecSupport, + /// `NV_ENC_CAPS_SUPPORT_YUV444_ENCODE` for the HEVC GUID — whether this chip can encode + /// full-chroma 4:4:4 HEVC. `false` when unanswered (fail CLOSED, unlike `codecs`: the honest + /// downgrade is a 4:2:0 session, not a dead one). + pub hevc_444: bool, +} + +/// The cached [`probe_support_uncached`] answer — one throwaway session per process lifetime. +pub(crate) fn probe_support() -> ProbedSupport { + static CACHE: std::sync::OnceLock = std::sync::OnceLock::new(); + *CACHE.get_or_init(probe_support_uncached) +} + +/// Which codecs **this GPU's** NVENC can actually encode — and whether HEVC can go 4:4:4 — asked +/// of the driver itself (`nvEncGetEncodeGUIDs` + `nvEncGetEncodeCaps`) instead of assumed from the +/// SDK version. +/// +/// Why this exists: the host used to advertise a static `H.264 | HEVC | AV1` superset for every +/// NVIDIA box, so a chip without HEVC NVENC (1st-gen Maxwell, e.g. GTX 960M — HEVC needs 2nd-gen +/// Maxwell+, AV1 needs Ada+) still offered HEVC. A client reasonably negotiated H265 and got a dead +/// session: `hevc_nvenc` "No capable devices found", eight pipeline retries, ~15 s of blank video, +/// then a disconnect. The GUID list is a property of the chip+driver, so it is equally right for +/// the direct-SDK backend and the libav `*_nvenc` one. +/// +/// ⚠️ Deliberately NOT the VAAPI probe's shape (open a tiny libav encoder per codec). That would run +/// ffmpeg's NVENC client, and mixing it with this direct-SDK client in one process is the prime +/// suspect for the open bug where one `probe_can_encode_444` open wedges NVENC **process-wide** +/// (`NV_ENC_ERR_INVALID_VERSION` on every later session until a host restart — LOG-3, Droff, +/// 0.19.2). This asks the SAME client, on the SAME shared CUDA context, that real sessions use — +/// one extra session open of a kind the encoder already performs per open (`query_caps`), cached +/// once per process by [`probe_support`]. The 4:4:4 cap rides the same session for the same +/// reason: it used to be its own libav `hevc_nvenc` FREXT open — the exact open LOG-3 caught +/// wedging NVENC — and the direct backend re-checks the same cap at session open anyway +/// (`query_caps` → `yuv444_supported`), so the caps bit is the answer the live session will obey. +/// +/// Every failure path returns "nothing probed" (see the [`ProbedSupport`] field docs for the +/// per-field fail direction). +fn probe_support_uncached() -> ProbedSupport { + let unknown = ProbedSupport { + codecs: crate::CodecSupport { + h264: false, + h265: false, + av1: false, + }, + hevc_444: false, + }; + let Ok(api) = try_api() else { + return unknown; + }; + let cu_ctx = match cuda::context() { + Ok(c) => c, + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), "NVENC codec probe: no CUDA context"); + return unknown; + } + }; + // SAFETY: `try_api()` returned Ok, so every fn pointer below is a live entry point from the + // driver's own function list. `params`/`enc`/`count`/`written` are live locals that outlive + // their synchronous calls; `device` is the process-shared CUDA context (`cuda::context()` + // returned Ok), the same handle `query_caps` passes. `guids` is sized to the count the driver + // just reported and its pointer is valid for that many `GUID`s, matching the + // `guidArraySize` argument. The session is destroyed on every path out — including the failed + // open, which the NVENC docs still require (the driver may have taken the slot before + // erroring; skipping it leaks toward the concurrent-session cap). + unsafe { + let mut params = nv::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS { + version: nv::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER, + deviceType: nv::NV_ENC_DEVICE_TYPE::NV_ENC_DEVICE_TYPE_CUDA, + device: cu_ctx, + apiVersion: nv::NVENCAPI_VERSION, + ..Default::default() + }; + let mut enc: *mut c_void = ptr::null_mut(); + if let Err(e) = (api.open_encode_session_ex)(&mut params, &mut enc).nv_ok() { + if !enc.is_null() { + let _ = (api.destroy_encoder)(enc); + } + tracing::warn!( + error = %format!("{:#}", nvenc_status::call_err("open_encode_session_ex (codec probe)", e)), + "NVENC codec probe failed — keeping the static codec advertisement" + ); + return unknown; + } + // The handshake with the kernel module succeeded (same latch `query_caps` sets). + nvenc_status::note_session_opened(); + let mut count = 0u32; + let counted = (api.get_encode_guid_count)(enc, &mut count).nv_ok().is_ok(); + let mut guids = vec![nv::GUID::default(); count as usize]; + let mut written = 0u32; + let listed = counted + && count > 0 + && (api.get_encode_guids)(enc, guids.as_mut_ptr(), count, &mut written) + .nv_ok() + .is_ok(); + guids.truncate(written as usize); + // The 4:4:4 cap needs the session that is still open — query it before the destroy. Only + // meaningful against a listed HEVC GUID (a cap query for an absent codec is undefined). + let mut hevc_444 = false; + if listed && guids.contains(&nv::NV_ENC_CODEC_HEVC_GUID) { + let mut param = nv::NV_ENC_CAPS_PARAM { + version: nv::NV_ENC_CAPS_PARAM_VER, + capsToQuery: nv::NV_ENC_CAPS::NV_ENC_CAPS_SUPPORT_YUV444_ENCODE, + reserved: [0; 62], + }; + let mut val: core::ffi::c_int = 0; + hevc_444 = (api.get_encode_caps)(enc, nv::NV_ENC_CODEC_HEVC_GUID, &mut param, &mut val) + .nv_ok() + .is_ok() + && val != 0; + } + let _ = (api.destroy_encoder)(enc); + if !listed { + tracing::warn!( + "NVENC codec probe: driver listed no encode GUIDs — keeping the static advertisement" + ); + return unknown; + } + ProbedSupport { + codecs: crate::CodecSupport { + h264: guids.contains(&nv::NV_ENC_CODEC_H264_GUID), + h265: guids.contains(&nv::NV_ENC_CODEC_HEVC_GUID), + av1: guids.contains(&nv::NV_ENC_CODEC_AV1_GUID), + }, + hevc_444, + } + } +} + fn load_api() -> std::result::Result { // SAFETY: `Library::new` runs `libnvidia-encode.so.1`'s initializers — the trusted NVIDIA driver // library, so loading has no unexpected effects; `map_err` handles its absence (AMD/Intel/no @@ -223,6 +364,8 @@ fn load_api() -> std::result::Result { reconfigure_encoder: list.nvEncReconfigureEncoder.ok_or(MISSING)?, destroy_encoder: list.nvEncDestroyEncoder.ok_or(MISSING)?, get_encode_caps: list.nvEncGetEncodeCaps.ok_or(MISSING)?, + get_encode_guid_count: list.nvEncGetEncodeGUIDCount.ok_or(MISSING)?, + get_encode_guids: list.nvEncGetEncodeGUIDs.ok_or(MISSING)?, get_encode_preset_config_ex: list.nvEncGetEncodePresetConfigEx.ok_or(MISSING)?, create_bitstream_buffer: list.nvEncCreateBitstreamBuffer.ok_or(MISSING)?, destroy_bitstream_buffer: list.nvEncDestroyBitstreamBuffer.ok_or(MISSING)?, @@ -2284,6 +2427,46 @@ mod tests { /// and assert the next AU carries the recovery-anchor tag (the F2 fix) and that `caps()` /// advertises RFI. Needs an NVIDIA GPU + driver. Run: /// cargo test -p punktfunk-host --features nvenc -- --ignored nvenc_cuda_smoke --nocapture + /// ON-HARDWARE: the codec/4:4:4 advertisement probe against the real driver. Asserts the two + /// invariants that matter for what the host advertises — every NVENC-capable GPU ever made can + /// encode H.264, so a probe that comes back with `h264 = false` while NVENC is otherwise + /// working means the enumeration itself is broken (and would silently narrow the host's + /// advertisement); and the answer must be stable across calls (asserted on the UNCACHED fn — + /// the cached [`probe_support`] would make it vacuous), since one cached answer drives every + /// negotiation. Prints the mask so a run on an OLD card (Maxwell GM107 = h264 only, no 4:4:4 — + /// the GPU this probe exists for) is self-documenting. Run: + /// cargo test -p pf-encode --features nvenc -- --ignored nvenc_codec_probe --nocapture + #[test] + #[ignore = "requires an NVIDIA GPU + driver — run manually on an NVIDIA box"] + fn nvenc_codec_probe_reports_real_gpu_support() { + let probed = probe_support_uncached(); + let caps = probed.codecs; + eprintln!( + "NVENC probe: h264={} h265={} av1={} hevc_444={}", + caps.h264, caps.h265, caps.av1, probed.hevc_444 + ); + assert!( + caps.h264, + "every NVENC generation encodes H.264 — a false here means the GUID enumeration \ + failed, which would narrow the host's codec advertisement" + ); + assert!( + !probed.hevc_444 || caps.h265, + "a 4:4:4-capable HEVC that is not in the GUID list is contradictory" + ); + let again = probe_support_uncached(); + assert_eq!( + (caps.h264, caps.h265, caps.av1, probed.hevc_444), + ( + again.codecs.h264, + again.codecs.h265, + again.codecs.av1, + again.hevc_444 + ), + "the probe must be stable — it is cached once and drives every later negotiation" + ); + } + #[test] #[ignore = "requires an NVIDIA GPU + driver — run manually on the RTX box (.21)"] fn nvenc_cuda_smoke_rfi_anchor() { diff --git a/crates/pf-encode/src/enc/windows/nvenc.rs b/crates/pf-encode/src/enc/windows/nvenc.rs index 9957f7ef..233a1743 100644 --- a/crates/pf-encode/src/enc/windows/nvenc.rs +++ b/crates/pf-encode/src/enc/windows/nvenc.rs @@ -89,6 +89,12 @@ struct EncodeApi { *mut nv::NV_ENC_CAPS_PARAM, *mut core::ffi::c_int, ) -> nv::NVENCSTATUS, + // The two entry points behind [`probe_codec_support`] — the driver's own list of encode GUIDs + // this chip exposes. Mandatory like every other entry: both have existed since NVENC 1.0, so a + // driver missing them is broken in ways the rest of this table would not survive either. + get_encode_guid_count: unsafe extern "C" fn(*mut c_void, *mut u32) -> nv::NVENCSTATUS, + get_encode_guids: + unsafe extern "C" fn(*mut c_void, *mut nv::GUID, u32, *mut u32) -> nv::NVENCSTATUS, get_encode_preset_config_ex: unsafe extern "C" fn( *mut c_void, nv::GUID, @@ -203,6 +209,8 @@ fn load_api() -> std::result::Result { reconfigure_encoder: list.nvEncReconfigureEncoder.ok_or(MISSING)?, destroy_encoder: list.nvEncDestroyEncoder.ok_or(MISSING)?, get_encode_caps: list.nvEncGetEncodeCaps.ok_or(MISSING)?, + get_encode_guid_count: list.nvEncGetEncodeGUIDCount.ok_or(MISSING)?, + get_encode_guids: list.nvEncGetEncodeGUIDs.ok_or(MISSING)?, get_encode_preset_config_ex: list.nvEncGetEncodePresetConfigEx.ok_or(MISSING)?, create_bitstream_buffer: list.nvEncCreateBitstreamBuffer.ok_or(MISSING)?, destroy_bitstream_buffer: list.nvEncDestroyBitstreamBuffer.ok_or(MISSING)?, @@ -1965,11 +1973,85 @@ pub fn probe_can_encode_10bit(codec: Codec) -> bool { probe_encode_cap(codec, nv::NV_ENC_CAPS::NV_ENC_CAPS_SUPPORT_10BIT_ENCODE) } -/// Query ONE NVENC capability for `codec`: creates a throwaway hardware D3D11 device + NVENC -/// session on the **selected render adapter**, reads the cap, and tears everything down. `false` -/// on any failure (no loadable NVENC, no device, failed open) — the honest answer for a +/// Query ONE NVENC capability for `codec` on a throwaway session (see [`with_probe_session`]). +/// `false` on any failure (no loadable NVENC, no device, failed open) — the honest answer for a /// capability that couldn't be confirmed. fn probe_encode_cap(codec: Codec, cap: nv::NV_ENC_CAPS) -> bool { + with_probe_session(|enc| { + let mut param = nv::NV_ENC_CAPS_PARAM { + version: nv::NV_ENC_CAPS_PARAM_VER, + capsToQuery: cap, + reserved: [0; 62], + }; + let mut val: i32 = 0; + // SAFETY: `get_encode_caps` reads one scalar cap into `val` (live locals) for the live + // session `enc` via the loaded API table (`with_probe_session` sits past `try_api`). + unsafe { + (api().get_encode_caps)(enc, codec_guid(codec), &mut param, &mut val) + .nv_ok() + .is_ok() + && val != 0 + } + }) + .unwrap_or(false) +} + +/// Which codecs **this GPU's** NVENC can actually encode, asked of the driver itself +/// (`nvEncGetEncodeGUIDs`) on a throwaway session — the Windows twin of the Linux +/// `nvenc_cuda::probe_support` codec half, probing the **selected render adapter** (the GPU the +/// session will really encode on) rather than CUDA device 0. Same field bug on both OSes: the +/// static `H.264 | HEVC | AV1` superset advertised HEVC on a 1st-gen Maxwell, and a client that +/// negotiated it got a dead session instead of a stream. +/// +/// Every failure path returns "nothing probed", which [`crate::CodecSupport::wire_mask`] turns +/// into `None` so the caller keeps the old static superset — a broken probe must never be able to +/// narrow an NVIDIA host's advertisement to nothing. Cached per selected GPU by the caller +/// ([`crate::windows_codec_support`]). +pub(crate) fn probe_codec_support() -> crate::CodecSupport { + let unknown = crate::CodecSupport { + h264: false, + h265: false, + av1: false, + }; + with_probe_session(|enc| { + // SAFETY: all NVENC calls go through the loaded API table against the live session `enc`; + // `count`/`written` are live locals, and `guids` is sized to the count the driver just + // reported, its pointer valid for that many `GUID`s (matching `guidArraySize`). + unsafe { + let mut count = 0u32; + let counted = (api().get_encode_guid_count)(enc, &mut count) + .nv_ok() + .is_ok(); + let mut guids = vec![nv::GUID::default(); count as usize]; + let mut written = 0u32; + let listed = counted + && count > 0 + && (api().get_encode_guids)(enc, guids.as_mut_ptr(), count, &mut written) + .nv_ok() + .is_ok(); + if !listed { + tracing::warn!( + "NVENC codec probe: driver listed no encode GUIDs — keeping the static \ + advertisement" + ); + return unknown; + } + guids.truncate(written as usize); + crate::CodecSupport { + h264: guids.contains(&codec_guid(Codec::H264)), + h265: guids.contains(&codec_guid(Codec::H265)), + av1: guids.contains(&codec_guid(Codec::Av1)), + } + } + }) + .unwrap_or(unknown) +} + +/// Open a throwaway NVENC session on a fresh hardware D3D11 device, hand it to `f`, and tear +/// everything down. `None` = no loadable NVENC / no device / failed open — the caller supplies +/// the honest "couldn't confirm" answer. Shared by [`probe_encode_cap`] and +/// [`probe_codec_support`], so every advertisement probe opens sessions exactly one way. +fn with_probe_session(f: impl FnOnce(*mut c_void) -> T) -> Option { // Same exclusion as `init_session`: this opens a real (throwaway) session, so it must never // overlap a zombie reap that could be destroying the very address the driver hands us. let _gate = DRIVER_SESSION_GATE @@ -1983,20 +2065,20 @@ fn probe_encode_cap(codec: Codec, cap: nv::NV_ENC_CAPS) -> bool { D3D11CreateDevice, D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_SDK_VERSION, }; use windows::Win32::Graphics::Dxgi::{CreateDXGIFactory1, IDXGIAdapter1, IDXGIFactory4}; - // No loadable NVENC on this box (non-NVIDIA / no driver) → the honest 4:4:4 answer is "no". - // This is also the `api()` gate for every NVENC call below. + // No loadable NVENC on this box (non-NVIDIA / no driver) → nothing to confirm. + // This is also the `api()` gate for every NVENC call below and inside `f`. if try_api().is_err() { - return false; + return None; } // SAFETY: a self-contained probe owning every handle it creates. `CreateDXGIFactory1`/ // `EnumAdapterByLuid` return owned COM objects or err (→ default-adapter fallback). // `D3D11CreateDevice` (explicit adapter + UNKNOWN driver type, or NULL adapter + HARDWARE) - // fills `device` or returns Err (→ false). `open_encode_session_ex` opens an NVENC session - // against that device's raw pointer (valid while `device` is held) or errors (→ false, after + // fills `device` or returns Err (→ None). `open_encode_session_ex` opens an NVENC session + // against that device's raw pointer (valid while `device` is held) or errors (→ None, after // destroying any residue session the failed open left — the docs require it). - // `get_encode_caps` reads one scalar cap into `val` via the loaded API table. - // `destroy_encoder` frees the session exactly once; `device`/its context drop with the COM - // wrappers. No handle escapes this call and nothing runs concurrently. + // `destroy_encoder` frees the session exactly once, after `f` returns (so `enc` is live for + // the whole closure call); `device`/its context drop with the COM wrappers. No handle + // escapes this call and nothing runs concurrently (the gate above). unsafe { // Probe on the SELECTED render adapter — the GPU the session will actually encode on // (web-console preference / PUNKTFUNK_RENDER_ADAPTER / max VRAM). The OS default adapter @@ -2032,9 +2114,9 @@ fn probe_encode_cap(codec: Codec, cap: nv::NV_ENC_CAPS) -> bool { ), }; if created.is_err() { - return false; + return None; } - let Some(device) = device else { return false }; + let device = device?; let mut params = nv::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS { version: nv::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER, deviceType: nv::NV_ENC_DEVICE_TYPE::NV_ENC_DEVICE_TYPE_DIRECTX, @@ -2051,23 +2133,14 @@ fn probe_encode_cap(codec: Codec, cap: nv::NV_ENC_CAPS) -> bool { if !enc.is_null() { let _ = (api().destroy_encoder)(enc); } - return false; + return None; } // Availability probe, but a real session open all the same: it proves the driver accepted // this build's version word, which is what rules a skew out later (see `nvenc_status`). nvenc_status::note_session_opened(); - let mut param = nv::NV_ENC_CAPS_PARAM { - version: nv::NV_ENC_CAPS_PARAM_VER, - capsToQuery: cap, - reserved: [0; 62], - }; - let mut val: i32 = 0; - let ok = (api().get_encode_caps)(enc, codec_guid(codec), &mut param, &mut val) - .nv_ok() - .is_ok() - && val != 0; + let out = f(enc); let _ = (api().destroy_encoder)(enc); - ok + Some(out) } } @@ -2356,4 +2429,36 @@ mod tests { "C:\\Users\\Public\\nvenc420_probe.h265", ); } + + /// ON-HARDWARE: the codec-advertisement probe against the real driver — the Windows twin of + /// the Linux `nvenc_codec_probe_reports_real_gpu_support` test, same invariants: every + /// NVENC-capable GPU ever made encodes H.264 (a `false` means the enumeration is broken and + /// would silently narrow the advertisement), and the answer must be stable across calls since + /// one cached answer drives every negotiation. Prints the mask so a run on an OLD card + /// (Maxwell GM107 = h264 only, the GPU this probe exists for) is self-documenting. Run: + /// cargo test -p pf-encode --features nvenc --release -- --ignored nvenc_codec_probe --nocapture + /// (`--release` is REQUIRED on Windows, and pre-dates this test: the debug lib-test link + /// fails LNK2019 because the sdk crate's unused lazy loader references the NvEncodeAPI + /// imports that runtime loading deliberately avoids — debug `/OPT:NOREF` keeps the dead + /// COMDAT, release strips it. Windows CI gates pf-encode with clippy for the same reason.) + #[test] + #[ignore = "requires an NVIDIA GPU + driver — run manually on the RTX box (.173)"] + fn nvenc_codec_probe_reports_real_gpu_support() { + let caps = probe_codec_support(); + eprintln!( + "NVENC (Windows) probe: h264={} h265={} av1={}", + caps.h264, caps.h265, caps.av1 + ); + assert!( + caps.h264, + "every NVENC generation encodes H.264 — a false here means the GUID enumeration \ + failed, which would narrow the host's codec advertisement" + ); + let again = probe_codec_support(); + assert_eq!( + (caps.h264, caps.h265, caps.av1), + (again.h264, again.h265, again.av1), + "the probe must be stable — it is cached once and drives every later negotiation" + ); + } } diff --git a/crates/pf-encode/src/lib.rs b/crates/pf-encode/src/lib.rs index 3a69ec43..99fe0eae 100644 --- a/crates/pf-encode/src/lib.rs +++ b/crates/pf-encode/src/lib.rs @@ -108,7 +108,21 @@ impl Codec { return m & pref_ceiling; } } - // NVENC (static superset, like GameStream) — or an empty VAAPI probe (see above). + // NVENC: ask the DRIVER which codecs this chip's encoder exposes, the same way the + // VAAPI arm above does — a static superset advertised HEVC on a 1st-gen Maxwell + // (HEVC needs 2nd-gen Maxwell+, AV1 needs Ada+), and a client that believed it got + // ~15 s of blank video and a disconnect instead of a stream. Fails OPEN: a probe + // that can't answer (no direct-SDK build, no CUDA, an old driver) yields `None` and + // leaves the historical superset standing, so this can only ever narrow the + // advertisement to something the GPU really encodes. + #[cfg(feature = "nvenc")] + if backend == LinuxBackend::Nvenc { + if let Some(m) = nvenc_codec_support().wire_mask() { + return m & pref_ceiling; + } + } + // NVENC without the probe (no `nvenc` feature / probe declined) — or an empty VAAPI + // probe (see above): the static superset, like GameStream. GPU_SUPERSET & pref_ceiling } #[cfg(target_os = "windows")] @@ -121,7 +135,8 @@ impl Codec { return m; } } - // NVENC (static superset, like GameStream) — or an empty AMF/QSV probe (see above). + // An unprobed backend (NVENC without the `nvenc` feature) — or an empty probe + // (see above): the static superset, like GameStream. GPU_SUPERSET } // The macOS dev/test host has no GPU encode backend — keep the pre-probe advertisement. @@ -1263,9 +1278,31 @@ impl CodecSupport { } } +/// Probe the active NVIDIA GPU for its encodable codecs (cached once per process). Asks the driver +/// for this chip's encode-GUID list over the direct SDK — see [`nvenc_cuda::probe_support`] for +/// why it is not shaped like the VAAPI probe below, and for the fail-open contract. Process-wide +/// (not per selected GPU) on purpose: the direct backend opens on the shared `cuda::context()`, +/// i.e. CUDA device 0, so that is the chip whose answer applies. +#[cfg(all(target_os = "linux", feature = "nvenc"))] +pub fn nvenc_codec_support() -> CodecSupport { + use std::sync::OnceLock; + static LOGGED: OnceLock<()> = OnceLock::new(); + let probed = nvenc_cuda::probe_support(); + LOGGED.get_or_init(|| { + tracing::info!( + h264 = probed.codecs.h264, + h265 = probed.codecs.h265, + av1 = probed.codecs.av1, + hevc_444 = probed.hevc_444, + "NVENC encode capabilities probed" + ); + }); + probed.codecs +} + /// Probe the active Linux GPU backend for its encodable codecs (cached; opens a tiny encoder per -/// codec, once). Only the VAAPI (AMD/Intel) backend is probed — NVENC keeps its Moonlight-validated -/// static advertisement (callers gate on [`linux_zero_copy_is_vaapi`]). +/// codec, once). The AMD/Intel backend — NVIDIA has [`nvenc_codec_support`] (callers gate on +/// [`linux_zero_copy_is_vaapi`]). #[cfg(target_os = "linux")] pub fn vaapi_codec_support() -> CodecSupport { use std::sync::OnceLock; @@ -1320,7 +1357,29 @@ pub fn can_encode_444(codec: Codec) -> bool { if linux_zero_copy_is_vaapi() { vaapi::probe_can_encode_444(codec) } else { - linux::probe_can_encode_444(codec) + // NVIDIA. On a direct-SDK host the answer comes from the driver's caps bit over + // the direct SDK ([`nvenc_cuda::probe_support`]) — the same + // `NV_ENC_CAPS_SUPPORT_YUV444_ENCODE` the live session re-checks at open + // (`query_caps` → `yuv444_supported`), and the same shape the Windows NVENC arm + // uses (`nvenc::probe_can_encode_444`). It must NOT fall back to the libav + // open-probe: one ffmpeg `hevc_nvenc` FREXT open in a direct-SDK process is the + // LOG-3 field bug — it wedged every later NVENC open process-wide + // (`NV_ENC_ERR_INVALID_VERSION`) until a host restart. Only a host that will + // really serve the session over libav (PUNKTFUNK_NVENC_DIRECT=0, or a build + // without `--features nvenc`) keeps the ffmpeg probe — there it validates the + // actual session path, and ffmpeg's NVENC client runs in that process anyway. + #[cfg(feature = "nvenc")] + { + if nvenc_direct_enabled() { + nvenc_cuda::probe_support().hevc_444 + } else { + linux::probe_can_encode_444(codec) + } + } + #[cfg(not(feature = "nvenc"))] + { + linux::probe_can_encode_444(codec) + } } } #[cfg(target_os = "windows")] @@ -1624,17 +1683,20 @@ pub fn resolved_backend_ingests_rgb_444() -> bool { } /// True if the active Windows backend's codec advertisement comes from a **real GPU probe** -/// ([`windows_codec_support`]) rather than the NVENC static superset. AMF always qualifies — the +/// ([`windows_codec_support`]) rather than the static superset. AMF always qualifies — the /// native factory probe (`amf::probe_can_encode`) needs no build feature — while QSV qualifies /// with either the native VPL build (`qsv`, the authoritative Query probe) or the `amf-qsv` -/// (libavcodec) build. Formerly `windows_backend_is_ffmpeg`, renamed when the -/// native AMF probe replaced the ffmpeg open-probe (design/native-amf-encoder.md §4, Phase 2). +/// (libavcodec) build, and NVENC with the `nvenc` build (the direct-SDK GUID-list probe, +/// `nvenc::probe_codec_support` — the Windows twin of the Linux probe that ended the +/// advertise-HEVC-on-a-GM107 dead sessions). Formerly `windows_backend_is_ffmpeg`, renamed when +/// the native AMF probe replaced the ffmpeg open-probe (design/native-amf-encoder.md §4, Phase 2). #[cfg(target_os = "windows")] pub fn windows_backend_is_probed() -> bool { match windows_resolved_backend() { WindowsBackend::Amf => true, WindowsBackend::Qsv => cfg!(feature = "qsv") || cfg!(feature = "amf-qsv"), - WindowsBackend::Nvenc | WindowsBackend::Software => false, + WindowsBackend::Nvenc => cfg!(feature = "nvenc"), + WindowsBackend::Software => false, } } @@ -1661,18 +1723,22 @@ fn windows_gpu_vendor() -> Option { .or_else(|| pf_gpu::enumerate().iter().find_map(|g| by_id(g.vendor_id))) } -/// Probe the active Windows AMF/QSV backend for its encodable codecs (cached **per (backend, +/// Probe the active Windows GPU backend for its encodable codecs (cached **per (backend, /// selected GPU)** — a web-console preference change re-probes on the newly selected adapter /// instead of serving the old GPU's answer for the process lifetime). Mirrors -/// [`vaapi_codec_support`]; called only when [`windows_backend_is_probed`] is true. AV1 is narrow -/// (AMD RDNA3+, Intel Arc/Xe2+), so it must be probed, not assumed. +/// [`vaapi_codec_support`]/[`nvenc_codec_support`]; called only when [`windows_backend_is_probed`] +/// is true. AV1 is narrow (NVIDIA Ada+, AMD RDNA3+, Intel Arc/Xe2+) and HEVC needs 2nd-gen +/// Maxwell+ on NVIDIA, so both must be probed, not assumed. /// /// Mirrors the session dispatch (design/native-amf-encoder.md Phase 3): **AMD advertises from the /// native AMF factory probe alone** (`amf::probe_can_encode`, on the selected adapter — the same /// path the session opens, so the advertisement can never claim a codec the session can't emit); /// **Intel/QSV advertises from the native VPL Query probe** (`qsv::probe_can_encode`, /// design/native-qsv-encoder.md §4), falling back to the libavcodec probe on builds without the -/// `qsv` feature (all-`false` without either feature, matching a build that cannot open QSV). +/// `qsv` feature (all-`false` without either feature, matching a build that cannot open QSV); +/// **NVIDIA advertises from the driver's own encode-GUID list** (`nvenc::probe_codec_support`, +/// one throwaway direct-SDK session on the selected adapter — NEVER an ffmpeg open-probe, see the +/// Linux `nvenc_cuda::probe_support` doc for why). #[cfg(target_os = "windows")] pub fn windows_codec_support() -> CodecSupport { use std::collections::HashMap; @@ -1706,22 +1772,31 @@ pub fn windows_codec_support() -> CodecSupport { false } } - // Callers gate on `windows_backend_is_probed` — defensively answer "nothing probed" - // (the advertisement then falls back to the static superset). + // NVENC answers below from ONE GUID-list session, not per-codec probes; Software is + // never probed. Callers gate on `windows_backend_is_probed` — defensively answer + // "nothing probed" (the advertisement then falls back to the static superset). WindowsBackend::Nvenc | WindowsBackend::Software => false, } }; - let caps = CodecSupport { - h264: probe_one(Codec::H264), - h265: probe_one(Codec::H265), - av1: probe_one(Codec::Av1), + let caps = match backend { + // NVIDIA: one throwaway session lists every encode GUID at once — no reason to open + // three sessions through `probe_one`. Featureless builds fall through to `probe_one`'s + // defensive all-false (= "nothing probed" → static superset), matching + // `windows_backend_is_probed`. + #[cfg(feature = "nvenc")] + WindowsBackend::Nvenc => nvenc::probe_codec_support(), + _ => CodecSupport { + h264: probe_one(Codec::H264), + h265: probe_one(Codec::H265), + av1: probe_one(Codec::Av1), + }, }; tracing::info!( ?backend, h264 = caps.h264, h265 = caps.h265, av1 = caps.av1, - "Windows AMF/QSV encode capabilities probed" + "Windows encode capabilities probed" ); // A concurrent first call may double-probe; both arrive at the same answer, last insert wins. cache.lock().unwrap().insert(key, caps); diff --git a/crates/punktfunk-host/src/gamestream/serverinfo.rs b/crates/punktfunk-host/src/gamestream/serverinfo.rs index 58e72e52..b41ea2bb 100644 --- a/crates/punktfunk-host/src/gamestream/serverinfo.rs +++ b/crates/punktfunk-host/src/gamestream/serverinfo.rs @@ -94,9 +94,20 @@ fn base_codec_mode_support() -> u32 { return m; } } - // Windows AMD/Intel (AMF/QSV): advertise only what the GPU actually encodes (AV1 is narrow, an - // old iGPU might lack HEVC). AMF probes natively (no build feature needed); QSV needs the - // libavcodec build. NVENC and the GPU-less software path keep the static superset. + // Linux NVIDIA: the driver's own encode-GUID list (`nvenc_codec_support`, one throwaway + // direct-SDK session, cached) — the same probe `host_wire_caps` consults, so both planes + // stop advertising HEVC/AV1 on a chip without them (the GM107 dead-session field bug). + // Fail-open like every arm here: an unanswerable probe → `probed_mask` = None → superset. + #[cfg(all(target_os = "linux", feature = "nvenc"))] + if !crate::encode::linux_zero_copy_is_vaapi() { + if let Some(m) = probed_mask(crate::encode::nvenc_codec_support()) { + return m; + } + } + // Windows: advertise only what the GPU actually encodes (AV1 is narrow, an old iGPU might + // lack HEVC, a 1st-gen-Maxwell NVENC is H.264-only). AMF probes natively (no build feature + // needed); QSV needs the libavcodec or VPL build, NVENC the `nvenc` build. The GPU-less + // software path keeps the static superset. #[cfg(target_os = "windows")] if crate::encode::windows_backend_is_probed() { if let Some(m) = probed_mask(crate::encode::windows_codec_support()) {