From 1cefd376031a07568d22033d0d64966bfba957db Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Sun, 26 Jul 2026 01:01:30 +0200 Subject: [PATCH] fix(pf-encode): arbitrate NVENC split-encode vs sub-frame readback (Phase 8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified against nvEncodeAPI.h's own splitEncodeMode doc (user-prompted — the audit's 'exclusive for HEVC' one-liner deserved checking): - H.264: split 'is not applicable' — hard-DISABLE the mode so the written config, CeilingKey, the split diagnostic log and the rejection-retry stay truthful (the retry used to re-open a byte-identical session after an H.264 'split rejection'). The libav path's operator arm gains the codec gate its auto arm always had. - HEVC: split 'not supported if … subframe mode' — when WE force split (TWO/THREE/AUTO_FORCED, the 4K120 throughput lever), sub-frame yields with a logged escape (PUNKTFUNK_SPLIT_ENCODE=0 chooses sub-frame). ⚠ Keyed on FORCED modes only, never != DISABLE: AUTO(0) is the resolver's fallthrough for every sub-950Mpix session, and the wider key would have disarmed the Phase-3 chunked-poll feature fleet-wide (critic catch). Under AUTO the driver arbitrates — the shipped state. - AV1: untouched — per-tile sub-frame + split are legal together. The arbitration is a pure nvenc_core fn called by each backend BEFORE the ladder, the ceiling key and the chunked-poll latch — all three see the post-arbitration truth. A drop inside build_init_params would have left poll_chunk busy-polling its whole budget every AU (numSlices stays 0 without reportSliceOffsets; both loop exits dead — critic catch). Linux latches subframe_forced beside subframe_on at query_caps (no env re-reads after open); Windows records the arbitrated state so reconfigure presents exactly the params the open had (also closes the pre-existing mid-session env-flip hazard there). Truth-table tests in nvenc_core; PUNKTFUNK_NVENC_SUBFRAME documented (it never was); PUNKTFUNK_SPLIT_ENCODE row updated. Co-Authored-By: Claude Opus 5 (1M context) --- crates/pf-encode/src/enc/linux/mod.rs | 12 +- crates/pf-encode/src/enc/linux/nvenc_cuda.rs | 20 ++- crates/pf-encode/src/enc/nvenc_core.rs | 135 +++++++++++++++++++ crates/pf-encode/src/enc/windows/nvenc.rs | 69 ++++++++-- docs-site/content/docs/configuration.md | 3 +- 5 files changed, 222 insertions(+), 17 deletions(-) diff --git a/crates/pf-encode/src/enc/linux/mod.rs b/crates/pf-encode/src/enc/linux/mod.rs index 9fc2fc0c..fa93bbe7 100644 --- a/crates/pf-encode/src/enc/linux/mod.rs +++ b/crates/pf-encode/src/enc/linux/mod.rs @@ -448,7 +448,17 @@ impl NvencEncoder { let pix_rate = width as u64 * height as u64 * fps as u64; let split = std::env::var("PUNKTFUNK_SPLIT_ENCODE").ok(); match split.as_deref() { - Some(mode) => opts.set("split_encode_mode", mode), + // The operator arm gains the codec gate the auto arm always had (Phase 8): split + // "is not applicable to H264" per nvEncodeAPI.h, and h264_nvenc has no such AVOption + // — setting it would fail the open on a leftover dict entry. + Some(mode) if matches!(codec, Codec::H265 | Codec::Av1) => { + opts.set("split_encode_mode", mode) + } + Some(_) => tracing::warn!( + codec = codec.nvenc_name(), + "PUNKTFUNK_SPLIT_ENCODE ignored — split encoding is not applicable to H.264 \ + (nvEncodeAPI.h)" + ), None if matches!(codec, Codec::H265 | Codec::Av1) && pix_rate >= super::SPLIT_FORCE_PIXEL_RATE => { diff --git a/crates/pf-encode/src/enc/linux/nvenc_cuda.rs b/crates/pf-encode/src/enc/linux/nvenc_cuda.rs index e274799b..82db0401 100644 --- a/crates/pf-encode/src/enc/linux/nvenc_cuda.rs +++ b/crates/pf-encode/src/enc/linux/nvenc_cuda.rs @@ -62,8 +62,8 @@ use super::nvenc_core::{ apply_low_latency_config, build_init_params, cached_ceiling, codec_guid, plan_range_recovery, - resolve_slices, resolve_split_mode, resolve_subframe, store_ceiling, CeilingKey, - LowLatencyConfig, NvStatusExt, RangePlan, + resolve_slices, resolve_split_mode, resolve_split_subframe, resolve_subframe, store_ceiling, + subframe_env_forced, CeilingKey, LowLatencyConfig, NvStatusExt, RangePlan, }; use super::nvenc_status; use super::{AuChunk, ChromaFormat, Codec, EncodedFrame, Encoder, EncoderCaps}; @@ -629,6 +629,10 @@ pub struct NvencCudaEncoder { /// [`subframe_cap`](Self::subframe_cap)); consumed by every `build_init_params` call so the /// open and the in-place reconfigure present identical init params. subframe_on: bool, + /// Whether `PUNKTFUNK_NVENC_SUBFRAME=1` was EXPLICITLY forced — latched with `subframe_on` + /// (same invariant: open and reconfigure must present identical init params, so no env + /// re-reads after `query_caps`). Only consumed by [`resolve_split_subframe`]'s log severity. + subframe_forced: bool, /// Sub-frame chunked poll armed for the live session (§7 LN1 Phase 1): multi-slice + /// sub-frame readback configured AND sync retrieve at init. See [`Encoder::poll_chunk`]. subframe_chunks: bool, @@ -721,6 +725,7 @@ impl NvencCudaEncoder { slices: 1, subframe_cap: false, subframe_on: false, + subframe_forced: false, subframe_chunks: false, chunk: None, }) @@ -921,6 +926,7 @@ impl NvencCudaEncoder { // escapes. self.slices = resolve_slices(self.codec, 4); self.subframe_on = resolve_subframe(self.subframe_cap); + self.subframe_forced = subframe_env_forced(); tracing::info!( rfi = self.rfi_supported, custom_vbv = self.custom_vbv, @@ -1136,6 +1142,16 @@ impl NvencCudaEncoder { // [`resolve_split_mode`] for the precedence (env override / 10-bit / pixel rate). let pixel_rate = self.width as u64 * self.height as u64 * self.fps.max(1) as u64; let split_mode: u32 = resolve_split_mode(self.bit_depth, pixel_rate); + // Split × sub-frame arbitration (Phase 8) BEFORE the ladder, the ceiling key and the + // chunked-poll latch — all three must see the post-arbitration truth (a drop inside + // build_init_params would leave poll_chunk busy-polling its whole budget per AU). + let (split_mode, subframe_on) = resolve_split_subframe( + self.codec, + split_mode, + self.subframe_on, + self.subframe_forced, + ); + self.subframe_on = subframe_on; const CLAMP_TOL_BPS: u64 = 20_000_000; // Ceiling cache (process lifetime, `nvenc_core`): a prior clamp search already found diff --git a/crates/pf-encode/src/enc/nvenc_core.rs b/crates/pf-encode/src/enc/nvenc_core.rs index ba835410..3b2ba46b 100644 --- a/crates/pf-encode/src/enc/nvenc_core.rs +++ b/crates/pf-encode/src/enc/nvenc_core.rs @@ -102,6 +102,141 @@ pub(super) fn resolve_split_mode(bit_depth: u8, pixel_rate: u64) -> u32 { mode } +/// Whether the operator EXPLICITLY forced sub-frame readback on (`PUNKTFUNK_NVENC_SUBFRAME=1`) +/// — the log-severity input to [`resolve_split_subframe`]: a forced knob being overridden +/// deserves a `warn`, a default being tuned an `info`. Callers LATCH this once next to their +/// resolved subframe state (an env re-read at reconfigure would violate the "open and +/// reconfigure present identical init params" invariant). +/// Linux-cfg'd like its ONLY caller (the `nvenc_cuda` query_caps latch) — Windows sessions have +/// `subframe == forced` by construction (env opt-in only) and never consult this; without the +/// cfg it is dead code on every Windows leg (item-level dead_code, the recurring trap). +#[cfg(target_os = "linux")] +pub(super) fn subframe_env_forced() -> bool { + matches!( + std::env::var("PUNKTFUNK_NVENC_SUBFRAME").as_deref(), + Ok("1") + ) +} + +/// The split-encode × sub-frame arbitration (Phase 8; verified against `nvEncodeAPI.h`'s own +/// `splitEncodeMode` doc, not folklore): +/// - **H.264**: split "is not applicable" — hard-DISABLE the mode so the written config, the +/// ceiling-cache key, the split diagnostic log and the rejection-retry all stay truthful (the +/// retry used to re-open a byte-identical session after an H.264 "split rejection"). +/// - **HEVC**: split is "not supported if … subframe mode" — when WE force split +/// (TWO/THREE/AUTO_FORCED, e.g. the 4K120 throughput requirement), sub-frame yields. Under +/// plain AUTO the driver arbitrates — the shipped fleet state (1080p–1440p240 all run +/// AUTO+subframe); keying on `!= DISABLE` here would have disarmed the Phase-3 chunked-poll +/// feature fleet-wide. +/// - **AV1**: both legal (sub-frame is per-tile; split is constrained only by +/// output-into-vidmem, which we never use) — untouched. +/// +/// Returns the `(split_mode, subframe)` to ACTUALLY configure. The caller must store BOTH back +/// (the chunked-poll latch and `CeilingKey` key on them) — a silent in-params drop would leave +/// `poll_chunk` busy-polling its full budget every AU (`numSlices` stays 0 without +/// `reportSliceOffsets`, so neither loop exit ever fires). +pub(super) fn resolve_split_subframe( + codec: Codec, + split_mode: u32, + subframe: bool, + subframe_forced: bool, +) -> (u32, bool) { + use nv::NV_ENC_SPLIT_ENCODE_MODE as M; + if codec == Codec::H264 { + return (M::NV_ENC_SPLIT_DISABLE_MODE as u32, subframe); + } + let split_forced = split_mode == M::NV_ENC_SPLIT_TWO_FORCED_MODE as u32 + || split_mode == M::NV_ENC_SPLIT_THREE_FORCED_MODE as u32 + || split_mode == M::NV_ENC_SPLIT_AUTO_FORCED_MODE as u32; + if codec == Codec::H265 && split_forced && subframe { + if subframe_forced { + tracing::warn!( + split_mode, + "HEVC forced split-encode and PUNKTFUNK_NVENC_SUBFRAME=1 are mutually \ + unsupported (nvEncodeAPI.h) — sub-frame readback disabled for this session; \ + set PUNKTFUNK_SPLIT_ENCODE=0 to choose sub-frame instead" + ); + } else { + tracing::info!( + split_mode, + "HEVC forced split-encode supersedes default-on sub-frame readback (mutually \ + unsupported per nvEncodeAPI.h; split is the 4K120 throughput lever) — set \ + PUNKTFUNK_SPLIT_ENCODE=0 to choose sub-frame instead" + ); + } + return (split_mode, false); + } + (split_mode, subframe) +} + +#[cfg(test)] +mod split_subframe_tests { + use super::{resolve_split_subframe, Codec}; + use nvidia_video_codec_sdk::sys::nvEncodeAPI::NV_ENC_SPLIT_ENCODE_MODE as M; + + const AUTO: u32 = M::NV_ENC_SPLIT_AUTO_MODE as u32; + const TWO: u32 = M::NV_ENC_SPLIT_TWO_FORCED_MODE as u32; + const AUTO_F: u32 = M::NV_ENC_SPLIT_AUTO_FORCED_MODE as u32; + const DISABLE: u32 = M::NV_ENC_SPLIT_DISABLE_MODE as u32; + + /// THE FLEET CASE: plain AUTO + default-on sub-frame must pass through untouched — the + /// driver arbitrates. Keying the rule on `!= DISABLE` would disarm sub-frame on every + /// default Linux HEVC session (AUTO == 0 is the resolver's fallthrough). + #[test] + fn hevc_auto_keeps_subframe() { + assert_eq!( + resolve_split_subframe(Codec::H265, AUTO, true, false), + (AUTO, true) + ); + } + + #[test] + fn hevc_forced_split_drops_subframe() { + assert_eq!( + resolve_split_subframe(Codec::H265, TWO, true, false), + (TWO, false) + ); + assert_eq!( + resolve_split_subframe(Codec::H265, AUTO_F, true, true), + (AUTO_F, false) + ); + // No sub-frame to drop → nothing changes. + assert_eq!( + resolve_split_subframe(Codec::H265, TWO, false, false), + (TWO, false) + ); + // Explicitly disabled split → sub-frame kept (the documented escape). + assert_eq!( + resolve_split_subframe(Codec::H265, DISABLE, true, true), + (DISABLE, true) + ); + } + + /// H.264: split "is not applicable" (nvEncodeAPI.h) — hard-DISABLE regardless of the + /// resolved mode; sub-frame (H.264 slices) is unaffected. + #[test] + fn h264_split_hard_disabled() { + assert_eq!( + resolve_split_subframe(Codec::H264, TWO, true, false), + (DISABLE, true) + ); + assert_eq!( + resolve_split_subframe(Codec::H264, AUTO, false, false), + (DISABLE, false) + ); + } + + /// AV1: both features are legal together (per-tile sub-frame; split constrained only by + /// output-into-vidmem) — the arbitration must not touch it. + #[test] + fn av1_untouched() { + assert_eq!( + resolve_split_subframe(Codec::Av1, TWO, true, true), + (TWO, true) + ); + } +} + /// One session config's identity for the process-lifetime bitrate-ceiling cache /// ([`cached_ceiling`]/[`store_ceiling`]). Everything the driver's codec-level validation keys /// off: the GPU (different NVENC generations have different level ceilings), dims/fps (the luma diff --git a/crates/pf-encode/src/enc/windows/nvenc.rs b/crates/pf-encode/src/enc/windows/nvenc.rs index 955738f6..94ae666d 100644 --- a/crates/pf-encode/src/enc/windows/nvenc.rs +++ b/crates/pf-encode/src/enc/windows/nvenc.rs @@ -38,8 +38,8 @@ use super::nvenc_core::{ apply_low_latency_config, build_init_params, cached_ceiling, codec_guid, plan_range_recovery, - resolve_slices, resolve_split_mode, resolve_subframe, store_ceiling, CeilingKey, - LowLatencyConfig, NvStatusExt, RangePlan, + resolve_slices, resolve_split_mode, resolve_split_subframe, resolve_subframe, store_ceiling, + CeilingKey, LowLatencyConfig, NvStatusExt, RangePlan, }; use super::nvenc_status; use super::{ChromaFormat, Codec, EncodedFrame, Encoder, EncoderCaps}; @@ -603,6 +603,11 @@ pub struct NvencD3d11Encoder { /// `reconfigure_bitrate` must present the SAME init params as the open (only the config's /// rate fields may move). Meaningless while `inited` is false. split_mode: u32, + /// The session's arbitrated sub-frame state ([`resolve_split_subframe`] at open) — recorded + /// so reconfigure presents EXACTLY the init params the session opened with (an env re-read + /// there could flip `enableSubFrameWrite` mid-session, and would re-log the arbitration on + /// every ABR retarget). + subframe_on: bool, session_async: bool, /// The last reference-frame range we invalidated — dedupes repeated RFI requests for the same /// loss event (the client resends until it sees recovery). @@ -686,6 +691,7 @@ impl NvencD3d11Encoder { rfi_supported: false, custom_vbv: false, split_mode: nv::NV_ENC_SPLIT_ENCODE_MODE::NV_ENC_SPLIT_DISABLE_MODE as u32, + subframe_on: false, session_async: false, last_rfi_range: None, init_device: ptr::null_mut(), @@ -980,6 +986,7 @@ impl NvencD3d11Encoder { bitrate: u64, split_mode: u32, enable_async: bool, + subframe: bool, ) -> Result<*mut c_void> { let mut params = nv::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS { version: nv::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER, @@ -1014,9 +1021,10 @@ impl NvencD3d11Encoder { &mut cfg, split_mode, enable_async, - // Windows: env opt-in only ("1"), never a default — and build_init_params - // additionally refuses it on an async session. - resolve_subframe(false), + // Windows: env opt-in only ("1"), never a default; arbitrated against split by the + // caller (resolve_split_subframe) — and build_init_params additionally refuses it + // on an async session. + subframe, ); match (api().initialize_encoder)(enc, &mut init).nv_ok() { @@ -1069,6 +1077,11 @@ impl NvencD3d11Encoder { // The init-failure fallback below disables it if a codec/config rejects it. let pixel_rate = self.width as u64 * self.height as u64 * self.fps.max(1) as u64; let split_mode: u32 = resolve_split_mode(self.bit_depth, pixel_rate); + // Split × sub-frame arbitration (Phase 8) before the ladder/ceiling key. On Windows + // sub-frame is env-opt-in only, so resolved == forced by construction. + let subframe_req = resolve_subframe(false); + let (split_mode, subframe_req) = + resolve_split_subframe(self.codec, split_mode, subframe_req, subframe_req); // Find the highest bitrate the GPU's codec LEVEL accepts and CLAMP to it. NVENC rejects // `initialize_encoder` (InvalidParam) when the bitrate exceeds the level ceiling (e.g. a // 1 Gbps request on HEVC). Strategy: try the requested rate; if the only problem is a forced @@ -1098,12 +1111,19 @@ impl NvencD3d11Encoder { } } - let mut probe = self.try_open_session(device, target_bps, split_mode, use_async); + let mut probe = + self.try_open_session(device, target_bps, split_mode, use_async, subframe_req); // The cache is advisory: a stale entry (driver change, identity collision) must not // wedge the open — retry the requested rate and let the search below rediscover. if probe.is_err() && target_bps < requested_bps { target_bps = requested_bps; - probe = self.try_open_session(device, requested_bps, split_mode, use_async); + probe = self.try_open_session( + device, + requested_bps, + split_mode, + use_async, + subframe_req, + ); } // Disambiguate a forced-split rejection from a bitrate-cap rejection: retry once at the // requested rate with split disabled — if THAT succeeds, split was the problem, not bitrate. @@ -1117,7 +1137,9 @@ impl NvencD3d11Encoder { split_mode != nv::NV_ENC_SPLIT_ENCODE_MODE::NV_ENC_SPLIT_DISABLE_MODE as u32; if probe.is_err() && split_on { let no_split = nv::NV_ENC_SPLIT_ENCODE_MODE::NV_ENC_SPLIT_DISABLE_MODE as u32; - if let Ok(e) = self.try_open_session(device, target_bps, no_split, use_async) { + if let Ok(e) = + self.try_open_session(device, target_bps, no_split, use_async, subframe_req) + { tracing::warn!("NVENC: split-encode rejected by codec/config — disabled"); used_split = no_split; probe = Ok(e); @@ -1144,7 +1166,13 @@ impl NvencD3d11Encoder { let mut best_bps = 0u64; while hi > lo + CLAMP_TOL_BPS { let mid = lo + (hi - lo) / 2; - match self.try_open_session(device, mid, used_split, use_async) { + match self.try_open_session( + device, + mid, + used_split, + use_async, + subframe_req, + ) { Ok(e) => { if !best.is_null() { let _ = (api().destroy_encoder)(best); @@ -1169,12 +1197,23 @@ impl NvencD3d11Encoder { // trying split-disabled in case a forced split (not the bitrate) is the blocker. let no_split = nv::NV_ENC_SPLIT_ENCODE_MODE::NV_ENC_SPLIT_DISABLE_MODE as u32; - best = match self.try_open_session(device, FLOOR_BPS, used_split, use_async) - { + best = match self.try_open_session( + device, + FLOOR_BPS, + used_split, + use_async, + subframe_req, + ) { Ok(e) => e, Err(_) => { let e = self - .try_open_session(device, FLOOR_BPS, no_split, use_async) + .try_open_session( + device, + FLOOR_BPS, + no_split, + use_async, + subframe_req, + ) .context( "NVENC initialize_encoder rejected even at the floor bitrate", )?; @@ -1200,6 +1239,7 @@ impl NvencD3d11Encoder { self.init_device_com = Some(device.clone()); // Session init params a later `reconfigure_bitrate` must re-present verbatim. self.split_mode = used_split; + self.subframe_on = subframe_req; self.session_async = use_async; // Session-budget accounting (Stage W3): record what this open holds so admission can // decline a parallel display the hardware can't afford. Weighted by the FINAL split @@ -1855,7 +1895,10 @@ impl Encoder for NvencD3d11Encoder { &mut cfg, self.split_mode, self.session_async, - resolve_subframe(false), + // The SESSION's recorded state, not a fresh env read: reconfigure must + // present exactly the init params the open arbitrated (an env re-read could + // flip enableSubFrameWrite mid-session — the Linux latch invariant). + self.subframe_on, ), ..Default::default() }; diff --git a/docs-site/content/docs/configuration.md b/docs-site/content/docs/configuration.md index e8ccaf3a..2f5118ee 100644 --- a/docs-site/content/docs/configuration.md +++ b/docs-site/content/docs/configuration.md @@ -143,7 +143,8 @@ notes for context. | Setting | Values | Meaning | |---|---|---| | `PUNKTFUNK_GSO` | `1` · `0` | UDP Generic Segmentation Offload on the send path (coalesce a frame's packets into kernel super-buffers) — cuts send CPU ~30%, but its line-rate packet trains can cost delivered throughput on constrained links (measured on a 2.5GbE hop). Off by default until send pacing spaces the super-buffers; set `1` to opt in (auto-falls back to `sendmmsg` on kernels/paths without support). | -| `PUNKTFUNK_SPLIT_ENCODE` | `0`/`disable` · `1`/`auto` · `2` · `3` | NVENC N-way split-encode for very high pixel rates (5K@240). `auto` picks automatically above ~1 Gpix/s. | +| `PUNKTFUNK_SPLIT_ENCODE` | `0`/`disable` · `1`/`auto` · `2` · `3` | NVENC N-way split-encode for very high pixel rates (5K@240). `auto` picks automatically above ~1 Gpix/s. H.264 never splits (not applicable per the SDK); on HEVC a *forced* split disables sub-frame readback (mutually unsupported) — set `0` to choose sub-frame instead. | +| `PUNKTFUNK_NVENC_SUBFRAME` | `0` · `1` | NVENC sub-frame (slice-level) readback for lower latency on sync sessions. Default: on where the GPU supports it (Linux direct NVENC). `0` = never; `1` = force. On HEVC it yields to a forced split-encode (the SDK documents the pair unsupported). | | `PUNKTFUNK_GPU_PRIORITY_CLASS` | `off` · `normal` · `high` · `realtime` · `auto` | **(Windows)** GPU scheduling priority for capture/encode under a GPU-saturating game. Default `auto` (starts `high`, upgrades to `realtime` when it's safe — e.g. HAGS off); `high` pins the static pre-gate behaviour; `realtime` is the strongest lever but can freeze NVENC on some setups. | | `PUNKTFUNK_IDD_DEPTH` | `N` (default `2`) | **(Windows)** IDD-push pipeline depth. `1` cuts latency once GPU priority is raised; higher smooths a contended GPU. |