diff --git a/crates/pf-encode/src/enc/linux/nvenc_cuda.rs b/crates/pf-encode/src/enc/linux/nvenc_cuda.rs index 59261447..418f9c88 100644 --- a/crates/pf-encode/src/enc/linux/nvenc_cuda.rs +++ b/crates/pf-encode/src/enc/linux/nvenc_cuda.rs @@ -20,9 +20,21 @@ //! pointer-keyed, so registering a fresh pool pointer each frame would thrash it) — so it is //! zero regression versus today; true zero-copy input registration is a follow-up. //! -//! **Sync-only.** NVENC async mode (`enableEncodeAsync` + Win32 completion events) is Windows-only, -//! so the whole two-thread async-retrieve subsystem of the Windows backend is absent here: `poll` -//! does the blocking `lock_bitstream`, exactly like the libav path. +//! **Two-thread retrieve** (`PUNKTFUNK_NVENC_ASYNC=1`, the same opt-in knob as the Windows +//! backend — gpu-contention plan §5.B, latency plan T2.2): NVENC *async mode* +//! (`enableEncodeAsync` + completion events) is Windows-only, so the session here stays SYNC — +//! but the NVENC guide's threading model still applies: the main thread should only *submit* +//! while a secondary thread does the (blocking) `nvEncLockBitstream`. With the flag set, an +//! internal retrieve thread owns exactly that blocking lock (+ copy + unlock); `submit` returns +//! after `encode_picture` and `poll` drains finished AUs without blocking, so under a +//! GPU-saturating game completed frames queue instead of serializing capture on the scheduler +//! wait. All input-resource calls (register/map/unmap) and every other session call stay on the +//! encode thread. Backpressure: `submit` blocks on the oldest completion at +//! `PUNKTFUNK_NVENC_ASYNC_DEPTH` (default 4) in-flight encodes. Without the flag, `poll` does +//! the blocking `lock_bitstream` on the encode thread, exactly like the libav path (unchanged +//! default). Caveat shared with the sync path: a driver wedge that hangs `lock_bitstream` hangs +//! the retrieve thread the same way it would hang the encode thread today (Linux has no +//! event-timeout escape) — no regression, just no new watchdog either. //! //! Needs a real NVIDIA GPU at runtime (session creation fails otherwise); compiles GPU-less and //! starts driver-less (the `.so` resolves at runtime — on an AMD/Intel box [`try_api`] fails cleanly @@ -42,6 +54,7 @@ use pf_zerocopy::cuda::{self, InputSurface}; use std::collections::VecDeque; use std::ffi::c_void; use std::ptr; +use std::sync::mpsc; use nvidia_video_codec_sdk::sys::nvEncodeAPI as nv; @@ -205,11 +218,135 @@ fn load_api() -> std::result::Result { } } -/// Output bitstream buffers = max in-flight encodes; equals the input-surface ring depth. The host -/// loop deep-pipelines (submits several frames before locking the oldest) so this must be ≥ the -/// helper's `PUNKTFUNK_ENCODE_DEPTH` (default 4, clamped ≤ 6). +/// Output bitstream buffers = max in-flight encodes; equals the input-surface ring depth. Must +/// stay ≥ the two-thread retrieve's in-flight cap ([`async_inflight_cap`], ≤ `POOL - 1`) so a +/// bitstream/ring slot is never reused mid-encode. const POOL: usize = 8; +/// Whether the operator asked for the two-thread retrieve (`PUNKTFUNK_NVENC_ASYNC` truthy — the +/// SAME knob as the Windows backend, so one env drives the split on either host OS). Opt-in +/// until on-glass validated. Unlike Windows this changes NO session parameter (Linux stays sync +/// mode; only the blocking lock moves off the encode thread), so there is no async-rejecting +/// config to fail the open. +fn async_retrieve_requested() -> bool { + std::env::var("PUNKTFUNK_NVENC_ASYNC") + .map(|v| matches!(v.trim(), "1" | "true" | "yes" | "on")) + .unwrap_or(false) +} + +/// Max encodes in flight in two-thread mode (`PUNKTFUNK_NVENC_ASYNC_DEPTH`, default 4, clamped +/// `2..=POOL-1` — a bitstream must never be reused mid-encode, and the input ring is the same +/// depth). Mirrors the Windows knob exactly. +fn async_inflight_cap() -> usize { + std::env::var("PUNKTFUNK_NVENC_ASYNC_DEPTH") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(4) + .clamp(2, POOL - 1) +} + +/// One in-flight encode handed to the retrieve thread: the output bitstream to (blocking-)lock. +/// Raw pointer travels as `usize` (a process-global driver handle; the thread is joined before +/// the session it belongs to is destroyed). +struct RetrieveJob { + bs: usize, +} + +/// A finished retrieve: the locked-and-copied AU (or the retrieve-side error) for the oldest +/// in-flight bitstream. `bs` lets the encode thread cross-check FIFO pairing with `pending`. +struct RetrieveDone { + bs: usize, + result: std::result::Result<(Vec, bool), String>, +} + +/// The two-thread-retrieve runtime: the job channel feeding the retrieve thread, the completion +/// channel back, the thread handle (joined in `teardown` BEFORE the session is destroyed), and +/// AUs already absorbed by backpressure that `poll` hands out first. +struct AsyncRetrieve { + work_tx: Option>, + done_rx: mpsc::Receiver, + join: Option>, + ready: VecDeque, +} + +impl AsyncRetrieve { + fn spawn(enc: usize) -> Self { + let (work_tx, work_rx) = mpsc::sync_channel::(POOL); + let (done_tx, done_rx) = mpsc::channel::(); + let join = std::thread::Builder::new() + .name("pf-nvenc-out".into()) + .spawn(move || retrieve_loop(enc, work_rx, done_tx)) + .expect("spawn pf-nvenc-out"); + AsyncRetrieve { + work_tx: Some(work_tx), + done_rx, + join: Some(join), + ready: VecDeque::new(), + } + } +} + +/// The retrieve-thread body (latency plan T2.2, the Linux half of gpu-contention §5.B): for each +/// submitted frame, BLOCKING-lock the bitstream (sync-mode `nvEncLockBitstream` returns when the +/// encode completes — the guide's sanctioned secondary-thread surface), copy the AU out, unlock, +/// and send it back. Exits when the job channel closes (teardown drops the sender and joins +/// BEFORE destroying the session, so `enc` and every `bs` outlive their uses here). +fn retrieve_loop( + enc: usize, + work_rx: mpsc::Receiver, + done_tx: mpsc::Sender, +) { + pf_frame::thread_qos::boost_thread_priority(false); + // The session is bound to the shared process-wide CUDA context; make it current here the + // same way the encode thread does before its own NVENC calls. + if let Err(e) = cuda::make_current() { + tracing::warn!(error = %format!("{e:#}"), "pf-nvenc-out: cuCtxSetCurrent failed"); + } + while let Ok(job) = work_rx.recv() { + // SAFETY: `job.bs` is one of the session's pool bitstreams a prior `encode_picture` + // targeted; both it and the session stay valid until `teardown`, which joins this thread + // first. `lock_bitstream` (version set, struct a live stack local for the synchronous + // call) BLOCKS until that encode finishes, then yields a CPU-readable + // `bitstreamBufferPtr`/`bitstreamSizeInBytes` valid until `unlock_bitstream`; the slice + // is copied (`to_vec`) before the unlock on the same buffer. Lock/unlock from a + // secondary thread while the encode thread submits is the NVENC guide's documented + // threading model. + let result = unsafe { + let mut lock = nv::NV_ENC_LOCK_BITSTREAM { + version: nv::NV_ENC_LOCK_BITSTREAM_VER, + outputBitstream: job.bs as *mut c_void, + ..Default::default() + }; + match (api().lock_bitstream)(enc as *mut c_void, &mut lock).nv_ok() { + Ok(()) => { + let data = std::slice::from_raw_parts( + lock.bitstreamBufferPtr as *const u8, + lock.bitstreamSizeInBytes as usize, + ) + .to_vec(); + let keyframe = matches!( + lock.pictureType, + nv::NV_ENC_PIC_TYPE::NV_ENC_PIC_TYPE_IDR + | nv::NV_ENC_PIC_TYPE::NV_ENC_PIC_TYPE_I + ); + let _ = (api().unlock_bitstream)( + enc as *mut c_void, + job.bs as nv::NV_ENC_OUTPUT_PTR, + ); + Ok((data, keyframe)) + } + Err(e) => Err(format!( + "lock_bitstream (retrieve thread): {e:?} — {}", + nvenc_status::explain(e) + )), + } + }; + if done_tx.send(RetrieveDone { bs: job.bs, result }).is_err() { + break; // encoder side gone (teardown drains us via join) + } + } +} + /// The NVENC input buffer format for a captured `DeviceBuffer`'s layout. NV12/YUV444 are the zero- /// copy worker's convert outputs; packed RGB (`ABGR`) is the fallback where NVENC does the internal /// CSC. 10-bit is never produced on Linux today (Phase 5.1), so everything is 8-bit. @@ -300,6 +437,9 @@ pub struct NvencCudaEncoder { /// One-shot latch for [`diagnose_failed_open`](Self::diagnose_failed_open) so a rebuild-retry /// burst (the session loop's bounded encoder resets) logs the diagnosis once, not per attempt. diagnosed: bool, + /// The two-thread retrieve runtime (`PUNKTFUNK_NVENC_ASYNC`) — `None` in the default + /// single-thread mode and between sessions. Exists only `init_session`→`teardown`. + async_rt: Option, } // SAFETY: the `!Send` fields are the raw NVENC session handle (`encoder`), the shared `CUcontext` @@ -373,6 +513,7 @@ impl NvencCudaEncoder { custom_vbv: false, split_mode: nv::NV_ENC_SPLIT_ENCODE_MODE::NV_ENC_SPLIT_DISABLE_MODE as u32, last_rfi_range: None, + async_rt: None, }) } @@ -381,6 +522,15 @@ impl NvencCudaEncoder { if self.encoder.is_null() { return; } + // Stop the retrieve thread FIRST: close its job channel and join. Any in-flight blocking + // lock returns once its encode completes (≤ a frame time on a live driver), so the join + // is bounded; after it no other thread can touch the session the code below destroys. + if let Some(mut rt) = self.async_rt.take() { + rt.work_tx.take(); + if let Some(j) = rt.join.take() { + let _ = j.join(); + } + } // Unmap any in-flight inputs, unregister every ring surface, destroy the bitstreams. for (_, map, _, _) in &self.pending { if !map.is_null() { @@ -804,6 +954,15 @@ impl NvencCudaEncoder { } self.inited = true; + // Two-thread retrieve (T2.2): spawn the lock thread against the live session. No + // session parameter differs — teardown/rebuild always stops it before destroy. + if async_retrieve_requested() { + self.async_rt = Some(AsyncRetrieve::spawn(self.encoder as usize)); + tracing::info!( + depth = async_inflight_cap(), + "NVENC two-thread retrieve enabled (submit thread + blocking-lock thread)" + ); + } tracing::info!( mode = %format_args!("{}x{}@{}", self.width, self.height, self.fps), bit_depth = self.bit_depth, @@ -845,6 +1004,40 @@ impl NvencCudaEncoder { _ => cuda::copy_device_to_device(buf, base, pitch), } } + + /// Fold one retrieve-thread completion into `ready` (two-thread mode only): pop the oldest + /// in-flight entry, cross-check FIFO pairing, unmap its input HERE (the encode thread — the + /// retrieve thread never touches input resources), and queue the finished AU. + fn absorb_done(&mut self, done: RetrieveDone) -> Result<()> { + let Some((bs, map, pts_ns, anchor)) = self.pending.pop_front() else { + bail!("NVENC retrieve: completion with no in-flight frame (pairing bug)"); + }; + if bs as usize != done.bs { + bail!("NVENC retrieve: completion out of order (pairing bug)"); + } + // SAFETY: `map` is the mapped input `submit` recorded for exactly this now-completed + // encode; the session is live (`async_rt` exists only between `init_session` and + // `teardown`) and this runs on the encode thread — the single unmap here mirrors the + // sync path's poll-side unmap, exactly once per mapping. + unsafe { + if !map.is_null() { + let _ = (api().unmap_input_resource)(self.encoder, map); + } + } + let (data, keyframe) = done.result.map_err(|e| anyhow!("{e}"))?; + self.async_rt + .as_mut() + .expect("absorb_done is only reachable in two-thread mode") + .ready + .push_back(EncodedFrame { + data, + pts_ns, + keyframe, + recovery_anchor: anchor, + chunk_aligned: false, + }); + Ok(()) + } } impl Encoder for NvencCudaEncoder { @@ -891,6 +1084,19 @@ impl Encoder for NvencCudaEncoder { // output slot counter (`teardown` zeroes it), NOT `pts`: `submit_indexed` pins pts to the // wire frame index, non-zero on a mid-session rebuild's first frame. let opening = self.next == 0; + // Two-thread backpressure: never more than the cap in flight — block on the OLDEST + // completion first, absorbing its AU into `ready` for `poll`. Bounds the added latency + // exactly like the sync path's blocking poll, just `cap` deep instead of 1, and keeps + // this slot's bitstream/input surface free before they're reused below. + while self.async_rt.is_some() && self.pending.len() >= async_inflight_cap() { + let done = { + let rt = self.async_rt.as_mut().expect("checked in loop condition"); + rt.done_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .map_err(|_| anyhow!("NVENC retrieve stalled (5s) — encoder wedged?"))? + }; + self.absorb_done(done)?; + } let slot = self.next % POOL; self.next += 1; @@ -1057,6 +1263,15 @@ impl Encoder for NvencCudaEncoder { anchor, )); } + // Two-thread mode: hand the blocking lock for this bitstream to the retrieve thread. + // The sync_channel(POOL) can never fill (in-flight is capped < POOL above). + if let Some(rt) = &self.async_rt { + if let Some(tx) = &rt.work_tx { + let _ = tx.send(RetrieveJob { + bs: self.bitstreams[slot] as usize, + }); + } + } Ok(()) } @@ -1130,6 +1345,26 @@ impl Encoder for NvencCudaEncoder { } fn poll(&mut self) -> Result> { + // Two-thread mode: drain whatever the retrieve thread has finished (non-blocking) and + // hand out the oldest ready AU. `None` = nothing completed yet — the session loop keeps + // the frame in flight and re-polls next tick; capture never blocks on the encode wait. + if self.async_rt.is_some() { + while let Ok(done) = self + .async_rt + .as_mut() + .expect("checked just above") + .done_rx + .try_recv() + { + self.absorb_done(done)?; + } + return Ok(self + .async_rt + .as_mut() + .expect("checked just above") + .ready + .pop_front()); + } let Some((bs, map, pts_ns, anchor)) = self.pending.pop_front() else { return Ok(None); }; diff --git a/crates/pf-frame/src/dxgi.rs b/crates/pf-frame/src/dxgi.rs index 479833e1..f2b48733 100644 --- a/crates/pf-frame/src/dxgi.rs +++ b/crates/pf-frame/src/dxgi.rs @@ -98,21 +98,40 @@ pub unsafe fn make_device(adapter: &IDXGIAdapter1) -> Result<(ID3D11Device, ID3D if let Ok(dxgi1) = device.cast::() { let _ = dxgi1.SetMaximumFrameLatency(1); } + // REALTIME auto-gate (gpu-contention §5.C / latency plan T2.3) — needs the device's adapter, + // so it runs here, after creation; internally once-per-process. + auto_priority_gate(&device); Ok((device, context)) } -/// Resolve the configured GPU scheduling-priority class from `PUNKTFUNK_GPU_PRIORITY_CLASS` -/// (`off|normal|high|realtime`, default high). `None` = leave it at the OS default (the `off` opt-out). -/// D3DKMT_SCHEDULINGPRIORITYCLASS: IDLE 0, BELOW_NORMAL 1, NORMAL 2, ABOVE_NORMAL 3, HIGH 4, REALTIME 5. -fn configured_gpu_priority_class() -> Option { +/// The configured GPU scheduling-priority policy (`PUNKTFUNK_GPU_PRIORITY_CLASS`). +enum PrioMode { + /// Leave the OS default untouched (`off`). + Off, + /// A fixed class the operator pinned (`normal`=2 / `high`=4 / `realtime`=5). + Static(i32), + /// The default: HIGH immediately, then upgrade to REALTIME when it is safe — HAGS off, or + /// HAGS on with comfortable VRAM headroom (with a monitor that downgrades the moment VRAM + /// tightens). REALTIME is the proven ceiling-raiser (it is how our brief encode preempts a + /// saturating game), but REALTIME + NVIDIA + HAGS + near-full VRAM is a documented NVENC + /// hang — the gate takes the win everywhere it cannot hit the hazard. + Auto, +} + +/// Resolve `PUNKTFUNK_GPU_PRIORITY_CLASS` (`off|normal|high|realtime|auto`, default **auto**). +/// D3DKMT_SCHEDULINGPRIORITYCLASS: IDLE 0, BELOW_NORMAL 1, NORMAL 2, ABOVE_NORMAL 3, HIGH 4, +/// REALTIME 5. `realtime` pins REALTIME statically (no gate — the operator owns the hazard); +/// `high` restores the pre-T2.3 static default. +fn configured_gpu_priority_mode() -> PrioMode { match std::env::var("PUNKTFUNK_GPU_PRIORITY_CLASS") .ok() .as_deref() { - Some("off") => None, - Some("normal") => Some(2), - Some("realtime") => Some(5), - _ => Some(4), // HIGH — safe on NVIDIA+HAGS (realtime can freeze NVENC) + Some("off") => PrioMode::Off, + Some("normal") => PrioMode::Static(2), + Some("high") => PrioMode::Static(4), + Some("realtime") => PrioMode::Static(5), + _ => PrioMode::Auto, } } @@ -186,11 +205,14 @@ unsafe fn d3dkmt_set_scheduling_priority_class( /// GPU-saturated game our capture+encode process is starved of GPU time slices — NVENC sits ~idle but /// `lock_bitstream` waits ~20 ms for our context to be scheduled. Elevating the PROCESS GPU scheduling /// priority class (the strong cross-process lever — far more effective than `SetGPUThreadPriority` -/// alone, which we measured as no help) lets our brief encode preempt the game. Uses HIGH, NOT -/// realtime: realtime on NVIDIA + HAGS can freeze/crash NVENC (Apollo downgrades it for exactly this). -/// Runs once per process; best-effort. `PUNKTFUNK_GPU_PRIORITY_CLASS = off|normal|high|realtime` -/// (default high). Best-effort: silently no-ops under a UAC-filtered token (the process will not -/// hold SE_INC_BASE_PRIORITY, so the D3DKMT call is a no-op). +/// alone, which we measured as no help) lets our brief encode preempt the game. Default is the +/// T2.3 `auto` mode: HIGH immediately here, then [`auto_priority_gate`] upgrades to REALTIME +/// where the NVIDIA+HAGS+full-VRAM NVENC-hang hazard cannot bite (and a monitor downgrades when +/// it could). Runs once per process; best-effort. +/// `PUNKTFUNK_GPU_PRIORITY_CLASS = off|normal|high|realtime|auto` (default auto; `high` = the +/// pre-gate static behavior; `realtime` = pinned, operator owns the hazard). Best-effort: +/// silently no-ops under a UAC-filtered token (the process will not hold SE_INC_BASE_PRIORITY, +/// so the D3DKMT call is a no-op). fn elevate_process_gpu_priority() { use std::sync::Once; static ONCE: Once = Once::new(); @@ -202,9 +224,15 @@ fn elevate_process_gpu_priority() { // `Once::call_once`; no raw pointers are dereferenced here. ONCE.call_once(|| unsafe { use windows::Win32::System::Threading::GetCurrentProcess; - let Some(prio) = configured_gpu_priority_class() else { - tracing::info!("GPU process scheduling priority class left at default (off)"); - return; + let prio = match configured_gpu_priority_mode() { + PrioMode::Off => { + tracing::info!("GPU process scheduling priority class left at default (off)"); + return; + } + PrioMode::Static(p) => p, + // Auto: HIGH is the immediately-safe floor; `auto_priority_gate` (running once a + // device exists, so it can see the adapter) decides the REALTIME upgrade. + PrioMode::Auto => 4, }; enable_inc_base_priority(); match d3dkmt_set_scheduling_priority_class(GetCurrentProcess(), prio) { @@ -220,3 +248,229 @@ fn elevate_process_gpu_priority() { } }); } + +// --- REALTIME auto-gate (gpu-contention §5.C / latency plan T2.3) -------------------------------- +// +// REALTIME GPU scheduling priority is the genuine cross-process ceiling-raiser under a saturating +// game (a higher-priority context preempts at pixel granularity — the Async-TimeWarp mechanism), +// and our SYSTEM service uniquely holds the SE_INC_BASE_PRIORITY it needs. The one documented +// hazard: REALTIME + NVIDIA + HAGS-on + near-full VRAM can hang NVENC. So: probe HAGS once via +// D3DKMT; HAGS off ⇒ REALTIME unconditionally; HAGS on ⇒ REALTIME gated on LOCAL-segment VRAM +// headroom, with a monitor thread that downgrades to HIGH the moment usage crosses +// [`VRAM_DOWNGRADE_PCT`] of the OS budget and restores REALTIME after it has stayed under +// [`VRAM_RESTORE_PCT`] for [`VRAM_RESTORE_TICKS`] consecutive polls (hysteresis against flapping +// on the boundary of the hazard window). + +/// Downgrade REALTIME→HIGH when local VRAM usage exceeds this share of the OS budget. +const VRAM_DOWNGRADE_PCT: u64 = 92; +/// Restore HIGH→REALTIME once usage has stayed at/below this share… +const VRAM_RESTORE_PCT: u64 = 85; +/// …for this many consecutive 2 s polls. +const VRAM_RESTORE_TICKS: u32 = 3; + +/// `KMTQAITYPE_WDDM_2_7_CAPS` — the adapter-info query that carries the HAGS (hardware GPU +/// scheduling) state. `D3DKMT_WDDM_2_7_CAPS` is a 4-byte bitfield: bit 0 `HwSchSupported`, +/// bit 1 `HwSchEnabled` (the one that matters — "is HAGS actually ON for this adapter"). +const KMTQAITYPE_WDDM_2_7_CAPS: u32 = 70; + +/// Probe whether HAGS (WDDM hardware scheduling) is ENABLED on the adapter with `luid`, via the +/// gdi32 D3DKMT surface (loaded by name — no stable windows-rs bindings, same as the priority +/// setter). `None` = could not determine (missing exports / query failed) — the caller treats +/// unknown as "assume the hazard exists". +/// +/// # Safety +/// Calls gdi32 exports through by-name transmuted pointers with locally built, correctly sized +/// `repr(C)` argument structs; the adapter handle is closed before returning on every path. +unsafe fn hags_enabled(luid: LUID) -> Option { + use windows::core::s; + use windows::Win32::System::LibraryLoader::{GetProcAddress, LoadLibraryA}; + #[repr(C)] + struct OpenFromLuid { + luid: LUID, + h_adapter: u32, + } + #[repr(C)] + struct CloseAdapter { + h_adapter: u32, + } + #[repr(C)] + struct QueryInfo { + h_adapter: u32, + ty: u32, + private_data: *mut std::ffi::c_void, + private_data_size: u32, + } + let gdi32 = LoadLibraryA(s!("gdi32.dll")).ok()?; + let open = GetProcAddress(gdi32, s!("D3DKMTOpenAdapterFromLuid"))?; + let query = GetProcAddress(gdi32, s!("D3DKMTQueryAdapterInfo"))?; + let close = GetProcAddress(gdi32, s!("D3DKMTCloseAdapter"))?; + type OpenFn = unsafe extern "system" fn(*mut OpenFromLuid) -> i32; + type QueryFn = unsafe extern "system" fn(*mut QueryInfo) -> i32; + type CloseFn = unsafe extern "system" fn(*mut CloseAdapter) -> i32; + let open: OpenFn = std::mem::transmute(open); + let query: QueryFn = std::mem::transmute(query); + let close: CloseFn = std::mem::transmute(close); + + let mut oa = OpenFromLuid { luid, h_adapter: 0 }; + if open(&mut oa) != 0 { + return None; + } + let mut caps: u32 = 0; + let mut qi = QueryInfo { + h_adapter: oa.h_adapter, + ty: KMTQAITYPE_WDDM_2_7_CAPS, + private_data: (&mut caps as *mut u32).cast(), + private_data_size: std::mem::size_of::() as u32, + }; + let st = query(&mut qi); + let mut ca = CloseAdapter { + h_adapter: oa.h_adapter, + }; + let _ = close(&mut ca); + if st != 0 { + return None; // pre-WDDM-2.7 driver: the query type doesn't exist ⇒ HAGS can't be on + } + Some(caps & 0x2 != 0) // bit 1 = HwSchEnabled +} + +/// Apply the auto-gate decision for `device`'s adapter (no-op unless the mode is `Auto`; runs +/// once per process). HAGS off ⇒ REALTIME now. HAGS on (or unknown) ⇒ spawn the VRAM monitor, +/// which flips REALTIME⇄HIGH on headroom. See the section comment above for the policy. +fn auto_priority_gate(device: &ID3D11Device) { + use std::sync::Once; + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + if !matches!(configured_gpu_priority_mode(), PrioMode::Auto) { + return; + } + // The adapter identity this device runs on. + let luid = match device + .cast::() + .and_then(|d| unsafe { d.GetAdapter() }) + .and_then(|a| unsafe { a.GetDesc() }) + { + Ok(desc) => desc.AdapterLuid, + Err(e) => { + tracing::warn!(error = %e, "REALTIME auto-gate: no adapter LUID — staying HIGH"); + return; + } + }; + // SAFETY: `hags_enabled` builds all its FFI arguments locally and closes the adapter + // handle before returning (see its own contract); `luid` is a plain value. + let hags = unsafe { hags_enabled(luid) }; + match hags { + Some(false) => { + // No HAGS ⇒ the NVENC-hang hazard cannot occur: take REALTIME outright. + // SAFETY: `GetCurrentProcess` returns the always-valid pseudo-handle; the setter + // loads gdi32 by name (its own contract). + let st = unsafe { + d3dkmt_set_scheduling_priority_class( + windows::Win32::System::Threading::GetCurrentProcess(), + 5, + ) + }; + match st { + Some(0) => tracing::info!( + "GPU priority REALTIME (auto: HAGS off — hang hazard not possible)" + ), + _ => { + tracing::warn!("REALTIME auto-gate: could not set REALTIME (staying HIGH)") + } + } + } + hags => { + let unknown = hags.is_none(); + tracing::info!( + hags_unknown = unknown, + "GPU priority auto-gate: HAGS on (or undeterminable) — REALTIME rides VRAM \ + headroom (monitor thread)" + ); + spawn_vram_gate(luid); + } + } + }); +} + +/// The VRAM-headroom monitor (auto mode, HAGS on): flips the process class REALTIME⇄HIGH on the +/// LOCAL memory segment's usage-vs-budget, with hysteresis. Its own DXGI factory/adapter (COM +/// objects never cross threads); polling a 2 s cadence — VRAM exhaustion is a seconds-scale +/// process, and the downgrade only has to beat the *next* NVENC submission pile-up, not a frame. +fn spawn_vram_gate(luid: LUID) { + let _ = std::thread::Builder::new() + .name("pf-gpu-prio".into()) + .spawn(move || { + use windows::Win32::Graphics::Dxgi::{ + CreateDXGIFactory1, IDXGIAdapter3, IDXGIFactory4, DXGI_MEMORY_SEGMENT_GROUP_LOCAL, + DXGI_QUERY_VIDEO_MEMORY_INFO, + }; + use windows::Win32::System::Threading::GetCurrentProcess; + // SAFETY: plain DXGI object creation + LUID lookup; the COM objects are created on + // and confined to this thread. + let adapter: Option = unsafe { + CreateDXGIFactory1::() + .and_then(|f| f.EnumAdapterByLuid::(luid)) + .ok() + }; + let Some(adapter) = adapter else { + tracing::warn!("pf-gpu-prio: adapter lookup failed — staying HIGH"); + return; + }; + let mut realtime = false; // we start at the HIGH floor + let mut clean_ticks = 0u32; + loop { + // SAFETY: `adapter` is a live IDXGIAdapter3 owned by this thread; the query + // fills the local out-struct `mi`. + let mut mi = DXGI_QUERY_VIDEO_MEMORY_INFO::default(); + let info = unsafe { + adapter.QueryVideoMemoryInfo(0, DXGI_MEMORY_SEGMENT_GROUP_LOCAL, &mut mi) + }; + if info.is_ok() { + let (usage, budget) = (mi.CurrentUsage, mi.Budget); + if budget > 0 { + let pct = usage * 100 / budget; + if realtime && pct > VRAM_DOWNGRADE_PCT { + // SAFETY: pseudo-handle + by-name gdi32 call (setter's contract). + let st = unsafe { + d3dkmt_set_scheduling_priority_class(GetCurrentProcess(), 4) + }; + if st == Some(0) { + realtime = false; + clean_ticks = 0; + tracing::warn!( + vram_pct = pct, + "GPU priority REALTIME→HIGH (VRAM tightened — NVENC-hang \ + hazard window)" + ); + } + } else if !realtime && pct <= VRAM_RESTORE_PCT { + clean_ticks += 1; + if clean_ticks >= VRAM_RESTORE_TICKS { + // SAFETY: same setter contract as above. + let st = unsafe { + d3dkmt_set_scheduling_priority_class(GetCurrentProcess(), 5) + }; + if st == Some(0) { + realtime = true; + tracing::info!( + vram_pct = pct, + "GPU priority HIGH→REALTIME (auto: VRAM headroom \ + comfortable)" + ); + } else { + // Can't ever reach REALTIME (privilege) — stop burning polls. + tracing::info!( + "pf-gpu-prio: REALTIME unavailable — monitor exiting \ + (HIGH stands)" + ); + return; + } + } + } else if !realtime { + clean_ticks = 0; + } + } + } + std::thread::sleep(std::time::Duration::from_secs(2)); + } + }); +}