From d381cdf7f47124b72a4d646c600c86c8c7e3cfa3 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Thu, 16 Jul 2026 01:57:36 +0200 Subject: [PATCH] =?UTF-8?q?fix(host):=20NVENC=20open-failure=20resilience?= =?UTF-8?q?=20=E2=80=94=20backoff,=20failed-open=20hygiene,=20self-diagnos?= =?UTF-8?q?is?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field report (Linux direct NVENC): after a codec switch, every session open failed with NV_ENC_ERR_INVALID_VERSION until the host process was restarted — so the poisoned state is per-process, not a driver install issue. On-hardware investigation (RTX 5070 Ti, 610.43.03) could not reproduce it with clean codec cycles, dirty teardowns, or open/destroy storms, but established the failure class: the driver enforces a per-process concurrent-session cap (12 there, status INCOMPATIBLE_CLIENT_KEY; other branches report differently) whose exhaustion is exactly this signature — persistent open failures healed only by a process restart. Harden every path that can feed or mask that state: * Rebuild backoff: the in-place encoder-rebuild retries slept one frame interval, so all 5 attempts burned within ~40 ms at 120 Hz — no driver-side transient (deferred teardown of the previous session, engine reset) can clear that fast. Exponential backoff 100 ms → 1.6 s (~3 s total) so transients heal instead of killing the session. * Destroy-on-failed-open (Linux + Windows, all four open sites): the NVENC docs require NvEncDestroyEncoder even when OpenEncodeSessionEx FAILS — the driver may have allocated the session slot before erroring. Without it a retry burst against a transient leaks slots toward the cap, converting the transient into permanent exhaustion. * Teardown: a destroy_encoder failure (a session slot the driver may keep) is now logged with its status instead of silently discarded. * One-shot self-diagnosis on a failed session open (Linux): retry the raw open on a fresh dedicated CUDA context and log which of the three causes applies — shared-context poisoned (fresh works), driver-level skew/exhaustion/GPU loss (fresh fails the same way), or CUDA itself unhealthy (no fresh context) — so the next field report pinpoints the root cause with zero reporter effort. On-hardware regression tests (RTX box .21, all green): codec-switch reopen cycle (H265→AV1→H265→H264→H265), dirty teardown with in-flight encodes, and the full open-failure→diagnosis→in-place-recovery path via real session-cap exhaustion. Existing RFI/reconfigure/4:4:4 smokes still pass; clippy clean. Co-Authored-By: Claude Fable 5 --- .../src/encode/linux/nvenc_cuda.rs | 262 +++++++++++++++++- .../src/encode/windows/nvenc.rs | 47 +++- .../punktfunk-host/src/linux/zerocopy/cuda.rs | 39 +++ crates/punktfunk-host/src/punktfunk1.rs | 17 +- 4 files changed, 344 insertions(+), 21 deletions(-) diff --git a/crates/punktfunk-host/src/encode/linux/nvenc_cuda.rs b/crates/punktfunk-host/src/encode/linux/nvenc_cuda.rs index 0a677e6d..978e8e9b 100644 --- a/crates/punktfunk-host/src/encode/linux/nvenc_cuda.rs +++ b/crates/punktfunk-host/src/encode/linux/nvenc_cuda.rs @@ -318,6 +318,9 @@ pub struct NvencCudaEncoder { cursor: Option, cursor_tried: bool, cursor_serial: u64, + /// One-shot latch for [`diagnose_failed_open`](Self::diagnose_failed_open) so a rebuild-retry + /// burst (the session loop's bounded encoder resets) logs the diagnosis once, not per attempt. + diagnosed: bool, } // SAFETY: the `!Send` fields are the raw NVENC session handle (`encoder`), the shared `CUcontext` @@ -383,6 +386,7 @@ impl NvencCudaEncoder { cursor: None, cursor_tried: false, cursor_serial: u64::MAX, + diagnosed: false, inited: false, rfi_supported: false, custom_vbv: false, @@ -408,7 +412,16 @@ impl NvencCudaEncoder { for &bs in &self.bitstreams { let _ = (api().destroy_bitstream_buffer)(self.encoder, bs); } - let _ = (api().destroy_encoder)(self.encoder); + // A destroy failure means the driver may still hold this session's slot (the concurrent- + // session cap is per process and only a restart clears a leak) — make it visible instead + // of silently discarding the status. + if let Err(e) = (api().destroy_encoder)(self.encoder).nv_ok() { + tracing::warn!( + status = ?e, + "NVENC destroy_encoder failed at teardown — the driver may have leaked this \ + session's slot toward the concurrent-session cap" + ); + } self.ring.clear(); // drops the InputSurfaces, freeing their CUDA allocations self.bitstreams.clear(); self.pending.clear(); @@ -447,9 +460,19 @@ impl NvencCudaEncoder { ..Default::default() }; let mut enc: *mut c_void = ptr::null_mut(); - (api().open_encode_session_ex)(&mut params, &mut enc) - .nv_ok() - .map_err(|e| nvenc_status::call_err("open_encode_session_ex (caps probe)", e))?; + if let Err(e) = (api().open_encode_session_ex)(&mut params, &mut enc).nv_ok() { + // The NVENC docs require NvEncDestroyEncoder even after a FAILED open (the driver may + // have allocated the session slot before erroring) — without it, every failed open in + // a retry loop leaks a slot toward the concurrent-session cap, turning a transient + // failure into permanent exhaustion that only a host restart clears. + if !enc.is_null() { + let _ = (api().destroy_encoder)(enc); + } + return Err(nvenc_status::call_err( + "open_encode_session_ex (caps probe)", + e, + )); + } let wmax = self.get_cap(enc, nv::NV_ENC_CAPS::NV_ENC_CAPS_WIDTH_MAX); let hmax = self.get_cap(enc, nv::NV_ENC_CAPS::NV_ENC_CAPS_HEIGHT_MAX); let yuv444 = self.get_cap(enc, nv::NV_ENC_CAPS::NV_ENC_CAPS_SUPPORT_YUV444_ENCODE); @@ -489,6 +512,62 @@ impl NvencCudaEncoder { Ok(()) } + /// One-shot self-diagnosis for a failed session open (2026-07 field report: after a codec + /// switch every open returned `NV_ENC_ERR_INVALID_VERSION` until the HOST PROCESS was + /// restarted — so the poisoned state is per-process, not the driver install). Retries the raw + /// open on a FRESH dedicated CUDA context to split the candidate causes apart in the log: + /// * fresh context WORKS → the shared process context (or its NVENC association) is in a + /// bad state — a host bug to report; + /// * fresh context fails the SAME way → driver-level: userspace/kernel version skew, + /// concurrent-session-cap exhaustion (leaked sessions), or a lost/reset GPU; + /// * no fresh context AT ALL → CUDA itself is unhealthy in this process. + /// + /// Log-only (the caller still fails the open); latched per encoder so a reset burst logs once. + fn diagnose_failed_open(&mut self) { + if self.diagnosed { + return; + } + self.diagnosed = true; + let fresh = cuda::with_fresh_context(|ctx| { + let mut params = nv::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS { + version: nv::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER, + deviceType: nv::NV_ENC_DEVICE_TYPE::NV_ENC_DEVICE_TYPE_CUDA, + device: ctx, + apiVersion: nv::NVENCAPI_VERSION, + ..Default::default() + }; + let mut enc: *mut c_void = ptr::null_mut(); + // SAFETY: `params`/`enc` are live stack locals across the synchronous call; `ctx` is + // the live diagnostic context `with_fresh_context` just created. Any session the probe + // opened (even on a failed status, per the NVENC docs) is destroyed exactly once here. + unsafe { + let st = (api().open_encode_session_ex)(&mut params, &mut enc); + if !enc.is_null() { + let _ = (api().destroy_encoder)(enc); + } + st + } + }); + match fresh { + Ok(nv::NVENCSTATUS::NV_ENC_SUCCESS) => tracing::error!( + "NVENC self-diagnosis: the session opens FINE on a fresh CUDA context — the \ + host's shared CUDA context is in a bad state (host bug; please report this log)" + ), + Ok(st) => tracing::error!( + fresh_ctx_status = ?st, + "NVENC self-diagnosis: the open fails on a fresh CUDA context too — driver-level \ + cause: {}", + nvenc_status::explain(st) + ), + Err(e) => tracing::error!( + error = %format!("{e:#}"), + "NVENC self-diagnosis: could not create a fresh CUDA context — CUDA itself is \ + unhealthy in this process (GPU reset/fell off the bus, or a poisoned driver \ + state); a host restart should clear it" + ), + } + } + /// Author the session's `NV_ENC_CONFIG` at `bitrate` (bps): the P1/ULL preset (queried on /// `enc`) seeded with the RC/tier/chroma/VUI/DPB shape this backend always runs. ONE builder /// shared by [`try_open_session`] and [`Encoder::reconfigure_bitrate`], so an in-place rate @@ -680,9 +759,14 @@ impl NvencCudaEncoder { ..Default::default() }; let mut enc: *mut c_void = ptr::null_mut(); - (api().open_encode_session_ex)(&mut params, &mut enc) - .nv_ok() - .map_err(|e| nvenc_status::call_err("open_encode_session_ex", e))?; + if let Err(e) = (api().open_encode_session_ex)(&mut params, &mut enc).nv_ok() { + // Destroy-on-failed-open, as in `query_caps`: a failed open may still hold a session + // slot that must be released. + if !enc.is_null() { + let _ = (api().destroy_encoder)(enc); + } + return Err(nvenc_status::call_err("open_encode_session_ex", e)); + } let mut cfg = match self.build_config(enc, bitrate) { Ok(cfg) => cfg, @@ -717,7 +801,12 @@ impl NvencCudaEncoder { self.cu_ctx = cuda::context().context("shared CUDA context (Linux direct NVENC)")?; cuda::make_current().context("cuCtxSetCurrent (encode thread)")?; - self.query_caps()?; + if let Err(e) = self.query_caps() { + // The one place every session-open failure funnels through (the probe is the first + // open of any session) — run the one-shot self-diagnosis before propagating. + self.diagnose_failed_open(); + return Err(e); + } const FLOOR_BPS: u64 = 10_000_000; let requested_bps = self.bitrate_bps; // 2-way NVENC split-frame encoding (Ada dual-NVENC) above ~1 Gpix/s; env override @@ -1513,4 +1602,161 @@ mod tests { "negative first → decline" ); } + + fn open_h265() -> NvencCudaEncoder { + NvencCudaEncoder::open( + Codec::H265, + PixelFormat::Nv12, + 1280, + 720, + 60, + 20_000_000, + true, + 8, + ChromaFormat::Yuv420, + ) + .expect("open NVENC CUDA encoder") + } + + /// ON-HARDWARE: the codec-switch lifecycle from the 2026-07 field report ("switching the + /// codec leaves the host unable to bring the encoder up until a restart") — cycle sessions + /// across codecs in ONE process, clean drain per leg. Every leg must open and encode. Run: + /// cargo test -p punktfunk-host --features nvenc -- --ignored nvenc_cuda_codec_switch --nocapture + #[test] + #[ignore = "requires an NVIDIA GPU + driver — run manually on the RTX box (.21)"] + fn nvenc_cuda_codec_switch_reopen() { + const W: u32 = 1280; + const H: u32 = 720; + crate::zerocopy::cuda::make_current().expect("shared CUDA context current"); + for (leg, codec) in [ + Codec::H265, + Codec::Av1, + Codec::H265, + Codec::H264, + Codec::H265, + ] + .into_iter() + .enumerate() + { + let mut enc = NvencCudaEncoder::open( + codec, + PixelFormat::Nv12, + W, + H, + 60, + 20_000_000, + true, + 8, + ChromaFormat::Yuv420, + ) + .expect("open"); + for f in 0..4u32 { + let frame = nv12_frame(W, H, f); + enc.submit_indexed(&frame, f) + .unwrap_or_else(|e| panic!("leg {leg} {codec:?} submit failed: {e:#}")); + while enc.poll().expect("poll").is_some() {} + } + drop(enc); + } + println!("nvenc_cuda codec-switch: 5 legs across H265/AV1/H264, all clean"); + } + + /// ON-HARDWARE: dirty teardown — drop encoders with encodes still in flight (what a + /// mid-stream session kill does), several times, then a fresh session must still open. Guards + /// the teardown-with-pending path against driver-side session-slot leaks. + #[test] + #[ignore = "requires an NVIDIA GPU + driver — run manually on the RTX box (.21)"] + fn nvenc_cuda_dirty_teardown_reopen() { + const W: u32 = 1280; + const H: u32 = 720; + crate::zerocopy::cuda::make_current().expect("shared CUDA context current"); + for round in 0..3 { + let mut enc = open_h265(); + for f in 0..4u32 { + let frame = nv12_frame(W, H, f); + enc.submit_indexed(&frame, f) + .unwrap_or_else(|e| panic!("round {round} submit {f} failed: {e:#}")); + } + drop(enc); // teardown with 4 in-flight encodes + } + let mut enc = open_h265(); + let frame = nv12_frame(W, H, 0); + enc.submit_indexed(&frame, 0) + .expect("reopen after dirty teardowns"); + while enc.poll().expect("poll").is_some() {} + println!("nvenc_cuda dirty-teardown: 3 dirty drops, reopen clean"); + } + + /// ON-HARDWARE: the session-open failure path end to end — exhaust the driver's concurrent- + /// session cap with raw opens, assert a real encoder open fails with the actionable error + /// (and fires the one-shot self-diagnosis), then free the slots and assert the SAME encoder + /// rebuilds in place and produces an AU. This is the transient the session loop's rebuild + /// backoff is sized to outlive; on the RTX 5070 Ti (driver 610.43.03) the cap is 12 sessions + /// and the failure status is `NV_ENC_ERR_INCOMPATIBLE_CLIENT_KEY`. + #[test] + #[ignore = "requires an NVIDIA GPU + driver — run manually on the RTX box (.21)"] + fn nvenc_cuda_open_failure_diagnosis_and_recovery() { + const W: u32 = 1280; + const H: u32 = 720; + crate::zerocopy::cuda::make_current().expect("shared CUDA context current"); + try_api().expect("nvenc api"); + let shared = cuda::context().expect("shared ctx"); + + let open_raw = |device: *mut c_void| -> (nv::NVENCSTATUS, *mut c_void) { + let mut params = nv::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS { + version: nv::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER, + deviceType: nv::NV_ENC_DEVICE_TYPE::NV_ENC_DEVICE_TYPE_CUDA, + device, + apiVersion: nv::NVENCAPI_VERSION, + ..Default::default() + }; + let mut enc: *mut c_void = ptr::null_mut(); + // SAFETY: live params/out-param across the synchronous call; test-only. + let st = unsafe { (api().open_encode_session_ex)(&mut params, &mut enc) }; + (st, enc) + }; + + // Exhaust the concurrent-session cap. + let mut held = Vec::new(); + loop { + let (st, enc) = open_raw(shared); + if st != nv::NVENCSTATUS::NV_ENC_SUCCESS { + if !enc.is_null() { + // SAFETY: destroy the failed-open residue per the NVENC docs. + unsafe { + let _ = (api().destroy_encoder)(enc); + } + } + break; + } + held.push(enc); + } + assert!(!held.is_empty(), "expected a finite session cap"); + + // A real encoder open must now fail (lazy init → caps probe) with the actionable error. + let mut enc = open_h265(); + let frame = nv12_frame(W, H, 0); + let err = enc + .submit_indexed(&frame, 0) + .expect_err("submit must fail while the cap is exhausted"); + println!("at-cap error (self-diagnosis logged alongside): {err:#}"); + + // The transient clears (slots freed) → the SAME encoder rebuilds in place and encodes. + for e in held { + // SAFETY: e came from a successful raw open above; destroyed exactly once. + unsafe { + let _ = (api().destroy_encoder)(e); + } + } + assert!(enc.reset(), "in-place reset must be available"); + let frame = nv12_frame(W, H, 1); + enc.submit_indexed(&frame, 1) + .expect("rebuild after the transient cleared"); + let mut got = false; + while enc.poll().expect("poll").is_some() { + got = true; + } + assert!(got, "recovered encoder must produce an AU"); + println!("nvenc_cuda open-failure recovery: cap hit → diagnosed → recovered in place"); + } } diff --git a/crates/punktfunk-host/src/encode/windows/nvenc.rs b/crates/punktfunk-host/src/encode/windows/nvenc.rs index 769b44b8..f4d2bcbb 100644 --- a/crates/punktfunk-host/src/encode/windows/nvenc.rs +++ b/crates/punktfunk-host/src/encode/windows/nvenc.rs @@ -589,7 +589,16 @@ impl NvencD3d11Encoder { for &bs in &self.bitstreams { let _ = (api().destroy_bitstream_buffer)(self.encoder, bs); } - let _ = (api().destroy_encoder)(self.encoder); + // A destroy failure means the driver may still hold this session's slot (the concurrent- + // session cap is per process and only a restart clears a leak) — make it visible instead + // of silently discarding the status. + if let Err(e) = (api().destroy_encoder)(self.encoder).nv_ok() { + tracing::warn!( + status = ?e, + "NVENC destroy_encoder failed at teardown — the driver may have leaked this \ + session's slot toward the concurrent-session cap" + ); + } // Return this session's units to the budget (see LIVE_SESSION_UNITS). LIVE_SESSION_UNITS.fetch_sub(self.session_units, std::sync::atomic::Ordering::Relaxed); self.session_units = 0; @@ -637,9 +646,19 @@ impl NvencD3d11Encoder { ..Default::default() }; let mut enc: *mut c_void = ptr::null_mut(); - (api().open_encode_session_ex)(&mut params, &mut enc) - .nv_ok() - .map_err(|e| nvenc_status::call_err("open_encode_session_ex (caps probe)", e))?; + if let Err(e) = (api().open_encode_session_ex)(&mut params, &mut enc).nv_ok() { + // The NVENC docs require NvEncDestroyEncoder even after a FAILED open (the driver may + // have allocated the session slot before erroring) — without it, every failed open in + // a retry loop leaks a slot toward the concurrent-session cap, turning a transient + // failure into permanent exhaustion that only a host restart clears. + if !enc.is_null() { + let _ = (api().destroy_encoder)(enc); + } + return Err(nvenc_status::call_err( + "open_encode_session_ex (caps probe)", + e, + )); + } let wmax = self.get_cap(enc, nv::NV_ENC_CAPS::NV_ENC_CAPS_WIDTH_MAX); let hmax = self.get_cap(enc, nv::NV_ENC_CAPS::NV_ENC_CAPS_HEIGHT_MAX); let ten_bit = self.get_cap(enc, nv::NV_ENC_CAPS::NV_ENC_CAPS_SUPPORT_10BIT_ENCODE); @@ -948,9 +967,14 @@ impl NvencD3d11Encoder { ..Default::default() }; let mut enc: *mut c_void = ptr::null_mut(); - (api().open_encode_session_ex)(&mut params, &mut enc) - .nv_ok() - .map_err(|e| nvenc_status::call_err("open_encode_session_ex", e))?; + if let Err(e) = (api().open_encode_session_ex)(&mut params, &mut enc).nv_ok() { + // Destroy-on-failed-open, as in `query_caps`: a failed open may still hold a session + // slot that must be released. + if !enc.is_null() { + let _ = (api().destroy_encoder)(enc); + } + return Err(nvenc_status::call_err("open_encode_session_ex", e)); + } let mut cfg = match self.build_config(enc, bitrate) { Ok(cfg) => cfg, @@ -1764,8 +1788,9 @@ pub fn probe_can_encode_444(codec: Codec) -> bool { // `EnumAdapterByLuid` return owned COM objects or err (→ default-adapter fallback). // `D3D11CreateDevice` (explicit adapter + UNKNOWN driver type, or NULL adapter + HARDWARE) // fills `device` or returns Err (→ false). `open_encode_session_ex` opens an NVENC session - // against that device's raw pointer (valid while `device` is held) or errors (→ false, tearing - // nothing down). `get_encode_caps` reads one scalar cap into `val` via the loaded API table. + // against that device's raw pointer (valid while `device` is held) or errors (→ false, after + // destroying any residue session the failed open left — the docs require it). + // `get_encode_caps` reads one scalar cap into `val` via the loaded API table. // `destroy_encoder` frees the session exactly once; `device`/its context drop with the COM // wrappers. No handle escapes this call and nothing runs concurrently. unsafe { @@ -1818,6 +1843,10 @@ pub fn probe_can_encode_444(codec: Codec) -> bool { .nv_ok() .is_err() { + // Destroy-on-failed-open: a failed open may still hold a session slot. + if !enc.is_null() { + let _ = (api().destroy_encoder)(enc); + } return false; } let mut param = nv::NV_ENC_CAPS_PARAM { diff --git a/crates/punktfunk-host/src/linux/zerocopy/cuda.rs b/crates/punktfunk-host/src/linux/zerocopy/cuda.rs index 0a24c58c..70f53445 100644 --- a/crates/punktfunk-host/src/linux/zerocopy/cuda.rs +++ b/crates/punktfunk-host/src/linux/zerocopy/cuda.rs @@ -120,6 +120,7 @@ struct CudaApi { cuInit: unsafe extern "C" fn(c_uint) -> CUresult, cuDeviceGet: unsafe extern "C" fn(*mut CUdevice, c_int) -> CUresult, cuCtxCreate_v2: unsafe extern "C" fn(*mut CUcontext, c_uint, CUdevice) -> CUresult, + cuCtxDestroy_v2: unsafe extern "C" fn(CUcontext) -> CUresult, cuCtxSetCurrent: unsafe extern "C" fn(CUcontext) -> CUresult, cuMemAllocPitch_v2: unsafe extern "C" fn(*mut CUdeviceptr, *mut usize, usize, usize, c_uint) -> CUresult, @@ -214,6 +215,7 @@ fn cuda_api() -> Option<&'static CudaApi> { cuInit: *lib.get(b"cuInit\0").ok()?, cuDeviceGet: *lib.get(b"cuDeviceGet\0").ok()?, cuCtxCreate_v2: *lib.get(b"cuCtxCreate_v2\0").ok()?, + cuCtxDestroy_v2: *lib.get(b"cuCtxDestroy_v2\0").ok()?, cuCtxSetCurrent: *lib.get(b"cuCtxSetCurrent\0").ok()?, cuMemAllocPitch_v2: *lib.get(b"cuMemAllocPitch_v2\0").ok()?, cuMemFree_v2: *lib.get(b"cuMemFree_v2\0").ok()?, @@ -275,6 +277,12 @@ unsafe fn cuCtxCreate_v2(pctx: *mut CUcontext, flags: c_uint, dev: CUdevice) -> None => CU_ERROR_NOT_LOADED, } } +unsafe fn cuCtxDestroy_v2(ctx: CUcontext) -> CUresult { + match cuda_api() { + Some(a) => (a.cuCtxDestroy_v2)(ctx), + None => CU_ERROR_NOT_LOADED, + } +} unsafe fn cuCtxSetCurrent(ctx: CUcontext) -> CUresult { match cuda_api() { Some(a) => (a.cuCtxSetCurrent)(ctx), @@ -610,6 +618,37 @@ pub fn make_current() -> Result<()> { unsafe { ck(cuCtxSetCurrent(ctx), "cuCtxSetCurrent") } } +/// DIAGNOSTIC-ONLY: create a fresh dedicated context on device 0, run `probe` with it, destroy it, +/// and restore the shared context as current. Used by the NVENC backend's session-open +/// self-diagnosis to split "this process's shared context is in a bad state" (probe succeeds on +/// the fresh context) from a driver-level condition like version skew or session exhaustion +/// (probe fails there too). Never used on a hot path. +pub fn with_fresh_context(probe: impl FnOnce(CUcontext) -> R) -> Result { + if cuda_api().is_none() { + bail!("libcuda.so.1 not available"); + } + // SAFETY: the driver table is present (checked above). `cuInit(0)` is idempotent. `&mut dev`/ + // `&mut ctx` are live, distinct stack out-params for their synchronous calls. On success `ctx` + // is a valid dedicated context, destroyed exactly once below; the shared context is restored + // as current afterwards (creation left the fresh one current on this thread). + unsafe { + ck(cuInit(0), "cuInit")?; + let mut dev: CUdevice = 0; + ck(cuDeviceGet(&mut dev, 0), "cuDeviceGet")?; + let mut ctx: CUcontext = std::ptr::null_mut(); + ck( + cuCtxCreate_v2(&mut ctx, CU_CTX_SCHED_BLOCKING_SYNC, dev), + "cuCtxCreate_v2 (diagnostic)", + )?; + let r = probe(ctx); + let _ = cuCtxDestroy_v2(ctx); + if let Some(c) = CONTEXT.get() { + let _ = cuCtxSetCurrent(c.0); + } + Ok(r) + } +} + thread_local! { /// Per-thread copy stream. `None` until first use; `Some(null)` means "creation failed, use the /// default (NULL) stream". Per-thread (not shared) so each worker's `cuStreamSynchronize` waits diff --git a/crates/punktfunk-host/src/punktfunk1.rs b/crates/punktfunk-host/src/punktfunk1.rs index 8fa0b8ab..c5117fbd 100644 --- a/crates/punktfunk-host/src/punktfunk1.rs +++ b/crates/punktfunk-host/src/punktfunk1.rs @@ -4935,10 +4935,19 @@ fn virtual_stream(ctx: SessionContext) -> Result<()> { max = MAX_ENCODER_RESETS, "encoder submit failed — encoder rebuilt in place, forcing an IDR"); last_au_at = std::time::Instant::now(); - // Re-pace from the rebuild and retry this frame next tick (gives the fresh encoder - // one frame period to come up instead of hammering it in a hot loop). - next = std::time::Instant::now() + interval; - std::thread::sleep(interval); + // Back off exponentially between rebuild attempts (100 ms → 1.6 s, ~3 s total across + // the reset budget). One frame period is NOT enough: a 2026-07 field report showed all + // 5 resets burning within 40 ms at 120 Hz against a driver-side condition (NVENC + // session open failing after a codec switch) that no 8 ms retry could outlive — any + // transient like the previous session's deferred driver teardown needs real time. A + // genuinely dead encoder now costs ~3 s before the session ends with the terminal + // error, which the client's stall UI already covers. + let backoff = std::cmp::max( + interval, + std::time::Duration::from_millis(100u64 << (encoder_resets - 1).min(4)), + ); + next = std::time::Instant::now() + backoff; + std::thread::sleep(backoff); continue; } let submit_us = if measure {