From 5b3ea6e8dbcf4b851251fb16b7d991b8d99d53e7 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Mon, 10 Aug 2026 07:46:47 +0200 Subject: [PATCH] fix(pf-encode): one NVENC open failure could kill every session on the box MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `punktfunk-host` died twice on home-nobara-1 with the same stack: __strlen_evex <- av_vbprintf <- format_line <- av_log_default_callback <- ff_cuda_check <- ff_nvenc_encode_init <- avcodec_open2 <- NvencEncoder::open <- NvencEncoder::reset <- virtual_stream once as an outright SIGSEGV mid-session, and once as a thread wedged in that stack so the service never answered SIGTERM and systemd escalated to SIGABRT ("State 'stop-sigterm' timed out. Aborting."). Both times a client's session was rebuilding its encoder. The blast radius is the whole host process — every other client's session goes with it. The fault is in libav, not here. `ff_cuda_check` logs the failing CUDA call as `"%s failed -> %s: %s"` using an `err_name`/`err_string` pair the error lookup does not always fill, and glibc then walks whatever was on the stack. We cannot patch the distro's FFmpeg, so the fix denies it the chance to format: the guard already used by the 4:4:4 probe drops the level to AV_LOG_FATAL across the open, and `av_log_default_callback` returns on the level check before `format_line` — these messages are AV_LOG_ERROR. The failure is not swallowed; it still comes back as `Err(e)` and is reported with our own context, which now says the libav text was deliberately silenced so nobody hunts for a message that will not come. Scoped to the `open_with` call ALONE. The ENOSYS arm immediately below recurses into `Self::open`, and `QuietLibavLog` holds a non-reentrant global mutex — wrapping the whole `match` would have deadlocked the intra-refresh retry. Verified on home-nobara-1 (fc44, libavcodec 62). With CUDA made unavailable so the open fails inside the CUDA layer, the old binary prints [hevc_nvenc @ ..] cuInit(0) failed -> CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected — that line IS `ff_cuda_check` formatting the two `%s` — and the fixed binary does not; both exit 1 with our error instead. A successful open is unaffected on both the direct-SDK and the libav paths (90/90 frames, identical output size). What this does NOT claim: the uninitialized-pointer condition itself was not reproduced on demand — it depends on the CUDA error lookup failing to fill the strings, and in the forced case above it filled them fine. What is demonstrated is that the formatting call which faulted is no longer reached during the open. --- crates/pf-encode/src/enc/linux/mod.rs | 43 +++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/crates/pf-encode/src/enc/linux/mod.rs b/crates/pf-encode/src/enc/linux/mod.rs index 192a994c..9149df81 100644 --- a/crates/pf-encode/src/enc/linux/mod.rs +++ b/crates/pf-encode/src/enc/linux/mod.rs @@ -525,7 +525,32 @@ impl NvencEncoder { None => {} } - let enc = match video.open_with(opts) { + // libav's OWN failure path can take the whole host down with it. When NVENC init fails, + // `ff_nvenc_encode_init` calls `ff_cuda_check`, which hands `av_log` an `err_name`/ + // `err_string` pair it did not initialize when the CUDA error lookup does not fill them — + // and glibc then walks that pointer in `strlen` inside `av_vbprintf`. Measured twice on + // home-nobara-1 (fc44, libavcodec 62), identical stack both times: + // + // __strlen_evex <- av_vbprintf <- format_line <- av_log_default_callback + // <- ff_cuda_check <- ff_nvenc_encode_init <- avcodec_open2 <- NvencEncoder::open + // + // once as an outright SIGSEGV mid-session, and once as a thread wedged in that stack so + // the service never answered SIGTERM and systemd SIGABRT'd it. Either way one encoder + // open failure kills every session on the box. + // + // We cannot fix the distro's FFmpeg, so deny it the chance to format: these messages are + // AV_LOG_ERROR, and `av_log_default_callback` returns on the level check before + // `format_line` when the level is AV_LOG_FATAL. The failure is NOT swallowed — it comes + // back as `Err(e)` below and is reported with our own context. + // + // Scoped to the call ALONE, deliberately: the ENOSYS arm below recurses into `Self::open`, + // and `QuietLibavLog` takes a non-reentrant global mutex — holding it across the match + // would deadlock the retry. + let opened = { + let _quiet = QuietLibavLog::new(); + video.open_with(opts) + }; + let enc = match opened { Ok(enc) => enc, // The GPU lacks NV_ENC_CAPS_SUPPORT_INTRA_REFRESH — ffmpeg fails the open with // ENOSYS ("Function not implemented"). Latch it (skip the doomed attempt on later @@ -560,9 +585,21 @@ impl NvencEncoder { ); } Err(e) => { + // libav's own message for this failure was suppressed on purpose (see above), so + // say so — otherwise the next person debugging an NVENC open wonders why the + // journal has our error and none of FFmpeg's. There is no env switch to get it + // back (the guard is unconditional, and it outranks PUNKTFUNK_FFMPEG_DEBUG for the + // duration of the call): to read libav's text, drop the guard in a local build. + // What it costs is one line of the shape + // [hevc_nvenc @ ..] cuInit(0) failed -> CUDA_ERROR_NO_DEVICE: no CUDA-capable .. + // and the AVERROR itself still travels in `e`. return Err(e).with_context(|| { - format!("open {name} ({width}x{height}@{fps}, {bitrate_bps} bps)") - }) + format!( + "open {name} ({width}x{height}@{fps}, {bitrate_bps} bps) — libav's own \ + diagnostic is silenced across this call because its CUDA error formatter \ + can fault the process" + ) + }); } }; if intra_refresh {