Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bf9386ecb2 | ||
|
|
bf9fb3fb22 | ||
|
|
1cefd37603 | ||
|
|
25765c53ec | ||
|
|
36589c39da | ||
|
|
e3354b6d5d |
@@ -66,6 +66,11 @@ struct CudaHw {
|
||||
|
||||
impl CudaHw {
|
||||
/// Build a CUDA hwdevice wrapping `cu_ctx` and a frames pool (`sw_format` = `pixel`).
|
||||
///
|
||||
/// The `bail!`s below format raw AVERROR ints eagerly BY DESIGN — do not convert them to
|
||||
/// typed errors: `open_nvenc_probed`'s bitrate ladder steps down on a typed EINVAL
|
||||
/// (`nvenc_open_einval`), and a hwdevice/hwframes EINVAL is a config error no bitrate can
|
||||
/// fix — enrolling it would burn ~10 doomed encoder opens before surfacing the real failure.
|
||||
unsafe fn new(cu_ctx: *mut std::ffi::c_void, sw_format: Pixel, w: u32, h: u32) -> Result<Self> {
|
||||
let mut device_ref = ffi::av_hwdevice_ctx_alloc(ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA);
|
||||
if device_ref.is_null() {
|
||||
@@ -448,7 +453,17 @@ impl NvencEncoder {
|
||||
let pix_rate = width as u64 * height as u64 * fps as u64;
|
||||
let split = std::env::var("PUNKTFUNK_SPLIT_ENCODE").ok();
|
||||
match split.as_deref() {
|
||||
Some(mode) => opts.set("split_encode_mode", mode),
|
||||
// The operator arm gains the codec gate the auto arm always had (Phase 8): split
|
||||
// "is not applicable to H264" per nvEncodeAPI.h, and h264_nvenc has no such AVOption
|
||||
// — setting it would fail the open on a leftover dict entry.
|
||||
Some(mode) if matches!(codec, Codec::H265 | Codec::Av1) => {
|
||||
opts.set("split_encode_mode", mode)
|
||||
}
|
||||
Some(_) => tracing::warn!(
|
||||
codec = codec.nvenc_name(),
|
||||
"PUNKTFUNK_SPLIT_ENCODE ignored — split encoding is not applicable to H.264 \
|
||||
(nvEncodeAPI.h)"
|
||||
),
|
||||
None if matches!(codec, Codec::H265 | Codec::Av1)
|
||||
&& pix_rate >= super::SPLIT_FORCE_PIXEL_RATE =>
|
||||
{
|
||||
@@ -468,7 +483,15 @@ impl NvencEncoder {
|
||||
// sessions) and reopen this session without intra-refresh; any other failure — and
|
||||
// any failure when IR wasn't requested — propagates untouched (the bitrate probe
|
||||
// keys on EINVAL, which must not trip the latch).
|
||||
Err(e) if intra_refresh && format!("{e:#}").contains("Function not implemented") => {
|
||||
Err(e)
|
||||
if intra_refresh
|
||||
&& matches!(
|
||||
e,
|
||||
ffmpeg::Error::Other {
|
||||
errno: ffmpeg::util::error::ENOSYS
|
||||
}
|
||||
) =>
|
||||
{
|
||||
tracing::warn!(
|
||||
encoder = name,
|
||||
"NVENC intra-refresh not supported by this GPU — falling back to IDR-only \
|
||||
|
||||
@@ -62,8 +62,8 @@
|
||||
|
||||
use super::nvenc_core::{
|
||||
apply_low_latency_config, build_init_params, cached_ceiling, codec_guid, plan_range_recovery,
|
||||
resolve_slices, resolve_split_mode, resolve_subframe, store_ceiling, CeilingKey,
|
||||
LowLatencyConfig, NvStatusExt, RangePlan,
|
||||
resolve_slices, resolve_split_mode, resolve_split_subframe, resolve_subframe, store_ceiling,
|
||||
subframe_env_forced, CeilingKey, LowLatencyConfig, NvStatusExt, RangePlan,
|
||||
};
|
||||
use super::nvenc_status;
|
||||
use super::{AuChunk, ChromaFormat, Codec, EncodedFrame, Encoder, EncoderCaps};
|
||||
@@ -629,6 +629,10 @@ pub struct NvencCudaEncoder {
|
||||
/// [`subframe_cap`](Self::subframe_cap)); consumed by every `build_init_params` call so the
|
||||
/// open and the in-place reconfigure present identical init params.
|
||||
subframe_on: bool,
|
||||
/// Whether `PUNKTFUNK_NVENC_SUBFRAME=1` was EXPLICITLY forced — latched with `subframe_on`
|
||||
/// (same invariant: open and reconfigure must present identical init params, so no env
|
||||
/// re-reads after `query_caps`). Only consumed by [`resolve_split_subframe`]'s log severity.
|
||||
subframe_forced: bool,
|
||||
/// Sub-frame chunked poll armed for the live session (§7 LN1 Phase 1): multi-slice +
|
||||
/// sub-frame readback configured AND sync retrieve at init. See [`Encoder::poll_chunk`].
|
||||
subframe_chunks: bool,
|
||||
@@ -721,6 +725,7 @@ impl NvencCudaEncoder {
|
||||
slices: 1,
|
||||
subframe_cap: false,
|
||||
subframe_on: false,
|
||||
subframe_forced: false,
|
||||
subframe_chunks: false,
|
||||
chunk: None,
|
||||
})
|
||||
@@ -921,6 +926,7 @@ impl NvencCudaEncoder {
|
||||
// escapes.
|
||||
self.slices = resolve_slices(self.codec, 4);
|
||||
self.subframe_on = resolve_subframe(self.subframe_cap);
|
||||
self.subframe_forced = subframe_env_forced();
|
||||
tracing::info!(
|
||||
rfi = self.rfi_supported,
|
||||
custom_vbv = self.custom_vbv,
|
||||
@@ -1136,6 +1142,16 @@ impl NvencCudaEncoder {
|
||||
// [`resolve_split_mode`] for the precedence (env override / 10-bit / pixel rate).
|
||||
let pixel_rate = self.width as u64 * self.height as u64 * self.fps.max(1) as u64;
|
||||
let split_mode: u32 = resolve_split_mode(self.bit_depth, pixel_rate);
|
||||
// Split × sub-frame arbitration (Phase 8) BEFORE the ladder, the ceiling key and the
|
||||
// chunked-poll latch — all three must see the post-arbitration truth (a drop inside
|
||||
// build_init_params would leave poll_chunk busy-polling its whole budget per AU).
|
||||
let (split_mode, subframe_on) = resolve_split_subframe(
|
||||
self.codec,
|
||||
split_mode,
|
||||
self.subframe_on,
|
||||
self.subframe_forced,
|
||||
);
|
||||
self.subframe_on = subframe_on;
|
||||
const CLAMP_TOL_BPS: u64 = 20_000_000;
|
||||
|
||||
// Ceiling cache (process lifetime, `nvenc_core`): a prior clamp search already found
|
||||
|
||||
@@ -314,7 +314,9 @@ struct DeviceHold {
|
||||
instance_ci: Box<vk::InstanceCreateInfo<'static>>,
|
||||
_queue_prio: Box<[f32; 1]>,
|
||||
_queue_ci: Box<[vk::DeviceQueueCreateInfo<'static>; 1]>,
|
||||
_dev_exts: Box<[*const c_char; 3]>,
|
||||
// A plain Vec (not Box<[_; N]> like its siblings): Phase 8 pushes queue_family_foreign
|
||||
// conditionally. The heap buffer as_ptr() feeds device_ci is move-stable like the Boxes.
|
||||
_dev_exts: Vec<*const c_char>,
|
||||
_feat2: Box<vk::PhysicalDeviceFeatures2<'static>>,
|
||||
_v12: Box<vk::PhysicalDeviceVulkan12Features<'static>>,
|
||||
_v13: Box<vk::PhysicalDeviceVulkan13Features<'static>>,
|
||||
@@ -329,6 +331,9 @@ pub struct PyroWaveEncoder {
|
||||
ext_fd: ash::khr::external_memory_fd::Device,
|
||||
queue: vk::Queue,
|
||||
family: u32,
|
||||
/// `src` family for the fresh-dmabuf acquire barrier: FOREIGN when the extension is
|
||||
/// enabled, else the core EXTERNAL substitute (Phase 8 — see `open_inner`).
|
||||
foreign_qfi: u32,
|
||||
mem_props: vk::PhysicalDeviceMemoryProperties,
|
||||
_hold: DeviceHold,
|
||||
|
||||
@@ -372,6 +377,10 @@ pub struct PyroWaveEncoder {
|
||||
cmd_pool: vk::CommandPool,
|
||||
cmd: vk::CommandBuffer,
|
||||
fence: vk::Fence,
|
||||
/// True between a successful `queue_submit` and its successful fence wait — i.e. exactly when
|
||||
/// GPU work may still be executing. `reset()` keys its bounded wait on this: a never-submitted
|
||||
/// fence would otherwise read as "wedged" (fences start unsignaled).
|
||||
gpu_pending: bool,
|
||||
|
||||
// --- state ---
|
||||
width: u32,
|
||||
@@ -448,11 +457,11 @@ impl PyroWaveEncoder {
|
||||
instance_ci: Box::new(vk::InstanceCreateInfo::default()),
|
||||
_queue_prio: Box::new([1.0f32]),
|
||||
_queue_ci: Box::new([vk::DeviceQueueCreateInfo::default()]),
|
||||
_dev_exts: Box::new([
|
||||
_dev_exts: vec![
|
||||
ash::khr::external_memory_fd::NAME.as_ptr(),
|
||||
ash::ext::external_memory_dma_buf::NAME.as_ptr(),
|
||||
ash::ext::image_drm_format_modifier::NAME.as_ptr(),
|
||||
]),
|
||||
],
|
||||
_feat2: Box::new(vk::PhysicalDeviceFeatures2::default()),
|
||||
_v12: Box::new(vk::PhysicalDeviceVulkan12Features::default()),
|
||||
_v13: Box::new(vk::PhysicalDeviceVulkan13Features::default()),
|
||||
@@ -544,6 +553,28 @@ impl PyroWaveEncoder {
|
||||
hold._feat2.p_next = &mut *hold._v12 as *mut _ as *mut std::ffi::c_void;
|
||||
hold._v12.p_next = &mut *hold._v13 as *mut _ as *mut std::ffi::c_void;
|
||||
|
||||
// VK_EXT_queue_family_foreign (Phase 8): the fresh-import acquire barrier names
|
||||
// FOREIGN as src — enable the extension when advertised (`pf-presenter/dmabuf.rs`
|
||||
// precedent), else fall back to the core QUEUE_FAMILY_EXTERNAL substitute. Must be
|
||||
// pushed BEFORE the count/as_ptr wiring below.
|
||||
let dev_ext_props = instance
|
||||
.enumerate_device_extension_properties(pd)
|
||||
.unwrap_or_default();
|
||||
let foreign_qfi = if crate::vk_util::ext_advertised(
|
||||
&dev_ext_props,
|
||||
ash::ext::queue_family_foreign::NAME,
|
||||
) {
|
||||
hold._dev_exts
|
||||
.push(ash::ext::queue_family_foreign::NAME.as_ptr());
|
||||
vk::QUEUE_FAMILY_FOREIGN_EXT
|
||||
} else {
|
||||
tracing::warn!(
|
||||
"pyrowave: VK_EXT_queue_family_foreign not advertised — dmabuf acquires \
|
||||
use the core QUEUE_FAMILY_EXTERNAL substitute (no fleet hardware takes \
|
||||
this arm; report it)"
|
||||
);
|
||||
vk::QUEUE_FAMILY_EXTERNAL
|
||||
};
|
||||
hold._queue_ci[0] = vk::DeviceQueueCreateInfo::default().queue_family_index(family);
|
||||
hold._queue_ci[0].queue_count = 1;
|
||||
hold._queue_ci[0].p_queue_priorities = hold._queue_prio.as_ptr();
|
||||
@@ -556,9 +587,9 @@ impl PyroWaveEncoder {
|
||||
let device = instance
|
||||
.create_device(pd, &hold.device_ci, None)
|
||||
.context("create device")?;
|
||||
Ok((pd, family, device))
|
||||
Ok((pd, family, device, foreign_qfi))
|
||||
})();
|
||||
let (pd, family, device) = match selected {
|
||||
let (pd, family, device, foreign_qfi) = match selected {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
instance.destroy_instance(None);
|
||||
@@ -584,6 +615,7 @@ impl PyroWaveEncoder {
|
||||
ext_fd,
|
||||
queue,
|
||||
family,
|
||||
foreign_qfi,
|
||||
mem_props,
|
||||
_hold: hold,
|
||||
pw_dev: std::ptr::null_mut(),
|
||||
@@ -614,6 +646,7 @@ impl PyroWaveEncoder {
|
||||
cmd_pool: vk::CommandPool::null(),
|
||||
cmd: vk::CommandBuffer::null(),
|
||||
fence: vk::Fence::null(),
|
||||
gpu_pending: false,
|
||||
width: w,
|
||||
height: h,
|
||||
fps,
|
||||
@@ -1158,11 +1191,7 @@ impl PyroWaveEncoder {
|
||||
FramePayload::Dmabuf(d) => {
|
||||
let (img, view, fresh) = self.import_cached(d, frame.width, frame.height)?;
|
||||
let (old, src_qf, dst_qf) = if fresh {
|
||||
(
|
||||
vk::ImageLayout::UNDEFINED,
|
||||
vk::QUEUE_FAMILY_FOREIGN_EXT,
|
||||
self.family,
|
||||
)
|
||||
(vk::ImageLayout::UNDEFINED, self.foreign_qfi, self.family)
|
||||
} else {
|
||||
(
|
||||
vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL,
|
||||
@@ -1399,8 +1428,10 @@ impl PyroWaveEncoder {
|
||||
let _ = dev.reset_command_buffer(self.cmd, vk::CommandBufferResetFlags::empty());
|
||||
return Err(e);
|
||||
}
|
||||
self.gpu_pending = true;
|
||||
dev.wait_for_fences(&[self.fence], true, 5_000_000_000)
|
||||
.context("pyrowave encode fence")?;
|
||||
self.gpu_pending = false;
|
||||
|
||||
// ---- packetize ----
|
||||
// Dense (default): boundary = whole buffer → the AU is exactly one pyrowave packet.
|
||||
@@ -1492,8 +1523,35 @@ impl Encoder for PyroWaveEncoder {
|
||||
fn reset(&mut self) -> bool {
|
||||
// Cheap in-place rebuild: recreate only the pyrowave encoder object — there is no
|
||||
// rate-control history or reference state worth preserving (plan §4.3).
|
||||
// SAFETY: the device is idle for this encoder's work (submit waits its fence) and the
|
||||
// pyrowave device outlives the encoder object being swapped.
|
||||
//
|
||||
// Bounded wait first: the only work possibly still executing is the one submitted frame
|
||||
// whose synchronous fence wait timed out (`gpu_pending`). Re-wait it under the same 5 s
|
||||
// cap as `encode_frame` — an untimed `device_wait_idle` here would park the recovery
|
||||
// thread on the exact device it suspects is wedged, until the kernel's GPU reset, if
|
||||
// ever. If the fence still won't signal, destroying the pyrowave encoder under live GPU
|
||||
// work would be a use-after-free, so report "no in-place rebuild" and let the session
|
||||
// surface a real error (`Drop`'s unbounded idle covers teardown, where blocking on the
|
||||
// kernel is acceptable).
|
||||
if self.gpu_pending {
|
||||
// SAFETY: waiting this encoder's own fence under `&mut self`.
|
||||
if unsafe {
|
||||
self.device
|
||||
.wait_for_fences(&[self.fence], true, 5_000_000_000)
|
||||
}
|
||||
.is_err()
|
||||
{
|
||||
tracing::error!(
|
||||
"pyrowave: in-flight encode did not complete within the reset budget — GPU \
|
||||
or driver wedged; in-place rebuild abandoned"
|
||||
);
|
||||
self.pending.clear();
|
||||
return false;
|
||||
}
|
||||
self.gpu_pending = false;
|
||||
}
|
||||
// SAFETY: the device is idle for this encoder's work (the fence wait above, or no submit
|
||||
// outstanding) — this sweep-up is instant — and the pyrowave device outlives the encoder
|
||||
// object being swapped.
|
||||
unsafe {
|
||||
self.device.device_wait_idle().ok();
|
||||
pw::pyrowave_encoder_destroy(self.pw_enc);
|
||||
|
||||
@@ -7,6 +7,18 @@ use anyhow::Result;
|
||||
use ash::vk;
|
||||
use pf_frame::PixelFormat;
|
||||
|
||||
/// Whether a device extension is in an enumerated properties list — the gate both Vulkan encode
|
||||
/// backends use before enabling `VK_EXT_queue_family_foreign` (Phase 8: the FOREIGN queue-family
|
||||
/// barriers were used without the extension ever being enabled; `pf-presenter/dmabuf.rs` is the
|
||||
/// in-repo precedent that enables it).
|
||||
pub(super) fn ext_advertised(exts: &[vk::ExtensionProperties], name: &std::ffi::CStr) -> bool {
|
||||
exts.iter().any(|e| {
|
||||
// SAFETY: `extension_name` is a spec-guaranteed NUL-terminated UTF-8 byte array inside
|
||||
// the driver-filled `VkExtensionProperties` (VK_MAX_EXTENSION_NAME_SIZE bound).
|
||||
unsafe { std::ffi::CStr::from_ptr(e.extension_name.as_ptr()) == name }
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn color_range(layer: u32) -> vk::ImageSubresourceRange {
|
||||
vk::ImageSubresourceRange {
|
||||
aspect_mask: vk::ImageAspectFlags::COLOR,
|
||||
@@ -398,6 +410,24 @@ pub(crate) unsafe fn make_plain_image(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn ext_advertised_matches_exact_name() {
|
||||
let mut e = ash::vk::ExtensionProperties::default();
|
||||
let name = b"VK_EXT_queue_family_foreign\0";
|
||||
for (i, b) in name.iter().enumerate() {
|
||||
e.extension_name[i] = *b as std::ffi::c_char;
|
||||
}
|
||||
let exts = [ash::vk::ExtensionProperties::default(), e];
|
||||
assert!(super::ext_advertised(
|
||||
&exts,
|
||||
ash::ext::queue_family_foreign::NAME
|
||||
));
|
||||
assert!(!super::ext_advertised(
|
||||
&exts[..1],
|
||||
ash::ext::queue_family_foreign::NAME
|
||||
));
|
||||
}
|
||||
|
||||
use super::*;
|
||||
|
||||
/// CSC mode (`bgra_target = false`): the 3→4 expand is a pure byte shuffle — no channel
|
||||
|
||||
@@ -508,6 +508,12 @@ pub struct VulkanVideoEncoder {
|
||||
compute_queue: vk::Queue,
|
||||
encode_family: u32,
|
||||
compute_family: u32,
|
||||
/// The queue family the dmabuf-acquire barriers name as `src`: `QUEUE_FAMILY_FOREIGN_EXT`
|
||||
/// when `VK_EXT_queue_family_foreign` is advertised+enabled (Phase 8 — the barriers used it
|
||||
/// without the enable, tolerated by RADV), else the core-1.1 `QUEUE_FAMILY_EXTERNAL`
|
||||
/// conservative substitute (adds a same-driver precondition FOREIGN doesn't have — the
|
||||
/// pragmatic fallback for devices that were never valid targets before).
|
||||
foreign_qfi: u32,
|
||||
mem_props: vk::PhysicalDeviceMemoryProperties,
|
||||
|
||||
// --- codec ---
|
||||
@@ -540,7 +546,7 @@ pub struct VulkanVideoEncoder {
|
||||
// Per-buffer dmabuf-import cache, keyed by (st_dev, st_ino) — PipeWire cycles a small fixed pool,
|
||||
// so each underlying buffer is imported ONCE and reused (no per-frame VkImage create/import/destroy).
|
||||
// Imports are read-only per frame, so the ring shares them (concurrent frames read distinct buffers).
|
||||
import_cache: Vec<(u64, u64, vk::Image, vk::DeviceMemory, vk::ImageView)>,
|
||||
import_cache: Vec<CachedImport>,
|
||||
|
||||
// --- in-flight frame ring (pipelining) ---
|
||||
frames: Vec<Frame>, // per-slot private resources
|
||||
@@ -995,6 +1001,24 @@ impl VulkanVideoEncoder {
|
||||
} else {
|
||||
bitrate
|
||||
};
|
||||
// VK_EXT_queue_family_foreign: enable when advertised so the dmabuf-acquire barriers'
|
||||
// FOREIGN src family is spec-legal (Phase 8; `pf-presenter/dmabuf.rs` precedent). The
|
||||
// rgb probe's enumerate is a probe-local (and skipped on native-NV12), so this is a
|
||||
// fresh open-time query.
|
||||
let dev_ext_props = instance
|
||||
.enumerate_device_extension_properties(pd)
|
||||
.unwrap_or_default();
|
||||
let foreign_ok =
|
||||
crate::vk_util::ext_advertised(&dev_ext_props, ash::ext::queue_family_foreign::NAME);
|
||||
let foreign_qfi = if foreign_ok {
|
||||
vk::QUEUE_FAMILY_FOREIGN_EXT
|
||||
} else {
|
||||
tracing::warn!(
|
||||
"VK_EXT_queue_family_foreign not advertised — dmabuf acquires use the core \
|
||||
QUEUE_FAMILY_EXTERNAL substitute (this arm has no fleet hardware; report it)"
|
||||
);
|
||||
vk::QUEUE_FAMILY_EXTERNAL
|
||||
};
|
||||
// logical device: encode + compute queues + video extensions (AV1 ext name is raw — ash lacks it)
|
||||
let mut dev_exts = vec![
|
||||
ash::khr::video_queue::NAME.as_ptr(),
|
||||
@@ -1011,6 +1035,9 @@ impl VulkanVideoEncoder {
|
||||
if rgb_cfg.is_some() {
|
||||
dev_exts.push(vrgb::EXTENSION_NAME.as_ptr());
|
||||
}
|
||||
if foreign_ok {
|
||||
dev_exts.push(ash::ext::queue_family_foreign::NAME.as_ptr());
|
||||
}
|
||||
let prio = [1.0f32];
|
||||
let mut qcis = vec![vk::DeviceQueueCreateInfo::default()
|
||||
.queue_family_index(encode_family)
|
||||
@@ -1383,6 +1410,7 @@ impl VulkanVideoEncoder {
|
||||
encode_queue,
|
||||
compute_queue,
|
||||
encode_family,
|
||||
foreign_qfi,
|
||||
compute_family,
|
||||
mem_props,
|
||||
codec,
|
||||
@@ -1654,8 +1682,22 @@ impl VulkanVideoEncoder {
|
||||
// frame imports fresh (as before) but is still owned by the cache and freed on evict/Drop.
|
||||
(u64::MAX, self.enc_count)
|
||||
};
|
||||
if let Some(&(_, _, img, _, view)) = self.import_cache.iter().find(|e| (e.0, e.1) == key) {
|
||||
return Ok((img, view, false));
|
||||
if let Some(pos) = self.import_cache.iter().position(|e| e.key == key) {
|
||||
let e = &self.import_cache[pos];
|
||||
if e.extent == (cw, ch) {
|
||||
return Ok((e.img, e.view, false));
|
||||
}
|
||||
// Key hit, wrong extent: the (st_dev, st_ino) now names a different allocation than
|
||||
// the one imported. Unreachable while every submit path guards frame-vs-session
|
||||
// dimensions first — but the cache must never hand out a stale-sized image if a
|
||||
// future path forgets, so evict and fall through to a fresh import. In-flight frames
|
||||
// may still read the old image (same hazard as the FIFO eviction below), so idle the
|
||||
// device before destroying. At most once per reallocation.
|
||||
let _ = self.device.device_wait_idle();
|
||||
let e = self.import_cache.remove(pos);
|
||||
self.device.destroy_image_view(e.view, None);
|
||||
self.device.destroy_image(e.img, None);
|
||||
self.device.free_memory(e.mem, None);
|
||||
}
|
||||
// Feed pf-zerocopy's raw-dmabuf degrade latch (wired for the libav path in 3efbe416):
|
||||
// a deterministic import refusal repeats identically forever, and the latch — capture
|
||||
@@ -1690,12 +1732,18 @@ impl VulkanVideoEncoder {
|
||||
let _ = self.device.device_wait_idle();
|
||||
}
|
||||
while self.import_cache.len() >= IMPORT_CACHE_CAP {
|
||||
let (_, _, oi, om, ov) = self.import_cache.remove(0);
|
||||
self.device.destroy_image_view(ov, None);
|
||||
self.device.destroy_image(oi, None);
|
||||
self.device.free_memory(om, None);
|
||||
let e = self.import_cache.remove(0);
|
||||
self.device.destroy_image_view(e.view, None);
|
||||
self.device.destroy_image(e.img, None);
|
||||
self.device.free_memory(e.mem, None);
|
||||
}
|
||||
self.import_cache.push((key.0, key.1, img, mem, view));
|
||||
self.import_cache.push(CachedImport {
|
||||
key,
|
||||
extent: (cw, ch),
|
||||
img,
|
||||
mem,
|
||||
view,
|
||||
});
|
||||
// Fires once per distinct pool buffer then goes quiet in steady state — the signal the cache
|
||||
// is hitting (a per-frame log here would mean inode keying failed and we're re-importing).
|
||||
tracing::debug!(
|
||||
@@ -2006,6 +2054,20 @@ impl VulkanVideoEncoder {
|
||||
}
|
||||
|
||||
// ---- 2. RGB source -> compute_cmd: prep barriers + CSC + copy into nv12_src ----
|
||||
// The CSC twin of the sibling guards (native NV12, RGB-direct's two arms, every other
|
||||
// backend's submit): the shader samples the source with clamped 1:1 texelFetch, so a
|
||||
// mismatched frame doesn't fault — it silently streams a cropped/edge-padded picture.
|
||||
// Refuse it into the encoder-rebuild path instead.
|
||||
if frame.width != self.render_w || frame.height != self.render_h {
|
||||
bail!(
|
||||
"vulkan-encode (csc): frame {}x{} != mode {}x{} — refusing a mismatched CSC \
|
||||
source",
|
||||
frame.width,
|
||||
frame.height,
|
||||
self.render_w,
|
||||
self.render_h
|
||||
);
|
||||
}
|
||||
let dev = self.device.clone(); // cheap handle clone -> lets us also call &mut self helpers
|
||||
dev.begin_command_buffer(
|
||||
compute_cmd,
|
||||
@@ -2049,7 +2111,7 @@ impl VulkanVideoEncoder {
|
||||
let (old, src_qf, dst_qf) = if fresh {
|
||||
(
|
||||
vk::ImageLayout::UNDEFINED,
|
||||
vk::QUEUE_FAMILY_FOREIGN_EXT,
|
||||
self.foreign_qfi,
|
||||
self.compute_family,
|
||||
)
|
||||
} else {
|
||||
@@ -2398,7 +2460,7 @@ impl VulkanVideoEncoder {
|
||||
.dst_access_mask(vk::AccessFlags2::TRANSFER_READ)
|
||||
.old_layout(vk::ImageLayout::UNDEFINED)
|
||||
.new_layout(vk::ImageLayout::TRANSFER_SRC_OPTIMAL)
|
||||
.src_queue_family_index(vk::QUEUE_FAMILY_FOREIGN_EXT)
|
||||
.src_queue_family_index(self.foreign_qfi)
|
||||
.dst_queue_family_index(self.compute_family)
|
||||
} else {
|
||||
vk::ImageMemoryBarrier2::default()
|
||||
@@ -2939,7 +3001,7 @@ impl VulkanVideoEncoder {
|
||||
.src_stage_mask(vk::PipelineStageFlags2::NONE)
|
||||
.src_access_mask(vk::AccessFlags2::NONE)
|
||||
.old_layout(vk::ImageLayout::UNDEFINED)
|
||||
.src_queue_family_index(vk::QUEUE_FAMILY_FOREIGN_EXT)
|
||||
.src_queue_family_index(self.foreign_qfi)
|
||||
.dst_queue_family_index(self.encode_family),
|
||||
SrcAcquire::DmabufCached => src_base
|
||||
.src_stage_mask(vk::PipelineStageFlags2::NONE)
|
||||
@@ -3794,9 +3856,58 @@ impl Encoder for VulkanVideoEncoder {
|
||||
fn reset(&mut self) -> bool {
|
||||
// Abandon everything in flight: wait the GPU idle, discard unread slots + queued output, and
|
||||
// restart GOP/DPB state so the next frame is a fresh IDR.
|
||||
// SAFETY: `device_wait_idle` guarantees no slot's fence is still pending before we drop them.
|
||||
//
|
||||
// The wait is BOUNDED, like every other wait on this thread (`ENCODE_FENCE_TIMEOUT_NS`,
|
||||
// same reasoning): reset() runs precisely because something upstream looks wedged, and an
|
||||
// untimed `device_wait_idle` would park the recovery path on the suspect device until the
|
||||
// kernel's GPU reset — if that ever comes. A timeout means the GPU really is wedged:
|
||||
// report "no in-place rebuild" so the caller ends the session with a real error instead.
|
||||
// (`Drop` still waits unbounded — teardown must be memory-safe even against a wedged
|
||||
// device, and by then the session is already over.)
|
||||
let fences: Vec<vk::Fence> = self
|
||||
.in_flight
|
||||
.iter()
|
||||
.map(|&s| self.frames[s].fence)
|
||||
.collect();
|
||||
if !fences.is_empty() {
|
||||
// SAFETY: every in-flight slot's fence was submitted with its batch (`enqueue` pushes
|
||||
// only after a successful submit), and we hold `&mut self`.
|
||||
match unsafe {
|
||||
self.device
|
||||
.wait_for_fences(&fences, true, ENCODE_FENCE_TIMEOUT_NS)
|
||||
} {
|
||||
Ok(()) => {}
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
error = ?e,
|
||||
"vulkan-encode: in-flight work did not go idle within {} ms — GPU or \
|
||||
driver wedged; in-place rebuild abandoned",
|
||||
ENCODE_FENCE_TIMEOUT_NS / 1_000_000
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
// The fences cover the encode queue, and each encode waited its slot's `csc_sem`, so the
|
||||
// paired compute batches are done too. The residual — a compute batch whose encode-queue
|
||||
// submit failed (so it was never fenced or enqueued) — is what this sweep-up covers; it
|
||||
// is instant unless that rare orphan is itself wedged (then the kernel's GPU reset bounds
|
||||
// it, as before). An error here is a lost device: no rebuild on top of that.
|
||||
// SAFETY: waiting this encoder's own device idle under `&mut self`.
|
||||
if unsafe { self.device.device_wait_idle() }.is_err() {
|
||||
return false;
|
||||
}
|
||||
// Drop the dmabuf import cache while the device is provably idle (the only safe point
|
||||
// outside teardown): whatever wedged the GPU may be a capture-side reallocation, and the
|
||||
// rebuilt stream must re-import from live buffers rather than trust images imported for
|
||||
// the old pool.
|
||||
// SAFETY: device idle (waits above); each entry is owned by the cache, destroyed once.
|
||||
unsafe {
|
||||
let _ = self.device.device_wait_idle();
|
||||
for e in std::mem::take(&mut self.import_cache) {
|
||||
self.device.destroy_image_view(e.view, None);
|
||||
self.device.destroy_image(e.img, None);
|
||||
self.device.free_memory(e.mem, None);
|
||||
}
|
||||
}
|
||||
self.in_flight.clear();
|
||||
self.pending.clear();
|
||||
@@ -3857,6 +3968,17 @@ impl Encoder for VulkanVideoEncoder {
|
||||
}
|
||||
}
|
||||
|
||||
/// One cached dmabuf import (see `import_cached`): keyed by `(st_dev, st_ino)`, carrying the
|
||||
/// extent it was imported at — a key hit must also prove the cached image still matches the
|
||||
/// caller's frame — plus the owning Vulkan objects.
|
||||
struct CachedImport {
|
||||
key: (u64, u64),
|
||||
extent: (u32, u32),
|
||||
img: vk::Image,
|
||||
mem: vk::DeviceMemory,
|
||||
view: vk::ImageView,
|
||||
}
|
||||
|
||||
/// Every destructible Vulkan object the encoder owns, with the one `Drop` that destroys them in
|
||||
/// dependency order. Both teardown paths run through it so they cannot drift:
|
||||
///
|
||||
@@ -3874,7 +3996,7 @@ struct VkTeardown {
|
||||
// infallible), so device-level objects can only exist once both are `Some`.
|
||||
device: Option<ash::Device>,
|
||||
vq_dev: Option<ash::khr::video_queue::Device>,
|
||||
import_cache: Vec<(u64, u64, vk::Image, vk::DeviceMemory, vk::ImageView)>,
|
||||
import_cache: Vec<CachedImport>,
|
||||
frames: Vec<Frame>,
|
||||
compute_pool: vk::CommandPool,
|
||||
cmd_pool: vk::CommandPool,
|
||||
@@ -3933,10 +4055,10 @@ impl Drop for VkTeardown {
|
||||
unsafe {
|
||||
if let Some(device) = self.device.take() {
|
||||
let _ = device.device_wait_idle();
|
||||
for (_, _, img, mem, view) in std::mem::take(&mut self.import_cache) {
|
||||
device.destroy_image_view(view, None);
|
||||
device.destroy_image(img, None);
|
||||
device.free_memory(mem, None);
|
||||
for e in std::mem::take(&mut self.import_cache) {
|
||||
device.destroy_image_view(e.view, None);
|
||||
device.destroy_image(e.img, None);
|
||||
device.free_memory(e.mem, None);
|
||||
}
|
||||
// Per-frame ring resources (command buffers, descriptor sets freed with their pools).
|
||||
for f in std::mem::take(&mut self.frames) {
|
||||
|
||||
@@ -102,6 +102,141 @@ pub(super) fn resolve_split_mode(bit_depth: u8, pixel_rate: u64) -> u32 {
|
||||
mode
|
||||
}
|
||||
|
||||
/// Whether the operator EXPLICITLY forced sub-frame readback on (`PUNKTFUNK_NVENC_SUBFRAME=1`)
|
||||
/// — the log-severity input to [`resolve_split_subframe`]: a forced knob being overridden
|
||||
/// deserves a `warn`, a default being tuned an `info`. Callers LATCH this once next to their
|
||||
/// resolved subframe state (an env re-read at reconfigure would violate the "open and
|
||||
/// reconfigure present identical init params" invariant).
|
||||
/// Linux-cfg'd like its ONLY caller (the `nvenc_cuda` query_caps latch) — Windows sessions have
|
||||
/// `subframe == forced` by construction (env opt-in only) and never consult this; without the
|
||||
/// cfg it is dead code on every Windows leg (item-level dead_code, the recurring trap).
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) fn subframe_env_forced() -> bool {
|
||||
matches!(
|
||||
std::env::var("PUNKTFUNK_NVENC_SUBFRAME").as_deref(),
|
||||
Ok("1")
|
||||
)
|
||||
}
|
||||
|
||||
/// The split-encode × sub-frame arbitration (Phase 8; verified against `nvEncodeAPI.h`'s own
|
||||
/// `splitEncodeMode` doc, not folklore):
|
||||
/// - **H.264**: split "is not applicable" — hard-DISABLE the mode so the written config, the
|
||||
/// ceiling-cache key, the split diagnostic log and the rejection-retry all stay truthful (the
|
||||
/// retry used to re-open a byte-identical session after an H.264 "split rejection").
|
||||
/// - **HEVC**: split is "not supported if … subframe mode" — when WE force split
|
||||
/// (TWO/THREE/AUTO_FORCED, e.g. the 4K120 throughput requirement), sub-frame yields. Under
|
||||
/// plain AUTO the driver arbitrates — the shipped fleet state (1080p–1440p240 all run
|
||||
/// AUTO+subframe); keying on `!= DISABLE` here would have disarmed the Phase-3 chunked-poll
|
||||
/// feature fleet-wide.
|
||||
/// - **AV1**: both legal (sub-frame is per-tile; split is constrained only by
|
||||
/// output-into-vidmem, which we never use) — untouched.
|
||||
///
|
||||
/// Returns the `(split_mode, subframe)` to ACTUALLY configure. The caller must store BOTH back
|
||||
/// (the chunked-poll latch and `CeilingKey` key on them) — a silent in-params drop would leave
|
||||
/// `poll_chunk` busy-polling its full budget every AU (`numSlices` stays 0 without
|
||||
/// `reportSliceOffsets`, so neither loop exit ever fires).
|
||||
pub(super) fn resolve_split_subframe(
|
||||
codec: Codec,
|
||||
split_mode: u32,
|
||||
subframe: bool,
|
||||
subframe_forced: bool,
|
||||
) -> (u32, bool) {
|
||||
use nv::NV_ENC_SPLIT_ENCODE_MODE as M;
|
||||
if codec == Codec::H264 {
|
||||
return (M::NV_ENC_SPLIT_DISABLE_MODE as u32, subframe);
|
||||
}
|
||||
let split_forced = split_mode == M::NV_ENC_SPLIT_TWO_FORCED_MODE as u32
|
||||
|| split_mode == M::NV_ENC_SPLIT_THREE_FORCED_MODE as u32
|
||||
|| split_mode == M::NV_ENC_SPLIT_AUTO_FORCED_MODE as u32;
|
||||
if codec == Codec::H265 && split_forced && subframe {
|
||||
if subframe_forced {
|
||||
tracing::warn!(
|
||||
split_mode,
|
||||
"HEVC forced split-encode and PUNKTFUNK_NVENC_SUBFRAME=1 are mutually \
|
||||
unsupported (nvEncodeAPI.h) — sub-frame readback disabled for this session; \
|
||||
set PUNKTFUNK_SPLIT_ENCODE=0 to choose sub-frame instead"
|
||||
);
|
||||
} else {
|
||||
tracing::info!(
|
||||
split_mode,
|
||||
"HEVC forced split-encode supersedes default-on sub-frame readback (mutually \
|
||||
unsupported per nvEncodeAPI.h; split is the 4K120 throughput lever) — set \
|
||||
PUNKTFUNK_SPLIT_ENCODE=0 to choose sub-frame instead"
|
||||
);
|
||||
}
|
||||
return (split_mode, false);
|
||||
}
|
||||
(split_mode, subframe)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod split_subframe_tests {
|
||||
use super::{resolve_split_subframe, Codec};
|
||||
use nvidia_video_codec_sdk::sys::nvEncodeAPI::NV_ENC_SPLIT_ENCODE_MODE as M;
|
||||
|
||||
const AUTO: u32 = M::NV_ENC_SPLIT_AUTO_MODE as u32;
|
||||
const TWO: u32 = M::NV_ENC_SPLIT_TWO_FORCED_MODE as u32;
|
||||
const AUTO_F: u32 = M::NV_ENC_SPLIT_AUTO_FORCED_MODE as u32;
|
||||
const DISABLE: u32 = M::NV_ENC_SPLIT_DISABLE_MODE as u32;
|
||||
|
||||
/// THE FLEET CASE: plain AUTO + default-on sub-frame must pass through untouched — the
|
||||
/// driver arbitrates. Keying the rule on `!= DISABLE` would disarm sub-frame on every
|
||||
/// default Linux HEVC session (AUTO == 0 is the resolver's fallthrough).
|
||||
#[test]
|
||||
fn hevc_auto_keeps_subframe() {
|
||||
assert_eq!(
|
||||
resolve_split_subframe(Codec::H265, AUTO, true, false),
|
||||
(AUTO, true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hevc_forced_split_drops_subframe() {
|
||||
assert_eq!(
|
||||
resolve_split_subframe(Codec::H265, TWO, true, false),
|
||||
(TWO, false)
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_split_subframe(Codec::H265, AUTO_F, true, true),
|
||||
(AUTO_F, false)
|
||||
);
|
||||
// No sub-frame to drop → nothing changes.
|
||||
assert_eq!(
|
||||
resolve_split_subframe(Codec::H265, TWO, false, false),
|
||||
(TWO, false)
|
||||
);
|
||||
// Explicitly disabled split → sub-frame kept (the documented escape).
|
||||
assert_eq!(
|
||||
resolve_split_subframe(Codec::H265, DISABLE, true, true),
|
||||
(DISABLE, true)
|
||||
);
|
||||
}
|
||||
|
||||
/// H.264: split "is not applicable" (nvEncodeAPI.h) — hard-DISABLE regardless of the
|
||||
/// resolved mode; sub-frame (H.264 slices) is unaffected.
|
||||
#[test]
|
||||
fn h264_split_hard_disabled() {
|
||||
assert_eq!(
|
||||
resolve_split_subframe(Codec::H264, TWO, true, false),
|
||||
(DISABLE, true)
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_split_subframe(Codec::H264, AUTO, false, false),
|
||||
(DISABLE, false)
|
||||
);
|
||||
}
|
||||
|
||||
/// AV1: both features are legal together (per-tile sub-frame; split constrained only by
|
||||
/// output-into-vidmem) — the arbitration must not touch it.
|
||||
#[test]
|
||||
fn av1_untouched() {
|
||||
assert_eq!(
|
||||
resolve_split_subframe(Codec::Av1, TWO, true, true),
|
||||
(TWO, true)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// One session config's identity for the process-lifetime bitrate-ceiling cache
|
||||
/// ([`cached_ceiling`]/[`store_ceiling`]). Everything the driver's codec-level validation keys
|
||||
/// off: the GPU (different NVENC generations have different level ceilings), dims/fps (the luma
|
||||
|
||||
@@ -38,8 +38,8 @@
|
||||
|
||||
use super::nvenc_core::{
|
||||
apply_low_latency_config, build_init_params, cached_ceiling, codec_guid, plan_range_recovery,
|
||||
resolve_slices, resolve_split_mode, resolve_subframe, store_ceiling, CeilingKey,
|
||||
LowLatencyConfig, NvStatusExt, RangePlan,
|
||||
resolve_slices, resolve_split_mode, resolve_split_subframe, resolve_subframe, store_ceiling,
|
||||
CeilingKey, LowLatencyConfig, NvStatusExt, RangePlan,
|
||||
};
|
||||
use super::nvenc_status;
|
||||
use super::{ChromaFormat, Codec, EncodedFrame, Encoder, EncoderCaps};
|
||||
@@ -603,6 +603,11 @@ pub struct NvencD3d11Encoder {
|
||||
/// `reconfigure_bitrate` must present the SAME init params as the open (only the config's
|
||||
/// rate fields may move). Meaningless while `inited` is false.
|
||||
split_mode: u32,
|
||||
/// The session's arbitrated sub-frame state ([`resolve_split_subframe`] at open) — recorded
|
||||
/// so reconfigure presents EXACTLY the init params the session opened with (an env re-read
|
||||
/// there could flip `enableSubFrameWrite` mid-session, and would re-log the arbitration on
|
||||
/// every ABR retarget).
|
||||
subframe_on: bool,
|
||||
session_async: bool,
|
||||
/// The last reference-frame range we invalidated — dedupes repeated RFI requests for the same
|
||||
/// loss event (the client resends until it sees recovery).
|
||||
@@ -686,6 +691,7 @@ impl NvencD3d11Encoder {
|
||||
rfi_supported: false,
|
||||
custom_vbv: false,
|
||||
split_mode: nv::NV_ENC_SPLIT_ENCODE_MODE::NV_ENC_SPLIT_DISABLE_MODE as u32,
|
||||
subframe_on: false,
|
||||
session_async: false,
|
||||
last_rfi_range: None,
|
||||
init_device: ptr::null_mut(),
|
||||
@@ -980,6 +986,7 @@ impl NvencD3d11Encoder {
|
||||
bitrate: u64,
|
||||
split_mode: u32,
|
||||
enable_async: bool,
|
||||
subframe: bool,
|
||||
) -> Result<*mut c_void> {
|
||||
let mut params = nv::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS {
|
||||
version: nv::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER,
|
||||
@@ -1014,9 +1021,10 @@ impl NvencD3d11Encoder {
|
||||
&mut cfg,
|
||||
split_mode,
|
||||
enable_async,
|
||||
// Windows: env opt-in only ("1"), never a default — and build_init_params
|
||||
// additionally refuses it on an async session.
|
||||
resolve_subframe(false),
|
||||
// Windows: env opt-in only ("1"), never a default; arbitrated against split by the
|
||||
// caller (resolve_split_subframe) — and build_init_params additionally refuses it
|
||||
// on an async session.
|
||||
subframe,
|
||||
);
|
||||
|
||||
match (api().initialize_encoder)(enc, &mut init).nv_ok() {
|
||||
@@ -1069,6 +1077,11 @@ impl NvencD3d11Encoder {
|
||||
// The init-failure fallback below disables it if a codec/config rejects it.
|
||||
let pixel_rate = self.width as u64 * self.height as u64 * self.fps.max(1) as u64;
|
||||
let split_mode: u32 = resolve_split_mode(self.bit_depth, pixel_rate);
|
||||
// Split × sub-frame arbitration (Phase 8) before the ladder/ceiling key. On Windows
|
||||
// sub-frame is env-opt-in only, so resolved == forced by construction.
|
||||
let subframe_req = resolve_subframe(false);
|
||||
let (split_mode, subframe_req) =
|
||||
resolve_split_subframe(self.codec, split_mode, subframe_req, subframe_req);
|
||||
// Find the highest bitrate the GPU's codec LEVEL accepts and CLAMP to it. NVENC rejects
|
||||
// `initialize_encoder` (InvalidParam) when the bitrate exceeds the level ceiling (e.g. a
|
||||
// 1 Gbps request on HEVC). Strategy: try the requested rate; if the only problem is a forced
|
||||
@@ -1098,12 +1111,19 @@ impl NvencD3d11Encoder {
|
||||
}
|
||||
}
|
||||
|
||||
let mut probe = self.try_open_session(device, target_bps, split_mode, use_async);
|
||||
let mut probe =
|
||||
self.try_open_session(device, target_bps, split_mode, use_async, subframe_req);
|
||||
// The cache is advisory: a stale entry (driver change, identity collision) must not
|
||||
// wedge the open — retry the requested rate and let the search below rediscover.
|
||||
if probe.is_err() && target_bps < requested_bps {
|
||||
target_bps = requested_bps;
|
||||
probe = self.try_open_session(device, requested_bps, split_mode, use_async);
|
||||
probe = self.try_open_session(
|
||||
device,
|
||||
requested_bps,
|
||||
split_mode,
|
||||
use_async,
|
||||
subframe_req,
|
||||
);
|
||||
}
|
||||
// Disambiguate a forced-split rejection from a bitrate-cap rejection: retry once at the
|
||||
// requested rate with split disabled — if THAT succeeds, split was the problem, not bitrate.
|
||||
@@ -1117,7 +1137,9 @@ impl NvencD3d11Encoder {
|
||||
split_mode != nv::NV_ENC_SPLIT_ENCODE_MODE::NV_ENC_SPLIT_DISABLE_MODE as u32;
|
||||
if probe.is_err() && split_on {
|
||||
let no_split = nv::NV_ENC_SPLIT_ENCODE_MODE::NV_ENC_SPLIT_DISABLE_MODE as u32;
|
||||
if let Ok(e) = self.try_open_session(device, target_bps, no_split, use_async) {
|
||||
if let Ok(e) =
|
||||
self.try_open_session(device, target_bps, no_split, use_async, subframe_req)
|
||||
{
|
||||
tracing::warn!("NVENC: split-encode rejected by codec/config — disabled");
|
||||
used_split = no_split;
|
||||
probe = Ok(e);
|
||||
@@ -1144,7 +1166,13 @@ impl NvencD3d11Encoder {
|
||||
let mut best_bps = 0u64;
|
||||
while hi > lo + CLAMP_TOL_BPS {
|
||||
let mid = lo + (hi - lo) / 2;
|
||||
match self.try_open_session(device, mid, used_split, use_async) {
|
||||
match self.try_open_session(
|
||||
device,
|
||||
mid,
|
||||
used_split,
|
||||
use_async,
|
||||
subframe_req,
|
||||
) {
|
||||
Ok(e) => {
|
||||
if !best.is_null() {
|
||||
let _ = (api().destroy_encoder)(best);
|
||||
@@ -1169,12 +1197,23 @@ impl NvencD3d11Encoder {
|
||||
// trying split-disabled in case a forced split (not the bitrate) is the blocker.
|
||||
let no_split =
|
||||
nv::NV_ENC_SPLIT_ENCODE_MODE::NV_ENC_SPLIT_DISABLE_MODE as u32;
|
||||
best = match self.try_open_session(device, FLOOR_BPS, used_split, use_async)
|
||||
{
|
||||
best = match self.try_open_session(
|
||||
device,
|
||||
FLOOR_BPS,
|
||||
used_split,
|
||||
use_async,
|
||||
subframe_req,
|
||||
) {
|
||||
Ok(e) => e,
|
||||
Err(_) => {
|
||||
let e = self
|
||||
.try_open_session(device, FLOOR_BPS, no_split, use_async)
|
||||
.try_open_session(
|
||||
device,
|
||||
FLOOR_BPS,
|
||||
no_split,
|
||||
use_async,
|
||||
subframe_req,
|
||||
)
|
||||
.context(
|
||||
"NVENC initialize_encoder rejected even at the floor bitrate",
|
||||
)?;
|
||||
@@ -1200,6 +1239,7 @@ impl NvencD3d11Encoder {
|
||||
self.init_device_com = Some(device.clone());
|
||||
// Session init params a later `reconfigure_bitrate` must re-present verbatim.
|
||||
self.split_mode = used_split;
|
||||
self.subframe_on = subframe_req;
|
||||
self.session_async = use_async;
|
||||
// Session-budget accounting (Stage W3): record what this open holds so admission can
|
||||
// decline a parallel display the hardware can't afford. Weighted by the FINAL split
|
||||
@@ -1855,7 +1895,10 @@ impl Encoder for NvencD3d11Encoder {
|
||||
&mut cfg,
|
||||
self.split_mode,
|
||||
self.session_async,
|
||||
resolve_subframe(false),
|
||||
// The SESSION's recorded state, not a fresh env read: reconfigure must
|
||||
// present exactly the init params the open arbitrated (an env re-read could
|
||||
// flip enableSubFrameWrite mid-session — the Linux latch invariant).
|
||||
self.subframe_on,
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -897,13 +897,29 @@ fn open_nvenc_probed(
|
||||
}
|
||||
// EINVAL = above this GPU's level ceiling → step down. Any other failure (no GPU,
|
||||
// bad mode, OOM) is real — surface it rather than masking it with bitrate retries.
|
||||
Err(e) if format!("{e:#}").contains("Invalid argument") => last = Some(e),
|
||||
Err(e) if nvenc_open_einval(&e) => last = Some(e),
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
Err(last.unwrap_or_else(|| anyhow::anyhow!("encoder open failed at every probed bitrate")))
|
||||
}
|
||||
|
||||
/// Whether a libav NVENC open failed with EINVAL — the "bitrate above this GPU's level ceiling"
|
||||
/// signal [`open_nvenc_probed`]'s ladder steps down on. Typed: the root `ffmpeg::Error` survives
|
||||
/// the `anyhow` context chain, so match it there instead of substring-matching the English
|
||||
/// strerror rendering of the whole chain — which also fired on any OTHER wrapped EINVAL (e.g. a
|
||||
/// CUDA-context errno) and steered the ladder on failures that have nothing to do with bitrate.
|
||||
#[cfg(target_os = "linux")]
|
||||
fn nvenc_open_einval(e: &anyhow::Error) -> bool {
|
||||
use ffmpeg_next as ffmpeg;
|
||||
matches!(
|
||||
e.downcast_ref::<ffmpeg::Error>(),
|
||||
Some(ffmpeg::Error::Other {
|
||||
errno: ffmpeg::util::error::EINVAL
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether the direct-SDK NVENC path is active. **Default ON** — on-glass validated 2026-07-12:
|
||||
/// real RFI landed 73/73 as clean P-frame recovery anchors (never IDR) on an RTX host with a real
|
||||
/// Steam Deck client (design/linux-direct-nvenc.md §9). `PUNKTFUNK_NVENC_DIRECT=0` (also `false`/
|
||||
@@ -1798,6 +1814,38 @@ mod tests {
|
||||
assert_eq!(trait_fns, impl_fns);
|
||||
}
|
||||
|
||||
/// The typed-EINVAL classifier the bitrate ladder keys on (Phase 8): the `ffmpeg::Error`
|
||||
/// must survive `with_context` layers as a downcastable source — pinned here because the
|
||||
/// entire ladder's step-down behavior rests on it, and an eager `format!` anywhere between
|
||||
/// `open_with` and the ladder would silently break it (the ladder would stop stepping and
|
||||
/// 4K sessions would surface errors instead of degrading).
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn nvenc_open_einval_survives_context_layers() {
|
||||
use ffmpeg_next as ffmpeg;
|
||||
let e = anyhow::Error::from(ffmpeg::Error::Other {
|
||||
errno: ffmpeg::util::error::EINVAL,
|
||||
})
|
||||
.context("open hevc_nvenc (3840x2160@120, 400000000 bps)")
|
||||
.context("outer");
|
||||
assert!(nvenc_open_einval(&e));
|
||||
// ENOSYS (or any other errno) must not step the ladder.
|
||||
let e = anyhow::Error::from(ffmpeg::Error::Other {
|
||||
errno: ffmpeg::util::error::ENOSYS,
|
||||
})
|
||||
.context("open");
|
||||
assert!(!nvenc_open_einval(&e));
|
||||
}
|
||||
|
||||
/// The phrase WITHOUT the type no longer classifies — the fragility that was removed: the
|
||||
/// old string match trusted any error whose rendering contained "Invalid argument".
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn nvenc_open_einval_ignores_untyped_text() {
|
||||
let e = anyhow::anyhow!("driver said: Invalid argument (not a typed libav errno)");
|
||||
assert!(!nvenc_open_einval(&e));
|
||||
}
|
||||
|
||||
/// WP7.6: the resolver's full alias table, pinned. The panicking closure is the laziness
|
||||
/// contract — an explicit pref must resolve WITHOUT running the auto probe (`/serverinfo`
|
||||
/// polls through the mirrors; `gamestream/serverinfo.rs` litigated exactly that cost).
|
||||
|
||||
@@ -757,6 +757,16 @@ fn stream_body(
|
||||
// dead source can't loop forever — it ends the stream after the cap, falling back to a reconnect.
|
||||
const MAX_REBUILDS: u32 = 5;
|
||||
let mut rebuilds: u32 = 0;
|
||||
// Encode-stall recovery, the GameStream twin of the native path's ladder (native/stream.rs,
|
||||
// `reset_stalled_encoder`): a submit/poll failure or a silent stall rebuilds the encoder in
|
||||
// place — bounded — instead of ending the stream. The backends deliberately turn a wedged
|
||||
// GPU into a bounded error so the caller can do exactly this; without the ladder here, every
|
||||
// such error cost a Moonlight client a full disconnect/reconnect. `last_au_at` feeds the
|
||||
// silent-wedge watchdog below (backends whose non-blocking poll returns `None` forever
|
||||
// instead of erroring); every received AU clears the reset budget.
|
||||
const MAX_ENCODER_RESETS: u32 = 5;
|
||||
let mut encoder_resets: u32 = 0;
|
||||
let mut last_au_at = Instant::now();
|
||||
|
||||
// Coalesce forced keyframes. Under loss Moonlight spams IDR/RFI requests; on an encoder without
|
||||
// RFI (VAAPI/AMD — `supports_rfi=false`) each one becomes a full IDR, so an un-coalesced request
|
||||
@@ -866,9 +876,10 @@ fn stream_body(
|
||||
// Honor a client recovery request. Prefer reference-frame invalidation (the encoder
|
||||
// re-references an older still-valid frame — no costly IDR spike); if the encoder can't
|
||||
// invalidate (range too old, or no NVENC RFI) it returns false and we force a keyframe.
|
||||
// A prior pipeline drop needs a fresh keyframe to re-anchor the reference chain (see below).
|
||||
// A prior pipeline drop needs a fresh keyframe to re-anchor the reference chain (see
|
||||
// below). Consumed only when the keyframe is actually EMITTED (in the coalesce gate) —
|
||||
// read-and-clear here let the gate swallow the request for good.
|
||||
let mut want_keyframe = recover_after_drop;
|
||||
recover_after_drop = false;
|
||||
if let Some((first, last)) = rfi_range.lock().unwrap().take() {
|
||||
// Prefer reference-frame invalidation when the encoder supports it (no costly IDR
|
||||
// spike); otherwise — or if the range is too old to invalidate — fall back to a keyframe.
|
||||
@@ -898,12 +909,48 @@ fn stream_body(
|
||||
if emit {
|
||||
enc.request_keyframe();
|
||||
last_keyframe = Some(now);
|
||||
// A drop-recovery request is satisfied by an EMITTED keyframe, not by being
|
||||
// read: coalesced away it would be lost — never retried — leaving duplicate wire
|
||||
// indices in the encoder's reference table for a later RFI to anchor on (the
|
||||
// stale-anchor case rfi.rs exists to prevent). Keep it armed until this point.
|
||||
recover_after_drop = false;
|
||||
} else {
|
||||
tracing::debug!("video: keyframe request coalesced (IDR still in flight)");
|
||||
}
|
||||
}
|
||||
enc.submit_indexed(&frame, au_seq.wrapping_add(enc_inflight))
|
||||
.context("encoder submit")?;
|
||||
if let Err(e) = enc.submit_indexed(&frame, au_seq.wrapping_add(enc_inflight)) {
|
||||
// The input half of an encode stall (see native/stream.rs): rebuild the encoder in
|
||||
// place instead of ending the stream. A backend without an in-place rebuild
|
||||
// (`reset` = false) or an exhausted budget still fails the session, with the cause.
|
||||
encoder_resets += 1;
|
||||
if encoder_resets > MAX_ENCODER_RESETS || !enc.reset() {
|
||||
tracing::error!(
|
||||
error = %format!("{e:#}"),
|
||||
resets = encoder_resets,
|
||||
"encoder did not recover after repeated in-place rebuilds — ending the \
|
||||
stream (see the error above for the cause)"
|
||||
);
|
||||
return Err(e).context("encoder submit");
|
||||
}
|
||||
// The owed AUs died with the discarded encoder state; numbering restarts at `au_seq`,
|
||||
// and the rebuilt encoder's reference state is empty so the reused predictions meet
|
||||
// no stale bookkeeping (same reasoning as the capture rebuild above). The IDR
|
||||
// bypasses the coalesce gate: a rebuilt encoder MUST resync the client.
|
||||
enc_inflight = 0;
|
||||
enc.request_keyframe();
|
||||
last_keyframe = Some(Instant::now());
|
||||
last_au_at = Instant::now();
|
||||
tracing::warn!(error = %format!("{e:#}"), reset = encoder_resets,
|
||||
max = MAX_ENCODER_RESETS,
|
||||
"encoder submit failed — encoder rebuilt in place, forcing an IDR");
|
||||
// Real backoff between attempts, not a frame period: five instant retries burn out
|
||||
// inside one driver hiccup (the native ladder's 2026-07 field lesson).
|
||||
let backoff =
|
||||
frame_interval.max(Duration::from_millis(100u64 << (encoder_resets - 1).min(4)));
|
||||
next_frame = Instant::now() + backoff;
|
||||
std::thread::sleep(backoff);
|
||||
continue;
|
||||
}
|
||||
enc_inflight = enc_inflight.wrapping_add(1);
|
||||
let t_enc = tick.elapsed();
|
||||
|
||||
@@ -914,7 +961,19 @@ fn stream_body(
|
||||
// stamped with its wire frameIndex here (`au_seq + position`); the numbering only
|
||||
// ADVANCES if the batch is actually enqueued below (a dropped batch consumes none).
|
||||
let mut aus: Vec<(Vec<u8>, FrameType, u32)> = Vec::new();
|
||||
while let Some(au) = enc.poll().context("encoder poll")? {
|
||||
// A poll error is the output half of an encode stall (e.g. a bounded fence timeout from
|
||||
// a wedged GPU) — carry it to the shared stall recovery below, after the AUs already
|
||||
// drained are handed off, instead of killing the session outright.
|
||||
let mut poll_err: Option<anyhow::Error> = None;
|
||||
loop {
|
||||
let au = match enc.poll() {
|
||||
Ok(Some(au)) => au,
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
poll_err = Some(e);
|
||||
break;
|
||||
}
|
||||
};
|
||||
let ft = if au.keyframe {
|
||||
FrameType::Idr
|
||||
} else {
|
||||
@@ -923,6 +982,9 @@ fn stream_body(
|
||||
let idx = au_seq.wrapping_add(aus.len() as u32);
|
||||
aus.push((au.data, ft, idx));
|
||||
enc_inflight = enc_inflight.saturating_sub(1);
|
||||
// Every AU proves the encoder is alive.
|
||||
last_au_at = Instant::now();
|
||||
encoder_resets = 0;
|
||||
}
|
||||
let t_pkt = tick.elapsed();
|
||||
|
||||
@@ -951,6 +1013,37 @@ fn stream_body(
|
||||
}
|
||||
}
|
||||
}
|
||||
// Encode-stall recovery, the poll half (mirrors the native path's watchdog): an explicit
|
||||
// poll error, or no AU within the window while frames are owed — the silent wedge, where
|
||||
// a non-blocking poll returns `None` forever and nothing else ever errors. The window
|
||||
// scales with the frame interval so low-fps modes can't false-trip.
|
||||
let stall_window = Duration::from_secs(2).max(frame_interval * 8);
|
||||
if poll_err.is_some() || (enc_inflight > 0 && last_au_at.elapsed() >= stall_window) {
|
||||
let why = match &poll_err {
|
||||
Some(e) => format!("poll failed: {e:#}"),
|
||||
None => format!(
|
||||
"no AU for {} ms with {} frame(s) owed",
|
||||
last_au_at.elapsed().as_millis(),
|
||||
enc_inflight
|
||||
),
|
||||
};
|
||||
encoder_resets += 1;
|
||||
if encoder_resets > MAX_ENCODER_RESETS || !enc.reset() {
|
||||
return Err(poll_err.unwrap_or_else(|| anyhow::anyhow!("{why}")))
|
||||
.context("encoder stalled — in-place rebuild unavailable or exhausted");
|
||||
}
|
||||
enc_inflight = 0;
|
||||
enc.request_keyframe();
|
||||
last_keyframe = Some(Instant::now());
|
||||
last_au_at = Instant::now();
|
||||
tracing::warn!(reset = encoder_resets, max = MAX_ENCODER_RESETS, %why,
|
||||
"encode stall detected — encoder rebuilt in place, forcing an IDR");
|
||||
let backoff =
|
||||
frame_interval.max(Duration::from_millis(100u64 << (encoder_resets - 1).min(4)));
|
||||
next_frame = Instant::now() + backoff;
|
||||
std::thread::sleep(backoff);
|
||||
continue;
|
||||
}
|
||||
if measure {
|
||||
let t_send = tick.elapsed();
|
||||
let cap_us = t_cap.as_micros();
|
||||
|
||||
@@ -143,7 +143,8 @@ notes for context.
|
||||
| Setting | Values | Meaning |
|
||||
|---|---|---|
|
||||
| `PUNKTFUNK_GSO` | `1` · `0` | UDP Generic Segmentation Offload on the send path (coalesce a frame's packets into kernel super-buffers) — cuts send CPU ~30%, but its line-rate packet trains can cost delivered throughput on constrained links (measured on a 2.5GbE hop). Off by default until send pacing spaces the super-buffers; set `1` to opt in (auto-falls back to `sendmmsg` on kernels/paths without support). |
|
||||
| `PUNKTFUNK_SPLIT_ENCODE` | `0`/`disable` · `1`/`auto` · `2` · `3` | NVENC N-way split-encode for very high pixel rates (5K@240). `auto` picks automatically above ~1 Gpix/s. |
|
||||
| `PUNKTFUNK_SPLIT_ENCODE` | `0`/`disable` · `1`/`auto` · `2` · `3` | NVENC N-way split-encode for very high pixel rates (5K@240). `auto` picks automatically above ~1 Gpix/s. H.264 never splits (not applicable per the SDK); on HEVC a *forced* split disables sub-frame readback (mutually unsupported) — set `0` to choose sub-frame instead. |
|
||||
| `PUNKTFUNK_NVENC_SUBFRAME` | `0` · `1` | NVENC sub-frame (slice-level) readback for lower latency on sync sessions. Default: on where the GPU supports it (Linux direct NVENC). `0` = never; `1` = force. On HEVC it yields to a forced split-encode (the SDK documents the pair unsupported). |
|
||||
| `PUNKTFUNK_GPU_PRIORITY_CLASS` | `off` · `normal` · `high` · `realtime` · `auto` | **(Windows)** GPU scheduling priority for capture/encode under a GPU-saturating game. Default `auto` (starts `high`, upgrades to `realtime` when it's safe — e.g. HAGS off); `high` pins the static pre-gate behaviour; `realtime` is the strongest lever but can freeze NVENC on some setups. |
|
||||
| `PUNKTFUNK_IDD_DEPTH` | `N` (default `2`) | **(Windows)** IDD-push pipeline depth. `1` cuts latency once GPU priority is raised; higher smooths a contended GPU. |
|
||||
|
||||
|
||||
Reference in New Issue
Block a user