diff --git a/crates/pf-capture/src/linux/mod.rs b/crates/pf-capture/src/linux/mod.rs index 65e7c5b1..60e43bd8 100644 --- a/crates/pf-capture/src/linux/mod.rs +++ b/crates/pf-capture/src/linux/mod.rs @@ -168,6 +168,10 @@ pub struct PortalCapturer { /// downgrade ([`pf_zerocopy::note_raw_dmabuf_negotiation_failed`]) so the pipeline rebuild /// retries on the CPU offer instead of failing identically forever. vaapi_dmabuf: bool, + /// PW3: this capture's dmabuf offer has been confirmed to negotiate (a frame arrived), so the + /// negotiation retry budget has already been credited back. One-shot — the credit is per + /// capture, not per frame. + negotiation_confirmed: bool, /// This capture ran the HDR (10-bit PQ/BT.2020 dmabuf) offer — see [`Self::open`]'s /// `want_hdr`. Read by the negotiation-timeout diagnosis (a failed HDR offer latches the /// process-wide SDR downgrade) and by [`hdr_meta`](Capturer::hdr_meta). @@ -412,6 +416,7 @@ impl PwHandles { signals: self.signals, stall_since: None, vaapi_dmabuf: self.vaapi_dmabuf, + negotiation_confirmed: false, hdr_offer: self.hdr_offer, hdr_source, node_id, @@ -468,6 +473,13 @@ fn spawn_pipewire( } else { want_hdr }; + // PW3: tell the raw-dmabuf latch which capture this is BEFORE reading its verdict below. A + // different node id is a different question — a fresh virtual output, a compositor restart, + // the Bazzite Gaming↔Desktop switch — and inheriting "dmabuf does not work here" from an + // unrelated capture is how one transient timeout used to cost a host CPU capture until it was + // restarted. The portal bit is in the key because a portal-fd capture and a virtual-output + // capture with the same node number are genuinely different sources. + pf_zerocopy::note_raw_dmabuf_capture(u64::from(node_id) | (u64::from(fd.is_some()) << 32)); // THE negotiation decision, resolved once here and handed to the thread — no mirror (L3/F1). // Every environment/latch read the decision depends on happens at this single point. let plan = pipewire::negotiation_plan(pipewire::NegotiationInputs { @@ -705,6 +717,7 @@ impl PortalCapturer { // The slot before the wakeup: a publish that coalesced its edge (or landed while we were // not waiting) is still visible here. if let Some(f) = self.take_frame() { + self.note_negotiation_confirmed(); return Ok(f); } let slice = Duration::from_millis(500) @@ -728,6 +741,16 @@ impl PortalCapturer { self.slot.lock().ok().and_then(|mut s| s.take()) } + /// PW3: a frame arrived, so this capture's dmabuf-only offer DID negotiate — credit the + /// negotiation retry budget back. Only meaningful for a capture that actually made that offer, + /// and only once per capture (the budget counts consecutive failed BUILDS, not frames). + fn note_negotiation_confirmed(&mut self) { + if self.vaapi_dmabuf && !self.negotiation_confirmed { + self.negotiation_confirmed = true; + pf_zerocopy::note_raw_dmabuf_negotiation_ok(); + } + } + /// The [`frame_within`](Self::frame_within) budget expired (or the thread ended) — turn it /// into the diagnosis-bearing error. Split out of the slicing loop above; behavior unchanged. fn next_frame_timed_out( diff --git a/crates/pf-capture/src/linux/pipewire.rs b/crates/pf-capture/src/linux/pipewire.rs index 37b17757..b202d5ae 100644 --- a/crates/pf-capture/src/linux/pipewire.rs +++ b/crates/pf-capture/src/linux/pipewire.rs @@ -70,6 +70,17 @@ struct UserData { linear_nv12_failed: bool, /// Rate-limit counter for the latest-frame-only diagnostic log (see `.process`). dbg_log_n: u64, + /// PW4 step 1: the producer-fence wait distribution, measured on this (the PipeWire loop) + /// thread. Per-session, like the fall-through tally. + fence_wait: FenceWaitStats, + /// PW5 step 1: the negotiated buffer-pool depth, counted from `add_buffer`/`remove_buffer`. + /// See [`PoolCensus`] — this is the number that decides whether a deeper encode pipeline is + /// safe, and until now nobody had it. + pool: PoolCensus, + /// Raw-passthrough frames that silently fell through to the CPU de-pad path, by reason — see + /// [`PassthroughFallbacks`]. Per-session: a fresh `UserData` is built per pipeline, so a + /// compositor that starts serving dmabufs again after a rebuild gets a fresh log budget. + passthrough_fallbacks: PassthroughFallbacks, /// Cursor-as-metadata state, composited into the CPU de-pad path (see `consume_frame`). cursor: CursorState, /// `Some((w, h))` while the producer's negotiated size is a sacrificial birth mode and a @@ -239,6 +250,335 @@ impl NegotiationPlan { } } +/// Which capture arm a negotiated pipeline actually resolved to. +/// +/// The 2026-08-08 PyroWave triage had to reconstruct this from four files, because no single line +/// ever states it: the arm is the product of a policy, a latch, an importer that may or may not +/// have constructed, and a modifier list. A degraded host and a healthy one logged the same +/// thing. [`resolved_capture_arm`] plus the one INFO line at pipeline build is the whole fix. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum CaptureArm { + /// Raw dmabufs handed straight to the encoder, which imports them itself — libva (VAAPI) or + /// the PyroWave encoder's own Vulkan device. No host pixel touch. + DmabufPassthrough, + /// dmabufs imported to CUDA device buffers by the EGL→CUDA worker, for NVENC. No host pixel + /// touch either, but a different failure surface (the worker, the modifier negotiation). + CudaImport, + /// CPU frames: an mmap de-pad of every frame, then whatever CSC + upload the encoder needs. + /// The slow path — always a downgrade when the consumer could have taken a dmabuf. + Cpu, +} + +impl CaptureArm { + pub(super) fn as_str(self) -> &'static str { + match self { + CaptureArm::DmabufPassthrough => "dmabuf-passthrough", + CaptureArm::CudaImport => "cuda-import", + CaptureArm::Cpu => "cpu", + } + } +} + +/// Resolve the arm this pipeline ended up on. **Pure** — `have_importer` and `want_dmabuf` are the +/// two runtime facts `negotiation_plan` cannot know (whether the importer constructed, and what +/// modifier list that yielded); everything else is already in the plan. +pub(super) fn resolved_capture_arm( + plan: &NegotiationPlan, + have_importer: bool, + want_dmabuf: bool, +) -> CaptureArm { + if !want_dmabuf { + // No dmabuf offer at all: SHM/CPU frames, whatever the plan wanted. + CaptureArm::Cpu + } else if plan.vaapi_passthrough { + CaptureArm::DmabufPassthrough + } else if have_importer { + CaptureArm::CudaImport + } else { + // Unreachable via `want_dmabuf` (it requires `have_importer || vaapi_passthrough`), but + // stated rather than `unreachable!()`: a logging helper must never be the thing that + // panics a capture thread. + CaptureArm::Cpu + } +} + +/// Who consumes the captured frames — the fact that decides whether a CPU arm is a *downgrade* +/// worth warning about, and what to call it in the log. +/// +/// Derived from the resolved [`ZeroCopyPolicy`](crate::ZeroCopyPolicy), **not** from the encoder +/// pref: `pyrowave_session` is per-session (the negotiated codec), so a PyroWave session on an +/// otherwise-NVENC host reads as PyroWave here. Naming the pref instead is exactly how a PyroWave +/// session's CPU downgrade came to be reported as an NVENC one — or, on an NVIDIA host, not +/// reported at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ConsumerKind { + /// This session encodes PyroWave: the wavelet encoder's own Vulkan device imports dmabufs on + /// any vendor, so a CPU arm costs it the passthrough it was designed around. + PyroWave, + /// The VAAPI backend (AMD/Intel): libva imports the dmabuf and CSCs on the GPU. + Vaapi, + /// NVENC, fed by the EGL→CUDA importer. + Nvenc, + /// The software encoder — CPU frames are its native input, so a CPU arm is no downgrade. + Software, +} + +impl ConsumerKind { + pub(super) fn as_str(self) -> &'static str { + match self { + ConsumerKind::PyroWave => "pyrowave", + ConsumerKind::Vaapi => "vaapi", + ConsumerKind::Nvenc => "nvenc", + ConsumerKind::Software => "software", + } + } + + /// Whether landing on [`CaptureArm::Cpu`] is a performance downgrade for this consumer (i.e. + /// worth a `warn!`). True for every GPU consumer; false for the software encoder, which wants + /// CPU frames anyway. + pub(super) fn cpu_is_downgrade(self) -> bool { + !matches!(self, ConsumerKind::Software) + } +} + +/// Classify the frames' consumer. **Pure.** `pyrowave_session` wins over `backend_is_vaapi` +/// because it is the per-session truth and the pref is host-global (a PyroWave session also flips +/// `backend_is_vaapi` on, via `linux_zero_copy_is_vaapi`'s `Pyrowave` arm — so testing vaapi first +/// would swallow every PyroWave session). +pub(super) fn consumer_kind( + pyrowave_session: bool, + backend_is_vaapi: bool, + backend_is_gpu: bool, +) -> ConsumerKind { + if pyrowave_session { + ConsumerKind::PyroWave + } else if !backend_is_gpu { + ConsumerKind::Software + } else if backend_is_vaapi { + ConsumerKind::Vaapi + } else { + ConsumerKind::Nvenc + } +} + +/// Why a frame on the raw-dmabuf passthrough could not be handed to the encoder and fell through +/// to the CPU de-pad path instead. +/// +/// Each variant is a *different* diagnosis with a different fix, and all four were silent: the +/// passthrough block simply fell out of its `if` and the frame took the slow path, so a session +/// that had negotiated zero-copy could pay CPU costs on every frame while logging a healthy +/// "advertising DMA-BUF modifiers" line at open. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum PassthroughFallback { + /// No format negotiated yet — transient, and expected around a renegotiation. + NoFormat, + /// The producer delivered an SHM/MemFd buffer for this frame, not a dmabuf. + NotDmabuf, + /// The negotiated pixel format has no DRM fourcc, so it cannot be described to the encoder. + NoFourcc, + /// `F_DUPFD_CLOEXEC` failed — the fd could not be duplicated to outlive the buffer recycle + /// (an fd-limit symptom, not a graphics one). + DupFailed, +} + +impl PassthroughFallback { + fn bit(self) -> u8 { + match self { + PassthroughFallback::NoFormat => 1 << 0, + PassthroughFallback::NotDmabuf => 1 << 1, + PassthroughFallback::NoFourcc => 1 << 2, + PassthroughFallback::DupFailed => 1 << 3, + } + } + + pub(super) fn as_str(self) -> &'static str { + match self { + PassthroughFallback::NoFormat => "no format negotiated yet", + PassthroughFallback::NotDmabuf => "the producer delivered an SHM/MemFd buffer", + PassthroughFallback::NoFourcc => "the negotiated format has no DRM fourcc", + PassthroughFallback::DupFailed => "F_DUPFD_CLOEXEC failed on the dmabuf fd", + } + } + + /// What actually happens to the frame. Three of the four reasons downgrade it to the CPU + /// de-pad path; `NoFormat` does not — the CPU path needs `ud.format` too and returns without + /// de-padding, so that frame is DROPPED. Worth the distinction: "slower" and "gone" are + /// different faults, and a diagnostic that conflates them is the thing PW2 exists to remove. + pub(super) fn falls_back_to_cpu(self) -> bool { + !matches!(self, PassthroughFallback::NoFormat) + } + + /// What to do about it — the half a log line is useless without. + pub(super) fn hint(self) -> &'static str { + match self { + PassthroughFallback::NoFormat => { + "harmless if it stops: the first buffers can arrive before param_changed" + } + PassthroughFallback::NotDmabuf => { + "the compositor accepted the dmabuf offer and is serving memory anyway — check \ + PUNKTFUNK_FORCE_SHM and the compositor's allocator" + } + PassthroughFallback::NoFourcc => { + "a capture format the encoder path cannot describe — file it, the negotiation \ + should not have accepted it" + } + PassthroughFallback::DupFailed => "out of file descriptors — raise the host's NOFILE", + } + } +} + +/// Upper bounds (µs) of the fence-wait histogram's buckets; the last bucket is everything above. +/// +/// Deliberately coarse and log-ish. The question this instrument exists to answer is not "what is +/// the wait, to the microsecond" but "**is the tail ~0, or is it milliseconds?**" — the first +/// answer retires PW4 as a comment correction, the second justifies moving the wait off the +/// PipeWire loop thread. Bucket edges are placed so those two worlds cannot be confused: anything +/// at or below 100 µs is noise, anything past 1 ms is a real stall on a 60 Hz budget of 16.6 ms. +const FENCE_WAIT_BUCKETS_US: [u64; 6] = [100, 500, 1_000, 2_000, 5_000, 10_000]; + +/// PW4 step 1: the distribution of the producer's implicit-fence wait, measured **on the PipeWire +/// loop thread**, which is exactly where it is expensive — that thread is the compositor's +/// consumer, so time spent blocked here delays buffer recycling for the NEXT frame. +/// +/// PW4 is pre-registered to be **abandoned on evidence**: if the p99 sits in the first bucket the +/// wait is already ~free and the package becomes a comment correction. Shipping the instrument +/// before the change is the whole point — the alternative is moving load-bearing synchronisation +/// off a thread on a hunch. +/// +/// One caveat the reader needs: measure this AFTER the priority levers land. The win case for +/// moving the wait is a loaded GPU, which is the same scenario PW1 targets, so a measurement taken +/// before PW1 would hand PW4 credit for PW1's problem. +#[derive(Debug, Default, Clone, Copy)] +pub(super) struct FenceWaitStats { + samples: u64, + total_us: u64, + max_us: u64, + /// One more than the bucket bounds: the overflow bucket. + buckets: [u64; FENCE_WAIT_BUCKETS_US.len() + 1], + /// Outcome split — a `NoFence` majority means the wait is structurally free on this producer + /// (nothing to wait for), which is a different finding from "the wait is short". + signaled: u64, + no_fence: u64, + timed_out: u64, + failed: u64, +} + +impl FenceWaitStats { + /// Record one wait. `bucket_of` is inlined here rather than exposed: the histogram is only + /// ever read through [`summary`](Self::summary). + pub(super) fn record(&mut self, us: u64) { + self.samples += 1; + self.total_us += us; + self.max_us = self.max_us.max(us); + let idx = FENCE_WAIT_BUCKETS_US + .iter() + .position(|&b| us <= b) + .unwrap_or(FENCE_WAIT_BUCKETS_US.len()); + self.buckets[idx] += 1; + } + + /// The bucket the `q`-quantile falls in, as its upper bound in µs — `None` for the overflow + /// bucket (i.e. "worse than the last edge"). Counting up to the quantile rather than + /// interpolating keeps this honest about what a histogram can actually say. + pub(super) fn quantile_bucket_us(&self, q: f64) -> Option> { + if self.samples == 0 { + return None; + } + // The index of the sample at `q`, 0-based, so q=1.0 picks the last sample. + let target = ((self.samples as f64) * q).ceil().max(1.0) as u64; + let mut seen = 0u64; + for (i, &count) in self.buckets.iter().enumerate() { + seen += count; + if seen >= target { + return Some(FENCE_WAIT_BUCKETS_US.get(i).copied()); + } + } + Some(None) + } + + pub(super) fn mean_us(&self) -> u64 { + self.total_us.checked_div(self.samples).unwrap_or(0) + } + + /// Whether enough has been seen for the p99 to mean anything. 100 frames is under two seconds + /// at 60 fps and is the point where one outlier stops dominating the answer. + pub(super) fn is_meaningful(&self) -> bool { + self.samples >= 100 + } +} + +/// PW5 stage 1: how many buffers the producer actually allocated for this stream. +/// +/// **Nothing in this codebase had ever counted them.** The zero-copy path dups the dmabuf fd and +/// publishes the frame while the SPA buffer is handed straight back to the producer at `.process` +/// return — so the only thing keeping capture untorn is that the producer round-robins a pool +/// deeper than our import+encode window. That depth was an unmeasured assumption; this makes it a +/// logged number, on every producer, before anything is built on it. +/// +/// `live` is maintained by the `add_buffer`/`remove_buffer` stream callbacks, which PipeWire fires +/// on the loop thread as the pool is allocated (and again, remove-then-add, on a renegotiation that +/// replaces it). There is no "pool complete" event, so the count is published from `.process`: by +/// the time the first buffer is dequeued the allocation has finished. +#[derive(Debug, Default, Clone, Copy)] +pub(super) struct PoolCensus { + /// Buffers currently in the pool (adds minus removes). + live: u32, + /// Deepest `live` seen this session — the number a depth decision must key on, since a + /// renegotiation can transiently shrink the pool to zero. + high_water: u32, + /// The `live` value already logged, so a stable pool logs exactly one line per distinct depth + /// (a renegotiation that changes the depth is worth a second line; 240 frames a second of the + /// same number is not). + logged: Option, +} + +impl PoolCensus { + fn add(&mut self) { + self.live += 1; + self.high_water = self.high_water.max(self.live); + } + + fn remove(&mut self) { + self.live = self.live.saturating_sub(1); + } + + /// Called from `.process`, once per buffer. Returns `Some(live)` the first time each distinct + /// depth is seen — the caller logs then and only then. + fn note_frame(&mut self) -> Option { + (self.logged != Some(self.live)).then(|| { + self.logged = Some(self.live); + self.live + }) + } +} + +/// Per-session tally of raw-passthrough frames that fell through to the CPU path, with a one-line +/// budget per distinct reason. +/// +/// Rate-limiting is what makes this shippable: `.process` runs per frame, so an unconditional log +/// would flood at the capture rate. Per *reason* rather than per session, because the four reasons +/// diagnose different faults and a transient `NoFormat` at open must not spend the budget a +/// persistent `NotDmabuf` needs. +#[derive(Debug, Default, Clone, Copy)] +pub(super) struct PassthroughFallbacks { + frames: u64, + logged: u8, +} + +impl PassthroughFallbacks { + /// Record one fall-through. Returns `Some(frames_so_far)` the FIRST time each distinct reason + /// is seen this session — the caller logs then and only then, so at most four lines per + /// session regardless of frame rate. + pub(super) fn note(&mut self, reason: PassthroughFallback) -> Option { + self.frames += 1; + let bit = reason.bit(); + (self.logged & bit == 0).then(|| { + self.logged |= bit; + self.frames + }) + } +} + /// Consecutive tiled-import failures (worker alive, e.g. a per-buffer `EGL_BAD_MATCH`) before /// the stream is poisoned for rebuild. A tiled import failure must NEVER fall through to the /// CPU mmap path — de-padding tiled bytes as linear produces a scrambled image — so after a @@ -345,9 +685,34 @@ fn consume_frame(ud: &mut UserData, spa_buf: *mut spa::sys::spa_buffer) { // the buffer's implicit fence and wait the producer's render before sampling — // closing the stale/old-frame race on NVIDIA. No-op for shm buffers or drivers that // attach no fence. Covers both the GPU import and the CPU mmap read below. + // + // MEASURED 2026-08-08 (Wave-2 PW4, which proposed moving this off the loop thread and was + // pre-registered to be abandoned if the wait was already free — it is): + // gamescope + NVIDIA (RTX 5070 Ti) outcome=NoFence + // Mutter + NVIDIA (RTX 5070 Ti) outcome=NoFence + // gamescope + RADV (Deck VANGOGH) 300 samples, ALL NoFence, mean 23us, max 48us, + // p50 and p99 both in the <=100us bucket + // KWin + RADV (Deck desktop) no implicit fence either + // That is EVERY compositor × vendor combination this fleet has, and not one of them attaches an + // implicit fence — so this is one ioctl and a return, not a block: there is nothing to wait on, + // and moving it to the consumer side buys nothing. The 100 ms budget stays as a guard for a + // producer that DOES fence, which is a real thing even if nothing here does it. The histogram + // below is how to re-check if that ever changes: run with PUNKTFUNK_PERF=1 and read the p99 + // bucket. if datas[0].type_() == pw::spa::buffer::DataType::DmaBuf { - match pf_zerocopy::dmabuf_fence::wait_read_ready(datas[0].fd(), 100) { + // PW4 step 1: time the wait. Two `Instant::now()` per frame on a path that is already + // making a syscall — and this is the measurement that decides whether PW4 ships at all. + let t0 = std::time::Instant::now(); + let waited = pf_zerocopy::dmabuf_fence::wait_read_ready(datas[0].fd(), 100); + ud.fence_wait.record(t0.elapsed().as_micros() as u64); + match waited { Ok(outcome) => { + use pf_zerocopy::dmabuf_fence::WaitOutcome; + match outcome { + WaitOutcome::Signaled => ud.fence_wait.signaled += 1, + WaitOutcome::NoFence => ud.fence_wait.no_fence += 1, + WaitOutcome::TimedOut => ud.fence_wait.timed_out += 1, + } static F1: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(true); if F1.swap(false, Ordering::Relaxed) { tracing::info!( @@ -360,6 +725,7 @@ fn consume_frame(ud: &mut UserData, spa_buf: *mut spa::sys::spa_buffer) { } } Err(e) => { + ud.fence_wait.failed += 1; static F2: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(true); if F2.swap(false, Ordering::Relaxed) { tracing::warn!( @@ -370,106 +736,161 @@ fn consume_frame(ud: &mut UserData, spa_buf: *mut spa::sys::spa_buffer) { } } } + // One line per ~5 s at 60 fps, under PUNKTFUNK_PERF only — the same gate and cadence the + // encode backends use for their submit splits, so a perf run reads as one instrument. + if pf_host_config::config().perf + && ud.fence_wait.is_meaningful() + && ud.fence_wait.samples % 300 == 0 + { + let q = |p: f64| match ud.fence_wait.quantile_bucket_us(p) { + Some(Some(us)) => format!("<={us}us"), + Some(None) => format!( + ">{}us", + FENCE_WAIT_BUCKETS_US[FENCE_WAIT_BUCKETS_US.len() - 1] + ), + None => "n/a".to_string(), + }; + tracing::info!( + samples = ud.fence_wait.samples, + mean_us = ud.fence_wait.mean_us(), + max_us = ud.fence_wait.max_us, + p50 = %q(0.50), + p99 = %q(0.99), + signaled = ud.fence_wait.signaled, + no_fence = ud.fence_wait.no_fence, + timed_out = ud.fence_wait.timed_out, + failed = ud.fence_wait.failed, + "dmabuf implicit-fence wait on the PipeWire loop thread (PW4: a p99 in the first \ + bucket means this wait is already free and moving it off-thread buys nothing)" + ); + } } // Raw DMA-BUF passthrough: packed RGB is imported for GPU CSC; producer-native NV12 can // be consumed by the Vulkan Video encoder without another color conversion. + // + // The block below either publishes a dmabuf and RETURNS, or breaks with the reason it could + // not — so every non-success exit is named and counted instead of silently falling out of + // three nested `if`s into the CPU path, which is how a session that had negotiated zero-copy + // could pay a full CPU pixel touch per frame while logging nothing but a healthy open. if ud.vaapi_passthrough { - if let Some(fmt) = ud.format { - if datas[0].type_() == pw::spa::buffer::DataType::DmaBuf { - if let Some(fourcc) = pf_frame::drm_fourcc(fmt) { - let chunk = datas[0].chunk(); - let offset = chunk.offset(); - let stride = chunk.stride().max(0) as u32; - // Native NV12 usually arrives as a two-plane SPA buffer over ONE buffer - // object; plane 1's chunk carries the REAL UV offset/stride (compositors - // may align the Y plane before UV). Pass it through instead of assuming - // contiguity. Each spa_data holds its own (dup'd) fd, so BO identity is - // by inode, not fd number; a genuinely two-BO frame cannot travel through - // the single-fd import — drop it with a diagnosis instead of streaming - // garbage chroma. - let plane1 = - if fmt == PixelFormat::Nv12 && datas.len() >= 2 && datas[1].fd() > 0 { - // SAFETY: zeroed `libc::stat` is a valid POD initializer; both fds are - // owned by the live PipeWire buffer for this callback, and `fstat` - // only writes the out-param structs, whose fields are read only after - // the `== 0` success checks. - let same_bo = unsafe { - let mut s0: libc::stat = std::mem::zeroed(); - let mut s1: libc::stat = std::mem::zeroed(); - libc::fstat(datas[0].fd() as i32, &mut s0) == 0 - && libc::fstat(datas[1].fd() as i32, &mut s1) == 0 - && (s0.st_dev, s0.st_ino) == (s1.st_dev, s1.st_ino) - }; - if !same_bo { - warn_once( - "NV12 planes live in different buffer objects — frames \ - dropped (single-fd import only)", - ); - return; - } - let c1 = datas[1].chunk(); - Some((c1.offset(), c1.stride().max(0) as u32)) - } else { - None - }; - // dup the fd so it survives the SPA buffer recycle — the encode thread - // imports it. Content stability across the brief import/encode window relies - // on the compositor's buffer-pool depth, like any zero-copy capture. - // SAFETY: `datas[0].fd()` is the dmabuf fd owned by the live PipeWire buffer (valid - // for this callback). `fcntl(fd, F_DUPFD_CLOEXEC, 0)` reads only the integer fd, - // touches no Rust memory, and returns a fresh independent CLOEXEC duplicate (or -1). - // The original stays owned by PipeWire; the dup is a new fd we own (checked >= 0). - let dup = - unsafe { libc::fcntl(datas[0].fd() as i32, libc::F_DUPFD_CLOEXEC, 0) }; - if dup >= 0 { - let pts_ns = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos() as u64) - .unwrap_or(0); - ud.publish(CapturedFrame { - width: w as u32, - height: h as u32, - pts_ns, - format: fmt, - payload: FramePayload::Dmabuf(DmabufFrame { - // SAFETY: `dup` is the fresh fd `fcntl(F_DUPFD_CLOEXEC)` just returned - // (checked `dup >= 0`); nothing else owns it, so `OwnedFd` takes sole - // ownership and closes it exactly once on drop — no alias, no - // double-close. - fd: unsafe { OwnedFd::from_raw_fd(dup) }, - fourcc, - modifier: ud.modifier, - offset, - stride, - plane1, - }), - // Cursor-as-metadata is blended only by RGB→NV12 backends. Gamescope - // embeds its pointer in the produced pixels, so native NV12 has none. - cursor: ud.cursor.overlay(), - }); - static ONCE: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(true); - if ONCE.swap(false, Ordering::Relaxed) { - tracing::info!( - w, - h, - modifier = ud.modifier, - fourcc = format_args!("{:#010x}", fourcc), - source = if fmt == PixelFormat::Nv12 { - "producer-native NV12" - } else { - "packed RGB (encoder GPU CSC)" - }, - "zero-copy: handing the raw DMA-BUF to the encoder" - ); - } - return; - } - } + let reason = 'passthrough: { + let Some(fmt) = ud.format else { + break 'passthrough PassthroughFallback::NoFormat; + }; + if datas[0].type_() != pw::spa::buffer::DataType::DmaBuf { + break 'passthrough PassthroughFallback::NotDmabuf; } + let Some(fourcc) = pf_frame::drm_fourcc(fmt) else { + break 'passthrough PassthroughFallback::NoFourcc; + }; + let chunk = datas[0].chunk(); + let offset = chunk.offset(); + let stride = chunk.stride().max(0) as u32; + // Native NV12 usually arrives as a two-plane SPA buffer over ONE buffer + // object; plane 1's chunk carries the REAL UV offset/stride (compositors + // may align the Y plane before UV). Pass it through instead of assuming + // contiguity. Each spa_data holds its own (dup'd) fd, so BO identity is + // by inode, not fd number; a genuinely two-BO frame cannot travel through + // the single-fd import — drop it with a diagnosis instead of streaming + // garbage chroma. + let plane1 = if fmt == PixelFormat::Nv12 && datas.len() >= 2 && datas[1].fd() > 0 { + // SAFETY: zeroed `libc::stat` is a valid POD initializer; both fds are + // owned by the live PipeWire buffer for this callback, and `fstat` + // only writes the out-param structs, whose fields are read only after + // the `== 0` success checks. + let same_bo = unsafe { + let mut s0: libc::stat = std::mem::zeroed(); + let mut s1: libc::stat = std::mem::zeroed(); + libc::fstat(datas[0].fd() as i32, &mut s0) == 0 + && libc::fstat(datas[1].fd() as i32, &mut s1) == 0 + && (s0.st_dev, s0.st_ino) == (s1.st_dev, s1.st_ino) + }; + if !same_bo { + warn_once( + "NV12 planes live in different buffer objects — frames \ + dropped (single-fd import only)", + ); + // Not a fall-through: this frame is DROPPED, not downgraded (de-padding it as + // linear would stream scrambled chroma), so it is not counted below. + return; + } + let c1 = datas[1].chunk(); + Some((c1.offset(), c1.stride().max(0) as u32)) + } else { + None + }; + // dup the fd so it survives the SPA buffer recycle — the encode thread + // imports it. Content stability across the brief import/encode window relies + // on the compositor's buffer-pool depth, like any zero-copy capture. + // SAFETY: `datas[0].fd()` is the dmabuf fd owned by the live PipeWire buffer (valid + // for this callback). `fcntl(fd, F_DUPFD_CLOEXEC, 0)` reads only the integer fd, + // touches no Rust memory, and returns a fresh independent CLOEXEC duplicate (or -1). + // The original stays owned by PipeWire; the dup is a new fd we own (checked >= 0). + let dup = unsafe { libc::fcntl(datas[0].fd() as i32, libc::F_DUPFD_CLOEXEC, 0) }; + if dup < 0 { + break 'passthrough PassthroughFallback::DupFailed; + } + let pts_ns = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0); + ud.publish(CapturedFrame { + width: w as u32, + height: h as u32, + pts_ns, + format: fmt, + payload: FramePayload::Dmabuf(DmabufFrame { + // SAFETY: `dup` is the fresh fd `fcntl(F_DUPFD_CLOEXEC)` just returned + // (checked `dup >= 0`); nothing else owns it, so `OwnedFd` takes sole + // ownership and closes it exactly once on drop — no alias, no + // double-close. + fd: unsafe { OwnedFd::from_raw_fd(dup) }, + fourcc, + modifier: ud.modifier, + offset, + stride, + plane1, + }), + // Cursor-as-metadata is blended only by RGB→NV12 backends. Gamescope + // embeds its pointer in the produced pixels, so native NV12 has none. + cursor: ud.cursor.overlay(), + }); + static ONCE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(true); + if ONCE.swap(false, Ordering::Relaxed) { + tracing::info!( + w, + h, + modifier = ud.modifier, + fourcc = format_args!("{:#010x}", fourcc), + source = if fmt == PixelFormat::Nv12 { + "producer-native NV12" + } else { + "packed RGB (encoder GPU CSC)" + }, + "zero-copy: handing the raw DMA-BUF to the encoder" + ); + } + return; + }; + // The passthrough declined this frame. Say so ONCE per distinct reason (`.process` runs + // per frame — see `PassthroughFallbacks`), carrying the running count so a persistent + // downgrade is distinguishable from a one-frame hiccup at renegotiation. + if let Some(frames) = ud.passthrough_fallbacks.note(reason) { + tracing::warn!( + frames, + "zero-copy raw-dmabuf passthrough did not take this frame: {} — {} ({})", + reason.as_str(), + if reason.falls_back_to_cpu() { + "it falls back to the CPU capture path, costing a full-resolution mmap \ + de-pad plus the encoder's own upload on every such frame" + } else { + "the frame is DROPPED — the CPU de-pad path needs the negotiated format \ + too, so nothing streams while this persists" + }, + reason.hint() + ); } - // Not a dmabuf (or unmappable format) — fall through to the CPU de-pad path. } // Zero-copy path: if the buffer is a dmabuf and we have an importer, import it @@ -912,6 +1333,31 @@ pub fn pipewire_thread( want_dmabuf && !vaapi_passthrough && !want_hdr, Ordering::Relaxed, ); + // The ONE line that says which arm this session actually got, and who consumes it. Everything + // below explains a particular arm; nothing below states the answer. Reconstructing it from the + // detail lines cost the 2026-08-08 PyroWave triage four files, and on the one arm that matters + // most — a PyroWave session downgraded to CPU on an NVIDIA host — there was no detail line to + // reconstruct it from at all (see the final `else if` of this chain). + let consumer = consumer_kind( + policy.pyrowave_session, + backend_is_vaapi, + policy.backend_is_gpu, + ); + let arm = resolved_capture_arm(&plan, importer.is_some(), want_dmabuf); + tracing::info!( + capture_arm = arm.as_str(), + consumer = consumer.as_str(), + modifier_count = modifiers.len(), + // PW3(c): the latch state belongs on the same line as the arm. A `cpu` arm has two very + // different explanations — "this host was never going to do dmabuf" and "something failed + // earlier and we are still living with the verdict" — and only the second is a bug worth + // chasing. Reading it here also means the retry/clear behaviour is observable rather than + // inferred. + raw_dmabuf_latch = pf_zerocopy::raw_dmabuf_latch_state(), + "capture pipeline resolved: {} → {}", + arm.as_str(), + consumer.as_str() + ); if force_shm { tracing::info!( "capture: PUNKTFUNK_FORCE_SHM — race-free SHM download path (no dmabuf, no zero-copy)" @@ -942,19 +1388,34 @@ pub fn pipewire_thread( sample = ?&modifiers[..modifiers.len().min(6)], "zero-copy: advertising EGL-importable dmabuf modifiers" ); - } else if backend_is_vaapi && policy.backend_is_gpu { + } else if consumer.cpu_is_downgrade() { // Reached only when no dmabuf is advertised at all (every arm above rules out a - // zero-copy path), so this genuinely IS the CPU capture path: a VAAPI session then pays - // three full-frame CPU touches (mmap de-pad + swscale RGB→NV12 + surface upload) — + // zero-copy path), so this genuinely IS the CPU capture path: the consumer then pays + // full-frame CPU touches (mmap de-pad + whatever CSC/upload it needs) — // make the silent fallback visible. // The `raw_dmabuf_latched` arm above catches the latched downgrade, so by here zero-copy // is off at the source: the env var, or the session's own output format. + // + // The gate used to be `backend_is_vaapi && backend_is_gpu`, which is why a PyroWave + // session's CPU downgrade was invisible on an NVIDIA/auto host: `backend_is_vaapi` reads + // the host-global encoder pref, so a per-session PyroWave negotiation there is `false` and + // fell out of the chain logging NOTHING. `consumer_kind` asks the per-session question + // instead, and excludes only the software encoder (which wants CPU frames). tracing::warn!( - "VAAPI encode with the CPU capture path (per-frame de-pad + swscale CSC + \ - upload) — zero-copy is off for this capture ({}); clear PUNKTFUNK_ZEROCOPY to \ - restore the dmabuf default", + consumer = consumer.as_str(), + "{} encode with the CPU capture path (per-frame de-pad + CSC + upload) — \ + zero-copy is off for this capture ({}); set PUNKTFUNK_ZEROCOPY=1 to restore the \ + dmabuf default", + consumer.as_str(), if std::env::var_os("PUNKTFUNK_ZEROCOPY").is_some() { "PUNKTFUNK_ZEROCOPY is set falsy" + } else if want_hdr && !policy.hdr_cuda_ok { + // Reachable and NOT the output format's doing: `build_importer` drops an HDR + // capture whose encoder cannot take a packed 10-bit CUDA payload (libav's HDR + // route swscales into a P010 hardware frame). Naming the output format here + // would send the reader hunting the wrong knob. + "this HDR session's encoder cannot ingest a 10-bit CUDA payload, so the capture \ + stays on CPU frames" } else { "this session's output format asked for CPU frames" } @@ -986,6 +1447,9 @@ pub fn pipewire_thread( yuv444: want_444, linear_nv12_failed: false, dbg_log_n: 0, + fence_wait: FenceWaitStats::default(), + pool: PoolCensus::default(), + passthrough_fallbacks: PassthroughFallbacks::default(), cursor: CursorState::new(cursor_id0_hides), expect_dims: if expect_exact_dims { preferred.map(|(w, h, _)| (w, h)) @@ -1083,6 +1547,11 @@ pub fn pipewire_thread( } } }) + // PW5 stage 1 — the pool census. PipeWire fires these on the loop thread as it allocates + // (and, on a renegotiation, frees then re-allocates) the stream's buffers. Counting only: + // the buffer pointer is not touched, so no lifetime question arises here. + .add_buffer(|_stream, ud, _buf| ud.pool.add()) + .remove_buffer(|_stream, ud, _buf| ud.pool.remove()) .process(|stream, ud| { // Latest-frame-only (OBS pattern): Mutter delivers buffers in bursts and recycles its // pool; an older queued buffer carries a STALE frame. Drain all queued buffers, requeue @@ -1111,6 +1580,25 @@ pub fn pipewire_thread( newest = next; drained += 1; } + // PW5 stage 1: publish the depth the producer actually negotiated, once per distinct + // value. MEASURED, not requested: `build_dmabuf_buffers` asks for a range and the + // producer picks — this line is the only place the picked number is visible. + // + // Why it matters beyond curiosity: `stream.queue_raw_buffer(newest)` at the end of this + // callback hands the buffer back while the encode thread may still be importing and + // reading its dmabuf, so content stability rests entirely on the producer not cycling + // back to this buffer before we are done with it. That window is `pool_depth` buffer + // periods wide. A pool of 2 has essentially none. + if let Some(depth) = ud.pool.note_frame() { + tracing::info!( + pool_depth = depth, + high_water = ud.pool.high_water, + drained, + "pipewire buffer pool negotiated — this is the producer's ACTUAL count \ + (add_buffer/remove_buffer), the window in which a buffer we handed back may \ + be rewritten while the encoder still reads it" + ); + } // Sacrificial-mode gate (kwin.rs `create`): until the producer renegotiates to the // expected dims, every buffer — frame AND cursor meta, whose positions are in the // doomed mode's space — belongs to the birth mode; consuming one would build the @@ -1708,4 +2196,271 @@ mod tests { assert!(!p.want_dmabuf(false, &[0])); } } + + // ---- PW2: capture-arm observability (pure halves) ------------------------------------- + // + // Env-var reads race under a shared test process, so these assert against the pure functions + // the logging sites call — the same rule `negotiation_plan_invariants` follows. + + use super::{ + consumer_kind, resolved_capture_arm, CaptureArm, ConsumerKind, FenceWaitStats, + PassthroughFallback, PassthroughFallbacks, PoolCensus, FENCE_WAIT_BUCKETS_US, + }; + + /// A PyroWave session is PyroWave even though it also flips `backend_is_vaapi` on (the + /// `linux_zero_copy_is_vaapi` `Pyrowave` arm). Getting this precedence backwards is the exact + /// shape of the bug PW2 fixes: the session gets reported as somebody else's backend. + #[test] + fn pyrowave_outranks_the_host_global_backend_pref() { + assert_eq!(consumer_kind(true, true, true), ConsumerKind::PyroWave); + // ...and on an NVIDIA/auto host, where `backend_is_vaapi` is false, it is still PyroWave — + // the case that previously logged nothing at all. + assert_eq!(consumer_kind(true, false, true), ConsumerKind::PyroWave); + } + + /// The non-PyroWave consumers, and the one that must NOT warn on a CPU arm. + #[test] + fn consumer_kinds_and_which_ones_a_cpu_arm_degrades() { + assert_eq!(consumer_kind(false, true, true), ConsumerKind::Vaapi); + assert_eq!(consumer_kind(false, false, true), ConsumerKind::Nvenc); + // No GPU backend ⇒ the software encoder, whose native input IS CPU frames. + assert_eq!(consumer_kind(false, false, false), ConsumerKind::Software); + assert!(ConsumerKind::PyroWave.cpu_is_downgrade()); + assert!(ConsumerKind::Vaapi.cpu_is_downgrade()); + assert!(ConsumerKind::Nvenc.cpu_is_downgrade()); + assert!(!ConsumerKind::Software.cpu_is_downgrade()); + } + + /// The arm is a function of the plan plus the two runtime facts. Pinned against every plan the + /// resolver can produce, so the headline line can never claim an arm the session did not take. + #[test] + fn resolved_arm_matches_the_plan_that_produced_it() { + // PyroWave/VAAPI raw passthrough. + let p = negotiation_plan(NegotiationInputs { + pyrowave_session: true, + ..nvenc() + }); + assert!(p.vaapi_passthrough); + assert_eq!( + resolved_capture_arm(&p, false, p.want_dmabuf(false, &[0])), + CaptureArm::DmabufPassthrough + ); + // NVENC via the EGL→CUDA importer. + let p = negotiation_plan(nvenc()); + assert!(p.build_importer); + assert_eq!( + resolved_capture_arm(&p, true, p.want_dmabuf(true, &[0])), + CaptureArm::CudaImport + ); + // The importer was meant to be built but did not construct (no driver): CPU, not a + // cuda-import the session never got. + assert_eq!( + resolved_capture_arm(&p, false, p.want_dmabuf(false, &[0])), + CaptureArm::Cpu + ); + // An empty modifier list is a CPU arm even under a live passthrough plan. + let p = negotiation_plan(NegotiationInputs { + pyrowave_session: true, + ..nvenc() + }); + assert_eq!( + resolved_capture_arm(&p, false, p.want_dmabuf(false, &[])), + CaptureArm::Cpu + ); + // Forced SHM: CPU regardless of everything else. + let p = negotiation_plan(NegotiationInputs { + force_shm: true, + ..nvenc() + }); + assert_eq!( + resolved_capture_arm(&p, true, p.want_dmabuf(true, &[0])), + CaptureArm::Cpu + ); + } + + /// The rate limiter: ONE line per distinct reason per session, counting every fall-through. + /// `.process` runs per frame, so an off-by-one here is a log flood at the capture rate. + #[test] + fn fallback_log_budget_is_one_line_per_reason() { + let mut f = PassthroughFallbacks::default(); + // First of a reason logs, and reports the running total (not a per-reason count). + assert_eq!(f.note(PassthroughFallback::NotDmabuf), Some(1)); + // Repeats of the SAME reason never log again, but are still counted. + for _ in 0..1_000 { + assert_eq!(f.note(PassthroughFallback::NotDmabuf), None); + } + // A DIFFERENT reason is a different diagnosis and gets its own line, carrying the + // now-large total — which is what distinguishes a persistent downgrade from a hiccup. + assert_eq!(f.note(PassthroughFallback::DupFailed), Some(1002)); + assert_eq!(f.note(PassthroughFallback::DupFailed), None); + // All four reasons fit the budget independently; the tally counts every frame. + assert_eq!(f.note(PassthroughFallback::NoFormat), Some(1004)); + assert_eq!(f.note(PassthroughFallback::NoFourcc), Some(1005)); + // Budget spent: every reason has logged once, so nothing logs again however long the + // session runs. + for r in [ + PassthroughFallback::NoFormat, + PassthroughFallback::NotDmabuf, + PassthroughFallback::NoFourcc, + PassthroughFallback::DupFailed, + ] { + assert_eq!(f.note(r), None); + } + } + + /// Every reason is distinguishable (a shared bit would silence one of them) and carries an + /// actionable hint — a reason with no fix is a line the reader cannot use. + #[test] + fn every_fallback_reason_is_distinct_and_actionable() { + let all = [ + PassthroughFallback::NoFormat, + PassthroughFallback::NotDmabuf, + PassthroughFallback::NoFourcc, + PassthroughFallback::DupFailed, + ]; + let mut f = PassthroughFallbacks::default(); + for r in all { + assert!( + f.note(r).is_some(), + "{r:?} shares a bit with an earlier reason" + ); + assert!(!r.as_str().is_empty()); + assert!(!r.hint().is_empty()); + } + // Only `NoFormat` drops the frame; the other three downgrade it. The log line picks its + // consequence clause off this, so an inverted answer would print the opposite of the truth. + assert!(!PassthroughFallback::NoFormat.falls_back_to_cpu()); + assert!(PassthroughFallback::NotDmabuf.falls_back_to_cpu()); + assert!(PassthroughFallback::NoFourcc.falls_back_to_cpu()); + assert!(PassthroughFallback::DupFailed.falls_back_to_cpu()); + } + + // ---- PW4 step 1: the fence-wait histogram ------------------------------------------------ + // + // This instrument decides whether PW4 ships, so its arithmetic has to be right: a p99 that + // reads one bucket low would retire a package that was worth doing, and one that reads high + // would justify moving load-bearing synchronisation off a thread for nothing. + + /// An empty histogram must say "no answer", not "zero" — those are different claims, and the + /// second one would look like a decisive p99 ≈ 0 result. + #[test] + fn an_empty_histogram_has_no_quantile() { + let s = FenceWaitStats::default(); + assert_eq!(s.quantile_bucket_us(0.99), None); + assert_eq!(s.mean_us(), 0); + assert!(!s.is_meaningful(), "it must not be trusted yet either"); + } + + /// The all-fast case — the one that RETIRES PW4. Every sample in the first bucket must put the + /// p99 there too. + #[test] + fn an_all_fast_distribution_puts_p99_in_the_first_bucket() { + let mut s = FenceWaitStats::default(); + for _ in 0..1000 { + s.record(3); + } + assert_eq!( + s.quantile_bucket_us(0.50), + Some(Some(FENCE_WAIT_BUCKETS_US[0])) + ); + assert_eq!( + s.quantile_bucket_us(0.99), + Some(Some(FENCE_WAIT_BUCKETS_US[0])) + ); + assert_eq!(s.mean_us(), 3); + } + + /// The case PW4 exists for: a fast median with a heavy tail. The p50 must stay low AND the p99 + /// must find the tail — a histogram that smeared them together could not tell the two worlds + /// apart, which is the entire decision. + #[test] + fn a_heavy_tail_moves_p99_without_moving_p50() { + let mut s = FenceWaitStats::default(); + for _ in 0..980 { + s.record(10); // fast majority + } + for _ in 0..20 { + s.record(6_000); // 2 % of frames stall milliseconds + } + assert_eq!( + s.quantile_bucket_us(0.50), + Some(Some(FENCE_WAIT_BUCKETS_US[0])), + "the median is still free" + ); + assert_eq!( + s.quantile_bucket_us(0.99), + Some(Some(10_000)), + "...but the p99 must land in the 5-10ms bucket, not with the median" + ); + assert_eq!(s.max_us, 6_000); + } + + /// Anything past the last edge reports as overflow rather than being clamped into the last + /// bucket — "worse than 10 ms" is a distinct finding and must not read as "10 ms". + #[test] + fn waits_past_the_last_edge_report_as_overflow() { + let mut s = FenceWaitStats::default(); + s.record(99_000); + assert_eq!(s.quantile_bucket_us(0.99), Some(None)); + } + + /// Bucket edges are inclusive upper bounds, so a sample exactly ON an edge belongs to that + /// bucket and not the next one up. + #[test] + fn bucket_edges_are_inclusive() { + for (i, &edge) in FENCE_WAIT_BUCKETS_US.iter().enumerate() { + let mut s = FenceWaitStats::default(); + s.record(edge); + assert_eq!( + s.quantile_bucket_us(1.0), + Some(Some(edge)), + "a sample of exactly {edge}us belongs in bucket {i}" + ); + } + } + + /// PW5 stage 1: a stable pool logs ONE line, not one per frame. `.process` runs at the capture + /// rate — an unconditional log here would be 240 lines a second of the same number. + #[test] + fn a_stable_pool_is_logged_once() { + let mut p = PoolCensus::default(); + for _ in 0..8 { + p.add(); + } + assert_eq!(p.note_frame(), Some(8)); + for _ in 0..100 { + assert_eq!(p.note_frame(), None, "the same depth must not re-log"); + } + } + + /// A renegotiation frees the pool and re-allocates it. The LIVE count therefore dips (and the + /// new depth is worth a second line), but `high_water` — the number a pipeline-depth decision + /// keys on — must not follow the dip down. + #[test] + fn a_renegotiated_pool_relogs_but_the_high_water_holds() { + let mut p = PoolCensus::default(); + for _ in 0..8 { + p.add(); + } + assert_eq!(p.note_frame(), Some(8)); + for _ in 0..8 { + p.remove(); + } + for _ in 0..4 { + p.add(); + } + assert_eq!(p.note_frame(), Some(4), "a changed depth is worth a line"); + assert_eq!(p.high_water, 8, "the deepest pool seen this session"); + } + + /// `remove_buffer` without a matching `add_buffer` must not wrap the count to `u32::MAX` — + /// a depth gate reading that would happily pipeline against a pool of zero. + #[test] + fn unmatched_removes_saturate_at_zero() { + let mut p = PoolCensus::default(); + p.remove(); + p.remove(); + assert_eq!(p.note_frame(), Some(0)); + assert_eq!(p.high_water, 0); + } } diff --git a/crates/pf-capture/src/linux/pw_pods.rs b/crates/pf-capture/src/linux/pw_pods.rs index b1bd7b71..777913a2 100644 --- a/crates/pf-capture/src/linux/pw_pods.rs +++ b/crates/pf-capture/src/linux/pw_pods.rs @@ -288,16 +288,57 @@ pub(super) fn build_shm_only_buffers() -> Result> { }) } -/// Build a Buffers param requesting dmabuf-only buffers. +/// PW5 stage 2: the buffer-pool depth we ASK for on the zero-copy path, as a Choice range. +/// +/// The zero-copy path hands the SPA buffer back to the producer at `.process` return, while the +/// encode thread still holds a dup of its dmabuf fd and has not yet imported, let alone read, the +/// contents. Nothing bounds that window — see the `queue_raw_buffer` comment in `pipewire.rs` — so +/// the only thing that keeps capture untorn is the producer round-robining a pool deeper than our +/// import+encode latency. Until PW5 stage 1 nobody had ever counted what that pool was; we never +/// even asked for a size (`build_dmabuf_buffers` set `dataType` and nothing else). +/// +/// A **range**, deliberately, not a fixed count: SPA intersects the consumer's and producer's +/// Buffers params, so a fixed 8 against a producer that can only afford 4 empties the intersection +/// and the link silently stalls in "negotiating" — the exact failure mode the cursor-meta `size` +/// property already cost this codebase once (see `build_cursor_meta_param`). With a range the +/// producer clamps into it and negotiation still succeeds. +/// +/// The numbers: `min` stays at 2 so nothing that works today stops working; `default` 8 is ~133 ms +/// of buffer at 60 Hz and ~33 ms at 240 Hz, comfortably past the ~3-4 ms capture→fence latency +/// measured in PW3/PW4 even with a second frame in flight; `max` 16 is a ceiling, not a request +/// (a 4K 4:4:4 buffer is ~25 MB, so 16 is ~400 MB of compositor allocation and worth capping). +/// **What the producer actually picks is logged by the stage-1 census — trust that line, not +/// these constants.** +const POOL_MIN: i32 = 2; +const POOL_DEFAULT: i32 = 8; +const POOL_MAX: i32 = 16; + +/// Build a Buffers param requesting dmabuf-only buffers, with pool headroom (see [`POOL_DEFAULT`]). pub(super) fn build_dmabuf_buffers() -> Result> { serialize_pod(pw::spa::pod::Object { type_: pw::spa::utils::SpaTypes::ObjectParamBuffers.as_raw(), id: pw::spa::param::ParamType::Buffers.as_raw(), - properties: vec![pw::spa::pod::Property { - key: pw::spa::sys::SPA_PARAM_BUFFERS_dataType, - flags: pw::spa::pod::PropertyFlags::empty(), - value: pw::spa::pod::Value::Int(1i32 << pw::spa::sys::SPA_DATA_DmaBuf), - }], + properties: vec![ + pw::spa::pod::Property { + key: pw::spa::sys::SPA_PARAM_BUFFERS_dataType, + flags: pw::spa::pod::PropertyFlags::empty(), + value: pw::spa::pod::Value::Int(1i32 << pw::spa::sys::SPA_DATA_DmaBuf), + }, + pw::spa::pod::Property { + key: pw::spa::sys::SPA_PARAM_BUFFERS_buffers, + flags: pw::spa::pod::PropertyFlags::empty(), + value: pw::spa::pod::Value::Choice(pw::spa::pod::ChoiceValue::Int( + pw::spa::utils::Choice( + pw::spa::utils::ChoiceFlags::empty(), + pw::spa::utils::ChoiceEnum::Range { + default: POOL_DEFAULT, + min: POOL_MIN, + max: POOL_MAX, + }, + ), + )), + }, + ], }) } @@ -512,4 +553,47 @@ mod tests { "libspa renumbered spa_video_transfer_function — update the hardcoded PQ id" ); } + + /// PW5 stage 2: the pool request must be a **Choice Range**, never a fixed Int. + /// + /// This is the whole safety argument for asking at all: SPA intersects the two sides' Buffers + /// params, so a fixed count a producer cannot afford empties the intersection and the link + /// stalls in "negotiating" with no error anywhere — the same trap that cost this codebase the + /// entire Linux cursor channel once (see `build_cursor_meta_param`). Asserting the pod shape + /// is what keeps a later "simplify" from turning the range back into a number. + #[test] + fn the_dmabuf_pool_request_is_a_range_not_a_fixed_count() { + let pod = build_dmabuf_buffers().unwrap(); + let key = spa::sys::SPA_PARAM_BUFFERS_buffers.to_ne_bytes(); + let at = pod + .windows(4) + .position(|w| w == key) + .expect("the dmabuf Buffers pod must carry a buffers count"); + let word = |off: usize| u32::from_ne_bytes(pod[off..off + 4].try_into().unwrap()); + // Property = { key, flags, value_pod }; value_pod = { size, type, body }. A Choice body + // is { type: u32, flags: u32, child_size: u32, child_type: u32, values… }. + assert_eq!( + word(at + 12), + spa::sys::SPA_TYPE_Choice, + "the buffers count must be a Choice, not a bare Int — a fixed count can fail \ + negotiation outright" + ); + assert_eq!( + word(at + 16), + spa::sys::SPA_CHOICE_Range, + "the Choice must be a Range (default, min, max)" + ); + assert_eq!(word(at + 24), 4, "Choice child pods are 4-byte Ints"); + assert_eq!(word(at + 28), spa::sys::SPA_TYPE_Int, "…of type Int"); + let vals: Vec = (0..3) + .map(|i| i32::from_ne_bytes(pod[at + 32 + i * 4..at + 36 + i * 4].try_into().unwrap())) + .collect(); + assert_eq!( + vals, + vec![POOL_DEFAULT, POOL_MIN, POOL_MAX], + "Range values are serialized default-first" + ); + // The minimum must not exceed what producers already serve, or the ask becomes a demand. + const { assert!(POOL_MIN <= 2) }; + } } diff --git a/crates/pf-encode/src/enc/linux/pyrowave.rs b/crates/pf-encode/src/enc/linux/pyrowave.rs index 1d9f0aac..8547930e 100644 --- a/crates/pf-encode/src/enc/linux/pyrowave.rs +++ b/crates/pf-encode/src/enc/linux/pyrowave.rs @@ -321,6 +321,15 @@ struct DeviceHold { instance_ci: Box>, _queue_prio: Box<[f32; 1]>, _queue_ci: Box<[vk::DeviceQueueCreateInfo<'static>; 1]>, + /// The elevated global-priority request chained into `_queue_ci[0].p_next` + /// (`PYROWAVE_QUEUE_PRIORITY`, see `queue_priority_candidates`). A `Box` for the same + /// move-stability reason as its siblings: `pyrowave_create_device` RETAINS `device_ci` for + /// the device's lifetime and Granite reads the chain back through + /// `get_existing_create_info()`, so this must stay put and — critically — must describe the + /// device that actually got created. The create ladder below is therefore required to write + /// its FINAL state back here (null the `p_next` if the no-priority attempt is the one that + /// succeeded); a chain the device was not created with is a lie Granite would believe. + _queue_gp: Box<[vk::DeviceQueueGlobalPriorityCreateInfoKHR<'static>; 1]>, // A plain Vec (not Box<[_; N]> like its siblings): Phase 8 pushes queue_family_foreign // conditionally. The heap buffer as_ptr() feeds device_ci is move-stable like the Boxes. _dev_exts: Vec<*const c_char>, @@ -330,6 +339,183 @@ struct DeviceHold { device_ci: Box>, } +/// Percentile of a sorted sample slice, by nearest-rank. **Pure.** Used for the `PUNKTFUNK_PERF` +/// encode split: a p99 is the whole point here (the game-load spike this codec suffers is a TAIL +/// event — the mean barely moves while individual frames go 2 ms → 18 ms), so a mean-only readout +/// would report "fine" through exactly the failure the priority lever exists to fix. +fn pct(sorted: &[u32], q: f64) -> u32 { + if sorted.is_empty() { + return 0; + } + let rank = ((sorted.len() as f64) * q).ceil() as usize; + sorted[rank.clamp(1, sorted.len()) - 1] +} + +/// The global-priority classes to try, in order, for `PYROWAVE_QUEUE_PRIORITY`. +/// +/// **Pure, and character-identical to the vendored C patch's grammar** +/// (`patches/0005-global-priority-queue.patch`, `context.cpp` ~2136-2152): unset → `realtime`; +/// ASCII-lowercased; `off` → no candidates at all; `high` → `[HIGH]`; anything else, including +/// junk → `[REALTIME, HIGH]`. Keeping the two grammars identical is the whole point — the same +/// env var drives the Windows path (where the patch is live, because Granite builds its own +/// device there) and this Linux path (where the patch is inert, because we pass our own +/// create-infos and Granite takes its `inherit_info` branch). One knob that meant two different +/// things per platform is exactly the documentation trap this wiring exists to close. +/// +/// Note `off` is the ONLY spelling that disables it; `0` is not, because the C side does not +/// accept `0` either. Do not "improve" that here without changing the patch in the same commit. +fn queue_priority_candidates(raw: Option<&str>) -> Vec { + let want = raw.map(|s| s.to_ascii_lowercase()); + match want.as_deref() { + Some("off") => Vec::new(), + Some("high") => vec![vk::QueueGlobalPriorityKHR::HIGH], + _ => vec![ + vk::QueueGlobalPriorityKHR::REALTIME, + vk::QueueGlobalPriorityKHR::HIGH, + ], + } +} + +/// Whether a `create_device` error means "this priority class was refused" — i.e. walk the ladder +/// down rather than failing the encoder open. +/// +/// `ERROR_NOT_PERMITTED_KHR` is the specified answer and the only one the C patch handles. +/// `ERROR_INITIALIZATION_FAILED` is here because the in-tree precedent that already ships this +/// ladder on Linux — pf-zerocopy's VkBridge — accepts it too, and because the failure mode this +/// guards against is severe and asymmetric: a PyroWave open is reached only by a NEGOTIATED +/// PyroWave session, so a hard error here is a dead stream, not a fallback to another encoder. +/// Treating one extra driver-specific refusal as a downgrade costs nothing; treating it as fatal +/// costs the session. +fn priority_refused(e: vk::Result) -> bool { + matches!( + e, + vk::Result::ERROR_NOT_PERMITTED_KHR | vk::Result::ERROR_INITIALIZATION_FAILED + ) +} + +/// One frame recorded and queue-submitted, whose fence has not been waited and whose bitstream has +/// therefore not been packetized yet. PW5 stage 3: this is what the `submit`/`poll` split created — +/// before it, no such state could exist because `submit` did the whole thing inline. +/// How many independent per-frame resource sets the encoder allocates. +/// +/// PW5 stage 4. Two, because Granite caps the overlap at two anyway: the pyrowave device defaults +/// to `init_frame_contexts(2)` and `next_frame_context()` — called at the top of every +/// `encode_gpu_synchronous` — waits the context it rotates into, so "frame N may not begin +/// recording until N-2 completed" is enforced below us. A third slot would buy nothing without a +/// vendored `init_frame_contexts(3)`, which is not exposed. +/// +/// **Allocated is not the same as used.** `max_inflight` decides how many are live at once and is +/// still 1 here; this stage is pure capacity so the depth change that follows is a one-line +/// behaviour change rather than a simultaneous re-plumbing. +const SLOTS: usize = 2; + +/// Everything ONE in-flight frame needs exclusively. +/// +/// PW5 stage 4 exists because the analysis found six single-slot resources beyond the y/uv images +/// the plan named, and every one of them is a correctness problem under overlap — not a +/// performance one: +/// +/// * `csc_set` was ONE descriptor set rewritten every frame by `bind_rgb`. Updating a descriptor +/// set still bound by a PENDING command buffer is a spec violation +/// (VUID-vkUpdateDescriptorSets-None-03047), and the kind that produces a wrong picture rather +/// than a validation error on most drivers. +/// * `y_img`/`uv_img` — the CSC of N+1 storage-writes exactly the images pyrowave is still +/// sampling for N. The old barrier comment ("the previous frame's encode already completed under +/// our synchronous fence") was load-bearing and said so. +/// * `cursor_img`/`cursor_stage` — the struct comment said it plainly: *"Single (not ring) because +/// PyroWave encodes one frame synchronously — no in-flight overlap to race."* A new cursor +/// bitmap's host write + copy races N's sampled read. +/// * `cmd` + `fence` — you cannot record into a PENDING command buffer at all. +/// * `cpu_img`/`cpu_stage` (software capture / tests) — the host writes staging while N's copy is +/// still pending. +/// +/// `bitstream` and `import_cache` are deliberately NOT here. `bitstream` is only touched during +/// packetize, i.e. only on the poll side, one frame at a time. `import_cache` retains the +/// `VkImage`/`VkDeviceMemory` per dmabuf inode, so dropping a `CapturedFrame` (and its dup'd fd) +/// while the GPU still reads it is not a use-after-free — do not "optimise" that retention away. +struct Slot { + cmd: vk::CommandBuffer, + fence: vk::Fence, + csc_set: vk::DescriptorSet, + y_img: vk::Image, + y_mem: vk::DeviceMemory, + y_view: vk::ImageView, + uv_img: vk::Image, + uv_mem: vk::DeviceMemory, + uv_view: vk::ImageView, + cursor_img: vk::Image, + cursor_mem: vk::DeviceMemory, + cursor_view: vk::ImageView, + cursor_stage: vk::Buffer, + cursor_stage_mem: vk::DeviceMemory, + /// Per-slot: each slot's cursor image is its own, so a bitmap change is uploaded once per slot + /// (`SLOTS` small uploads instead of one) rather than once globally, which would leave the + /// other slot showing the previous pointer. + cursor_serial: u64, + cursor_ready: bool, + /// CPU-input staging (software capture / smoke tests), lazily (re)created on format change. + cpu_img: Option<(vk::Image, vk::DeviceMemory, vk::ImageView, vk::Format)>, + cpu_stage: Option<(vk::Buffer, vk::DeviceMemory, u64)>, +} + +impl Slot { + /// All-null, matching `open_inner`'s "construct then fill" unwind discipline: every + /// `vkDestroy*`/`vkFree*` of `VK_NULL_HANDLE` is the spec-defined no-op, so `Drop` running on a + /// partially-built slot is sound. + fn null() -> Self { + Self { + cmd: vk::CommandBuffer::null(), + fence: vk::Fence::null(), + csc_set: vk::DescriptorSet::null(), + y_img: vk::Image::null(), + y_mem: vk::DeviceMemory::null(), + y_view: vk::ImageView::null(), + uv_img: vk::Image::null(), + uv_mem: vk::DeviceMemory::null(), + uv_view: vk::ImageView::null(), + cursor_img: vk::Image::null(), + cursor_mem: vk::DeviceMemory::null(), + cursor_view: vk::ImageView::null(), + cursor_stage: vk::Buffer::null(), + cursor_stage_mem: vk::DeviceMemory::null(), + cursor_serial: u64::MAX, + cursor_ready: false, + cpu_img: None, + cpu_stage: None, + } + } +} + +#[derive(Clone, Copy)] +struct InFlight { + /// Which [`Slot`] this frame's command buffer, fence, descriptor set and images belong to. + /// Carried per-frame rather than recomputed so `wait_and_packetize` cannot wait the wrong + /// fence — the failure that would look like corruption, not like an error. + slot: usize, + /// The capture timestamp this frame's AU must carry. Held here rather than re-read from the + /// `CapturedFrame` because the frame is the CALLER's and is gone by the time we packetize. + pts_ns: u64, + /// The bitstream buffer size this frame was ENCODED against (`frame_budget + BS_SLACK`), and + /// the packetize boundary in dense mode. + /// + /// Snapshotted at submit rather than re-read at poll because the split opened a window that + /// did not exist before: `reconfigure_bitrate` can land between the two, and in dense mode the + /// boundary IS this number — a shrunk budget would make `compute_num_packets` return more than + /// one packet and the encode would bail with "unexpected packet count" on a frame that was + /// perfectly fine. + cap: usize, + /// The wire sequence value stamped into this frame's block headers, so the AU can be checked + /// against what we asked for — see the self-check in `wait_and_packetize`. + seq: u8, + /// The datagram alignment this frame was encoded for; `set_wire_chunking` can likewise land + /// mid-flight, and a frame packetized at a boundary it was not rate-controlled for would ship + /// with the wrong `chunk_aligned` flag. + wire_chunk: Option, + /// `PUNKTFUNK_PERF`: when `submit` started. The summary keeps measuring submit→AU (what + /// `92326312` measured, so stage 0's baseline stays comparable), not just the wait half. + t0: std::time::Instant, +} + pub struct PyroWaveEncoder { // --- vulkan core (owned; private to this encoder) --- _entry: ash::Entry, @@ -346,48 +532,65 @@ pub struct PyroWaveEncoder { // --- pyrowave (borrows our device; destroyed before it) --- pw_dev: pw::pyrowave_device, - pw_enc: pw::pyrowave_encoder, + /// ONE `pyrowave_encoder` per [`Slot`] (PW5 stage 5), alternated. + /// + /// The object CANNOT hold two frames in flight — not "probably not", structurally not. + /// `Encoder::Impl` owns one each of `wavelet_img_high_res`, `bucket_buffer`, `meta_buffer`, + /// `block_stat_buffer`, `payload_data` and `quant_buffer`, and `Impl::encode` OPENS by + /// discarding them: an image barrier with `VK_IMAGE_LAYOUT_UNDEFINED` as the old layout (a + /// written promise that nothing else is reading it) plus three `fill_buffer` clears. Two + /// encodes recorded into two command buffers and submitted to the same queue have NO execution + /// dependency in Vulkan — submission order orders the start, not the completion — so N+1's DWT + /// would overwrite the bands and zero the RDO buckets while N's block packing still reads them. + /// + /// So overlap means two handles on one device, and within a handle the encodes stay strictly + /// serialized (a slot's next frame is only recorded after that slot's previous one was + /// retired), which keeps patch 0004's scratch-pool invariant intact without touching it. + pw_encs: Vec, + /// The wire sequence counter, kept HERE rather than in the encoder objects. + /// + /// ⚠ pyrowave's own `sequence_count` is PER-ENCODER, so two alternating handles each count + /// 1,2,3… independently and the wire sees 1,1,2,2,3,3…. The decoder restarts a frame only when + /// the value CHANGES (`diff = (hdr.sequence - last_seq) & 0x7; restart = diff != 0`), so a + /// repeat reads as MORE BLOCKS OF THE SAME FRAME: `clear()` never runs and every second frame + /// is silently swallowed, on every client. `patches/0007-encoder-sequence-override.patch` + /// exposes a setter so this single counter is stamped regardless of which handle encodes. + wire_seq: u32, - // --- CSC + planes (single slot: encode is synchronous per frame) --- + // --- CSC pipeline + sampler: SHARED by every slot (immutable once built, read-only in + // recording, so no overlap hazard). The per-frame resources live in `slots`. --- csc_pipe: vk::Pipeline, csc_layout: vk::PipelineLayout, csc_dsl: vk::DescriptorSetLayout, csc_pool: vk::DescriptorPool, - csc_set: vk::DescriptorSet, sampler: vk::Sampler, - y_img: vk::Image, - y_mem: vk::DeviceMemory, - y_view: vk::ImageView, - uv_img: vk::Image, - uv_mem: vk::DeviceMemory, - uv_view: vk::ImageView, - - // Cursor overlay (cursor-as-metadata): a fixed CURSOR_MAX² RGBA8 sampled image (bound at binding - // 3) + host staging, re-uploaded only when the bitmap changes (`cursor_serial`). Single (not - // ring) because PyroWave encodes one frame synchronously — no in-flight overlap to race. - cursor_img: vk::Image, - cursor_mem: vk::DeviceMemory, - cursor_view: vk::ImageView, - cursor_stage: vk::Buffer, - cursor_stage_mem: vk::DeviceMemory, - cursor_serial: u64, - cursor_ready: bool, // Per-buffer dmabuf-import cache keyed by (st_dev, st_ino) — mirrors `vulkan_video.rs`. + // NOT per-slot: it retains the VkImage/VkDeviceMemory per inode, which is exactly what makes + // it safe for two slots to sample the same imported buffer. import_cache: Vec<(u64, u64, vk::Image, vk::DeviceMemory, vk::ImageView)>, - // CPU-input staging (software capture / smoke tests), lazily (re)created on format change. - cpu_img: Option<(vk::Image, vk::DeviceMemory, vk::ImageView, vk::Format)>, - cpu_stage: Option<(vk::Buffer, vk::DeviceMemory, u64)>, /// Reused 3→4 expansion buffer for 24-bpp CPU payloads (`vk_util::normalize_cpu_rgb`). + /// Not per-slot: it is consumed synchronously inside `submit_frame` (copied into staging + /// before the call returns), so no GPU work ever reads it. cpu_expand: Vec, cmd_pool: vk::CommandPool, - cmd: vk::CommandBuffer, - fence: vk::Fence, - /// True between a successful `queue_submit` and its successful fence wait — i.e. exactly when - /// GPU work may still be executing. `reset()` keys its bounded wait on this: a never-submitted - /// fence would otherwise read as "wedged" (fences start unsignaled). - gpu_pending: bool, + /// The `SLOTS` independent per-frame resource sets — see [`Slot`] for why each member is in + /// there and why `bitstream`/`import_cache` are not. + slots: Vec, + /// Which slot the NEXT submit records into; advances modulo `SLOTS` per submitted frame. + next_slot: usize, + /// Frames recorded and queue-submitted whose fence has not been waited yet — i.e. exactly the + /// work that may still be executing on the GPU. `reset()` keys its bounded wait on this being + /// non-empty: a never-submitted fence would otherwise read as "wedged" (fences start + /// unsignaled). At today's depth of 1 this holds at most one entry. + inflight: VecDeque, + /// How many frames may be submitted-but-not-polled at once. **Still 1**, even though stage 4 + /// allocated `SLOTS` resource sets: raising it is stage 6's job and needs the second pyrowave + /// encoder handle (stage 5) first — pyrowave's own `Encoder` object structurally cannot hold + /// two frames (single wavelet/scratch buffers, and `Impl::encode` opens by discarding them + /// with an UNDEFINED old layout). Never exceeds `SLOTS`. + max_inflight: usize, // --- state --- width: u32, @@ -398,6 +601,13 @@ pub struct PyroWaveEncoder { chroma444: bool, /// Per-frame bitstream budget (hard CBR): `bitrate / (8 * fps)`. frame_budget: usize, + /// `PUNKTFUNK_PERF`: the synchronous encode's own duration, which is the quantity the + /// GPU-priority work is about — patch 0005's header records it going ~2 ms → 15-18 ms at + /// 95 % game load. Every other backend (VAAPI, direct NVENC) already logs a submit split; + /// this one did not, so the one encoder whose cost the priority lever exists to protect was + /// the one you could not measure. Reservoir of recent samples, summarised on a slow cadence. + perf_us: Vec, + perf_logged_at: Option, /// Datagram-aligned mode (plan §4.4): packetize at this boundary and pad every codec /// packet to it, so each wire shard carries whole self-delimiting packets. `None` = /// one packet per AU (the dense MVP shape). @@ -407,6 +617,11 @@ pub struct PyroWaveEncoder { wire_budget: crate::pyrowave_wire::WireBudget, bitstream: Vec, pending: VecDeque, + /// The AU currently being handed out in streamed chunks (PW6 — `Some` strictly between a + /// `first` chunk and its `last`). See [`crate::pyrowave_wire::AuChunker`]: this backend's + /// encode is synchronous, so the AU is COMPLETE before the first chunk leaves — the split is + /// for the send side, never an encode/send overlap. + chunker: Option, frame_count: u64, } @@ -420,6 +635,50 @@ fn budget_for(bitrate_bps: u64, fps: u32) -> usize { } impl PyroWaveEncoder { + /// `PUNKTFUNK_PERF`: record one encode duration and summarise on a slow cadence. + /// + /// **submit→AU on purpose**, and unchanged by PW5 stage 3's `submit`/`poll` split: the sample + /// is stamped when `submit` starts and taken when the AU becomes readable, so it still covers + /// CSC + encode + fence wait + packetize and stays directly comparable to the pre-split + /// baseline (`92326312`). What the split changed is WHERE the wait sits — the host loop's own + /// `submit_us` now excludes it, which is the shape every other backend already had. + /// + /// ⚠ At a depth greater than 1 this number legitimately grows by roughly one loop period, + /// because frame N's AU is not retrieved until after N+1 has been submitted. That is real + /// added latency, not an instrumentation artefact — see PW5's escalation gate. + fn note_encode_us(&mut self, us: u32) { + if !pf_host_config::config().perf { + return; + } + self.perf_us.push(us); + let now = std::time::Instant::now(); + let since = self.perf_logged_at.map(|t| now.duration_since(t)); + // Every 2 s, matching the other backends' submit-split cadence, and never before there + // are enough samples for a p99 to mean anything. + if self.perf_us.len() < 30 || since.is_some_and(|d| d.as_secs() < 2) { + if self.perf_logged_at.is_none() { + self.perf_logged_at = Some(now); + } + return; + } + self.perf_logged_at = Some(now); + let mut s = std::mem::take(&mut self.perf_us); + s.sort_unstable(); + let n = s.len() as u64; + let mean = s.iter().map(|&v| u64::from(v)).sum::() / n.max(1); + tracing::info!( + frames = n, + mean_us = mean, + p50_us = pct(&s, 0.50), + p99_us = pct(&s, 0.99), + max_us = *s.last().unwrap_or(&0), + depth = self.max_inflight, + "pyrowave encode, submit->AU (CSC + encode + fence wait + packetize). Under a \ + GPU-bound game this is the number the global-priority queue exists to protect — \ + watch p99, not the mean. At depth > 1 it includes one loop period of pipelining" + ); + } + pub fn open( width: u32, height: u32, @@ -467,6 +726,7 @@ impl PyroWaveEncoder { instance_ci: Box::new(vk::InstanceCreateInfo::default()), _queue_prio: Box::new([1.0f32]), _queue_ci: Box::new([vk::DeviceQueueCreateInfo::default()]), + _queue_gp: Box::new([vk::DeviceQueueGlobalPriorityCreateInfoKHR::default()]), _dev_exts: vec![ ash::khr::external_memory_fd::NAME.as_ptr(), ash::ext::external_memory_dma_buf::NAME.as_ptr(), @@ -487,8 +747,14 @@ impl PyroWaveEncoder { // error arm. From the device on, a partially-constructed `Self` (below) makes the // existing `Drop` the sole unwind path — these used to be a dozen `?`s that each leaked // everything created before them. - // SAFETY: plain physical-device queries on the live instance just created, and a - // `create_device` whose create-infos are pinned in `hold` for the call's duration. + // SAFETY: plain physical-device queries on the live instance just created, and + // `create_device` calls whose create-infos are pinned in `hold` for each call's duration. + // The global-priority ladder (WP14 step 4) may call `create_device` several times; every + // attempt reads the SAME pinned `hold`, and the only thing that varies between attempts is + // `hold._queue_gp[0].global_priority` (a plain enum field) and, for the final attempt, + // `hold._queue_ci[0].p_next` being nulled. Both live in `Box`es owned by `hold`, so the + // pointers `device_ci` holds stay valid across the retries; a failed `create_device` does + // not consume or invalidate its create-info, so re-passing it is sound. let selected = (|| unsafe { // The SAME selector `capture_modifiers` uses, so the two can never disagree about // the device (see `select_physical_device` — including why the selection itself is @@ -585,18 +851,117 @@ impl PyroWaveEncoder { ); vk::QUEUE_FAMILY_EXTERNAL }; + // VK_KHR_global_priority (WP14 step 4): PyroWave encodes on the SAME shader cores a + // game saturates, and `encode_gpu_synchronous` measurably collapses under that load + // (patch 0005's header: ~2 ms → 15-18 ms at 95 % game load on an RTX 4090). An + // elevated global-priority queue is the actual compute-PREEMPTION lever — unlike a + // process-priority raise, which only orders submission. The vendored patch requests + // it, but it is gated `if (!inherit_info)` and Linux passes its OWN create-infos, so + // Granite takes the inherit branch and the patch has never done anything here. This + // is the Linux half. Must be pushed BEFORE the count/as_ptr wiring below, exactly + // like queue_family_foreign above. + let gp_candidates = + queue_priority_candidates(std::env::var("PYROWAVE_QUEUE_PRIORITY").ok().as_deref()); + // Enable whichever alias the driver advertises (KHR = the promoted name), mirroring + // pf-zerocopy's VkBridge probe so the two can never disagree about the spelling. + let gp_ext = + if crate::vk_util::ext_advertised(&dev_ext_props, vk::KHR_GLOBAL_PRIORITY_NAME) { + Some(vk::KHR_GLOBAL_PRIORITY_NAME) + } else if crate::vk_util::ext_advertised( + &dev_ext_props, + vk::EXT_GLOBAL_PRIORITY_NAME, + ) { + Some(vk::EXT_GLOBAL_PRIORITY_NAME) + } else { + None + }; + let gp = gp_ext.filter(|_| !gp_candidates.is_empty()); + if let Some(name) = gp { + hold._dev_exts.push(name.as_ptr()); + } + hold._queue_ci[0] = vk::DeviceQueueCreateInfo::default().queue_family_index(family); hold._queue_ci[0].queue_count = 1; hold._queue_ci[0].p_queue_priorities = hold._queue_prio.as_ptr(); + if gp.is_some() { + hold._queue_ci[0].p_next = &*hold._queue_gp as *const _ as *const std::ffi::c_void; + } hold.device_ci.p_next = &*hold._feat2 as *const _ as *const std::ffi::c_void; hold.device_ci.queue_create_info_count = 1; hold.device_ci.p_queue_create_infos = hold._queue_ci.as_ptr(); hold.device_ci.enabled_extension_count = hold._dev_exts.len() as u32; hold.device_ci.pp_enabled_extension_names = hold._dev_exts.as_ptr(); - let device = instance - .create_device(pd, &hold.device_ci, None) - .context("create device")?; + // The downgrade ladder, mirroring the C patch: try each class in turn, step down only + // on a REFUSAL, and if every class is refused create with no global priority at all. + // A refused class must NEVER fail the open — that graceful property is the entire + // reason patch 0005 was kept despite its negative RTX/WDDM measurement, and it matters + // more here: this path is reached only by a negotiated PyroWave session, so a hard + // error is a dead stream rather than a fallback to another encoder. + let mut chosen = None; + let mut device = None; + for want in &gp_candidates { + if gp.is_none() { + break; + } + hold._queue_gp[0].global_priority = *want; + match instance.create_device(pd, &hold.device_ci, None) { + Ok(d) => { + chosen = Some(*want); + device = Some(d); + break; + } + Err(e) if priority_refused(e) => { + tracing::debug!( + priority = ?want, + error = ?e, + "pyrowave: global queue priority not permitted — downgrading" + ); + } + Err(e) => { + return Err(e).context("create device"); + } + } + } + let device = match device { + Some(d) => { + tracing::info!( + priority = ?chosen, + ext = ?gp, + "pyrowave: elevated global queue priority (the encode dispatch preempts a \ + GPU-bound game where the driver honors it)" + ); + d + } + None => { + // Either nothing was requested (`off`, or no extension), or every class was + // refused. EITHER WAY the retained create-info must now describe a device + // created WITHOUT a priority chain — `pyrowave_create_device` keeps + // `device_ci` for the device's lifetime and Granite reads it back through + // `get_existing_create_info()`. Leaving `p_next` pointing at the + // global-priority struct here would hand Granite a chain the device was not + // created with. (The extension itself stays enabled and that is correct: it + // IS enabled on the device, it just carries no request.) + hold._queue_ci[0].p_next = std::ptr::null(); + if !gp_candidates.is_empty() && gp.is_some() { + // MEASURED on .21 (RTX 5070 Ti, NVIDIA 610.43.02, 2026-08-08), and it is + // not a vendor quirk: an unprivileged host is refused EVERY class, and the + // same binary with `cap_sys_nice+ep` is granted REALTIME on the first + // attempt. So this arm is the normal state of a packaged host today, the + // lever is inert until the capability ships, and the message has to say + // which capability rather than leave an operator guessing. + tracing::warn!( + "pyrowave: every global queue priority class was refused — encoding \ + at default priority. The GPU-preemption lever is INERT without \ + CAP_SYS_NICE on the host binary (measured on both NVIDIA and RADV); \ + PYROWAVE_QUEUE_PRIORITY=off silences this" + ); + } + instance + .create_device(pd, &hold.device_ci, None) + .context("create device")? + } + }; Ok((pd, family, device, foreign_qfi)) })(); let (pd, family, device, foreign_qfi) = match selected { @@ -613,7 +978,7 @@ impl PyroWaveEncoder { // Construct `Self` NOW, every not-yet-created resource at its null value, and assign // into it as resources come up. Any `?` from here drops `me`, and the existing `Drop` // tears down exactly the prefix that exists: it `device_wait_idle()`s first, null-guards - // `pw_enc` (`pyrowave_encoder_destroy` dereferences before deleting), + // `pw_encs` (`pyrowave_encoder_destroy` dereferences before deleting), // `pyrowave_device_destroy(null)` is a plain `delete nullptr` (pyrowave_c.cpp) and // every `vkDestroy*`/`vkFree*` of a VK_NULL_HANDLE is the spec-defined no-op. One // teardown path serves both the error unwind and the normal drop, so an open-path leak @@ -629,43 +994,33 @@ impl PyroWaveEncoder { mem_props, _hold: hold, pw_dev: std::ptr::null_mut(), - pw_enc: std::ptr::null_mut(), + pw_encs: vec![std::ptr::null_mut(); SLOTS], + wire_seq: 0, csc_pipe: vk::Pipeline::null(), csc_layout: vk::PipelineLayout::null(), csc_dsl: vk::DescriptorSetLayout::null(), csc_pool: vk::DescriptorPool::null(), - csc_set: vk::DescriptorSet::null(), sampler: vk::Sampler::null(), - y_img: vk::Image::null(), - y_mem: vk::DeviceMemory::null(), - y_view: vk::ImageView::null(), - uv_img: vk::Image::null(), - uv_mem: vk::DeviceMemory::null(), - uv_view: vk::ImageView::null(), - cursor_img: vk::Image::null(), - cursor_mem: vk::DeviceMemory::null(), - cursor_view: vk::ImageView::null(), - cursor_stage: vk::Buffer::null(), - cursor_stage_mem: vk::DeviceMemory::null(), - cursor_serial: u64::MAX, - cursor_ready: false, import_cache: Vec::new(), - cpu_img: None, - cpu_stage: None, cpu_expand: Vec::new(), cmd_pool: vk::CommandPool::null(), - cmd: vk::CommandBuffer::null(), - fence: vk::Fence::null(), - gpu_pending: false, + slots: (0..SLOTS).map(|_| Slot::null()).collect(), + next_slot: 0, + inflight: VecDeque::new(), + // PW5: depth 1 still. Stage 4 allocates the capacity; stage 6 spends it. + max_inflight: 1, width: w, height: h, fps, chroma444, frame_budget: budget_for(bitrate, fps), + perf_us: Vec::new(), + perf_logged_at: None, wire_chunk: None, wire_budget: crate::pyrowave_wire::WireBudget::new(), bitstream: Vec::new(), pending: VecDeque::new(), + chunker: None, frame_count: 0, }; @@ -720,38 +1075,18 @@ impl PyroWaveEncoder { pw::pyrowave_chroma_subsampling_PYROWAVE_CHROMA_SUBSAMPLING_420 }, }; - pw_check( - pw::pyrowave_encoder_create(&einfo, &mut me.pw_enc), - "encoder_create", - )?; + for i in 0..SLOTS { + pw_check( + pw::pyrowave_encoder_create(&einfo, &mut me.pw_encs[i]), + "encoder_create", + )?; + } // ---- CSC planes: full-res R8 luma + RG8 chroma (half-res for 4:2:0, full-res for // 4:4:4), storage-written by the CSC and sampled directly by pyrowave (R/G view // swizzles synthesize Cb/Cr) ---- let device = me.device.clone(); // cheap fn-table clone; lets `me.*` assignments interleave let (cw, ch) = if chroma444 { (w, h) } else { (w / 2, h / 2) }; - let (y_img, y_mem, y_view) = make_plain_image( - &device, - &me.mem_props, - vk::Format::R8_UNORM, - w, - h, - vk::ImageUsageFlags::STORAGE | vk::ImageUsageFlags::SAMPLED, - )?; - me.y_img = y_img; - me.y_mem = y_mem; - me.y_view = y_view; - let (uv_img, uv_mem, uv_view) = make_plain_image( - &device, - &me.mem_props, - vk::Format::R8G8_UNORM, - cw, - ch, - vk::ImageUsageFlags::STORAGE | vk::ImageUsageFlags::SAMPLED, - )?; - me.uv_img = uv_img; - me.uv_mem = uv_mem; - me.uv_view = uv_view; // ---- CSC compute pipeline (same shader + layout as vulkan_video.rs) ---- me.sampler = device.create_sampler( @@ -816,91 +1151,140 @@ impl PyroWaveEncoder { device.destroy_shader_module(shader, None); me.csc_pipe = pipe_res.map_err(|(_, e)| e)?[0]; + // Pool sized for ALL slots: 2 combined-image-samplers (binding 0 RGB + binding 3 cursor) + // and 2 storage images (Y, UV) per set. let pool_sizes = [ vk::DescriptorPoolSize::default() .ty(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) - // binding 0 (RGB) + binding 3 (cursor). - .descriptor_count(2), + .descriptor_count(2 * SLOTS as u32), vk::DescriptorPoolSize::default() .ty(vk::DescriptorType::STORAGE_IMAGE) - .descriptor_count(2), + .descriptor_count(2 * SLOTS as u32), ]; me.csc_pool = device.create_descriptor_pool( &vk::DescriptorPoolCreateInfo::default() - .max_sets(1) + .max_sets(SLOTS as u32) .pool_sizes(&pool_sizes), None, )?; - me.csc_set = device.allocate_descriptor_sets( - &vk::DescriptorSetAllocateInfo::default() - .descriptor_pool(me.csc_pool) - .set_layouts(&dsls), - )?[0]; - // Cursor overlay: fixed CURSOR_MAX² RGBA8 sampled image + host staging (bound at binding 3). - let (cursor_img, cursor_mem, cursor_view) = make_plain_image( - &device, - &me.mem_props, - vk::Format::R8G8B8A8_UNORM, - CURSOR_MAX, - CURSOR_MAX, - vk::ImageUsageFlags::SAMPLED | vk::ImageUsageFlags::TRANSFER_DST, - )?; - me.cursor_img = cursor_img; - me.cursor_mem = cursor_mem; - me.cursor_view = cursor_view; - let (cursor_stage, cursor_stage_mem) = make_host_buffer( - &device, - &me.mem_props, - (CURSOR_MAX * CURSOR_MAX * 4) as u64, - vk::BufferUsageFlags::TRANSFER_SRC, - )?; - me.cursor_stage = cursor_stage; - me.cursor_stage_mem = cursor_stage_mem; - // Bindings 1/2 (Y, UV storage targets) + 3 (cursor sampler) are fixed for the encoder's life. - let yi = [vk::DescriptorImageInfo::default() - .image_view(me.y_view) - .image_layout(vk::ImageLayout::GENERAL)]; - let uvi = [vk::DescriptorImageInfo::default() - .image_view(me.uv_view) - .image_layout(vk::ImageLayout::GENERAL)]; - let curi = [vk::DescriptorImageInfo::default() - .sampler(me.sampler) - .image_view(me.cursor_view) - .image_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL)]; - device.update_descriptor_sets( - &[ - vk::WriteDescriptorSet::default() - .dst_set(me.csc_set) - .dst_binding(1) - .descriptor_type(vk::DescriptorType::STORAGE_IMAGE) - .image_info(&yi), - vk::WriteDescriptorSet::default() - .dst_set(me.csc_set) - .dst_binding(2) - .descriptor_type(vk::DescriptorType::STORAGE_IMAGE) - .image_info(&uvi), - vk::WriteDescriptorSet::default() - .dst_set(me.csc_set) - .dst_binding(3) - .descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) - .image_info(&curi), - ], - &[], - ); - me.cmd_pool = device.create_command_pool( &vk::CommandPoolCreateInfo::default() .queue_family_index(family) .flags(vk::CommandPoolCreateFlags::RESET_COMMAND_BUFFER), None, )?; - me.cmd = device.allocate_command_buffers( - &vk::CommandBufferAllocateInfo::default() - .command_pool(me.cmd_pool) - .level(vk::CommandBufferLevel::PRIMARY) - .command_buffer_count(1), - )?[0]; - me.fence = device.create_fence(&vk::FenceCreateInfo::default(), None)?; + + // ---- the per-frame resource sets (PW5 stage 4) ---- + // Each iteration builds ONE complete `Slot` and assigns as it goes, so a failure part-way + // leaves the earlier slots fully formed and the rest null — which `Drop` handles, since + // every `vkDestroy*` of VK_NULL_HANDLE is the spec-defined no-op. + for i in 0..SLOTS { + let (y_img, y_mem, y_view) = make_plain_image( + &device, + &me.mem_props, + vk::Format::R8_UNORM, + w, + h, + vk::ImageUsageFlags::STORAGE | vk::ImageUsageFlags::SAMPLED, + )?; + me.slots[i].y_img = y_img; + me.slots[i].y_mem = y_mem; + me.slots[i].y_view = y_view; + let (uv_img, uv_mem, uv_view) = make_plain_image( + &device, + &me.mem_props, + vk::Format::R8G8_UNORM, + cw, + ch, + vk::ImageUsageFlags::STORAGE | vk::ImageUsageFlags::SAMPLED, + )?; + me.slots[i].uv_img = uv_img; + me.slots[i].uv_mem = uv_mem; + me.slots[i].uv_view = uv_view; + // Cursor overlay: fixed CURSOR_MAX² RGBA8 sampled image + host staging (binding 3). + let (cursor_img, cursor_mem, cursor_view) = make_plain_image( + &device, + &me.mem_props, + vk::Format::R8G8B8A8_UNORM, + CURSOR_MAX, + CURSOR_MAX, + vk::ImageUsageFlags::SAMPLED | vk::ImageUsageFlags::TRANSFER_DST, + )?; + me.slots[i].cursor_img = cursor_img; + me.slots[i].cursor_mem = cursor_mem; + me.slots[i].cursor_view = cursor_view; + let (cursor_stage, cursor_stage_mem) = make_host_buffer( + &device, + &me.mem_props, + (CURSOR_MAX * CURSOR_MAX * 4) as u64, + vk::BufferUsageFlags::TRANSFER_SRC, + )?; + me.slots[i].cursor_stage = cursor_stage; + me.slots[i].cursor_stage_mem = cursor_stage_mem; + let csc_set = device.allocate_descriptor_sets( + &vk::DescriptorSetAllocateInfo::default() + .descriptor_pool(me.csc_pool) + .set_layouts(&dsls), + )?[0]; + me.slots[i].csc_set = csc_set; + // Bindings 1/2 (Y, UV storage targets) + 3 (cursor sampler) are fixed for the slot's + // life; only binding 0 (the frame's RGB view) is rewritten per frame, by `bind_rgb`, + // and THAT is why each slot needs its own set — see `Slot`. + let yi = [vk::DescriptorImageInfo::default() + .image_view(y_view) + .image_layout(vk::ImageLayout::GENERAL)]; + let uvi = [vk::DescriptorImageInfo::default() + .image_view(uv_view) + .image_layout(vk::ImageLayout::GENERAL)]; + let curi = [vk::DescriptorImageInfo::default() + .sampler(me.sampler) + .image_view(cursor_view) + .image_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL)]; + device.update_descriptor_sets( + &[ + vk::WriteDescriptorSet::default() + .dst_set(csc_set) + .dst_binding(1) + .descriptor_type(vk::DescriptorType::STORAGE_IMAGE) + .image_info(&yi), + vk::WriteDescriptorSet::default() + .dst_set(csc_set) + .dst_binding(2) + .descriptor_type(vk::DescriptorType::STORAGE_IMAGE) + .image_info(&uvi), + vk::WriteDescriptorSet::default() + .dst_set(csc_set) + .dst_binding(3) + .descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) + .image_info(&curi), + ], + &[], + ); + me.slots[i].cmd = device.allocate_command_buffers( + &vk::CommandBufferAllocateInfo::default() + .command_pool(me.cmd_pool) + .level(vk::CommandBufferLevel::PRIMARY) + .command_buffer_count(1), + )?[0]; + me.slots[i].fence = device.create_fence(&vk::FenceCreateInfo::default(), None)?; + } + + // What the extra slot actually COST, measured from the driver's own requirements rather + // than estimated from the dimensions — the plan's estimate is not evidence, and on an iGPU + // at 4K/4:4:4 this is the number that decides whether the capacity is affordable. The + // per-frame CPU staging is excluded because it is allocated lazily and only on the + // software-capture path. + let slot_bytes: u64 = [ + me.slots[0].y_img, + me.slots[0].uv_img, + me.slots[0].cursor_img, + ] + .iter() + .map(|&i| device.get_image_memory_requirements(i).size) + .sum::() + + device + .get_buffer_memory_requirements(me.slots[0].cursor_stage) + .size; let props = me.instance.get_physical_device_properties(pd); tracing::info!( @@ -908,21 +1292,29 @@ impl PyroWaveEncoder { mode = %format!("{w}x{h}@{fps}"), budget_kib = me.frame_budget / 1024, chroma = if chroma444 { "4:4:4" } else { "4:2:0" }, + slots = SLOTS, + slot_kib = slot_bytes / 1024, + slots_kib = slot_bytes * SLOTS as u64 / 1024, "PyroWave encoder open (intra-only wavelet, BT.709 limited)" ); Ok(me) } - /// Point CSC binding 0 at this frame's RGB view. - unsafe fn bind_rgb(&self, rgb_view: vk::ImageView) { + /// Point slot `slot`'s CSC binding 0 at this frame's RGB view. + /// + /// ⚠ This is the `vkUpdateDescriptorSets` the analysis flagged: writing a set that is still + /// bound by a PENDING command buffer violates VUID-vkUpdateDescriptorSets-None-03047. It is + /// safe because the set belongs to the slot we are about to record into, and that slot's + /// previous frame was retired before `submit` chose it. + unsafe fn bind_rgb(&self, slot: usize, rgb_view: vk::ImageView) { let ii = [vk::DescriptorImageInfo::default() .sampler(self.sampler) .image_view(rgb_view) .image_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL)]; self.device.update_descriptor_sets( &[vk::WriteDescriptorSet::default() - .dst_set(self.csc_set) + .dst_set(self.slots[slot].csc_set) .dst_binding(0) .descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) .image_info(&ii)], @@ -933,13 +1325,22 @@ impl PyroWaveEncoder { /// Cursor-as-metadata: bring the cursor image up to date for this frame and return the shader /// push constant `[origin_x, origin_y, size_w, size_h]` (size 0 ⇒ the CSC skips the blend). /// Records the small upload (only when the bitmap `serial` changed) + layout transition into - /// `cmd`, ahead of the CSC dispatch that samples binding 3. Encode is synchronous, so the single - /// shared image never races a prior frame; the first use transitions it to SHADER_READ_ONLY. - unsafe fn prep_cursor(&mut self, cursor: Option<&pf_frame::CursorOverlay>) -> Result<[i32; 4]> { + /// slot `slot`'s command buffer, ahead of the CSC dispatch that samples binding 3. + /// + /// PER SLOT since PW5 stage 4 — image, staging buffer and `cursor_serial` all. The old comment + /// said it outright: a single shared image was only safe because there was no in-flight + /// overlap to race. The cost of per-slot is that a changed bitmap uploads once per slot. + unsafe fn prep_cursor( + &mut self, + slot: usize, + cursor: Option<&pf_frame::CursorOverlay>, + ) -> Result<[i32; 4]> { let dev = self.device.clone(); - let cmd = self.cmd; - let img = self.cursor_img; - let ready = self.cursor_ready; + let cmd = self.slots[slot].cmd; + let img = self.slots[slot].cursor_img; + let stage = self.slots[slot].cursor_stage; + let stage_mem = self.slots[slot].cursor_stage_mem; + let ready = self.slots[slot].cursor_ready; let barrier = |old: vk::ImageLayout, new: vk::ImageLayout, ss, sa, ds, da| { vk::ImageMemoryBarrier2::default() .src_stage_mask(ss) @@ -957,20 +1358,16 @@ impl PyroWaveEncoder { Some(c) if !c.rgba.is_empty() => { let cw = c.w.min(CURSOR_MAX); let ch = c.h.min(CURSOR_MAX); - if self.cursor_serial != c.serial { + if self.slots[slot].cursor_serial != c.serial { let bytes = (cw as usize) * (ch as usize) * 4; - let ptr = dev.map_memory( - self.cursor_stage_mem, - 0, - bytes as u64, - vk::MemoryMapFlags::empty(), - )?; + let ptr = + dev.map_memory(stage_mem, 0, bytes as u64, vk::MemoryMapFlags::empty())?; std::ptr::copy_nonoverlapping( c.rgba.as_ptr(), ptr as *mut u8, bytes.min(c.rgba.len()), ); - dev.unmap_memory(self.cursor_stage_mem); + dev.unmap_memory(stage_mem); let old = if ready { vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL } else { @@ -989,7 +1386,7 @@ impl PyroWaveEncoder { ); dev.cmd_copy_buffer_to_image( cmd, - self.cursor_stage, + stage, img, vk::ImageLayout::TRANSFER_DST_OPTIMAL, &[vk::BufferImageCopy::default() @@ -1015,8 +1412,8 @@ impl PyroWaveEncoder { vk::AccessFlags2::SHADER_READ, )]), ); - self.cursor_serial = c.serial; - self.cursor_ready = true; + self.slots[slot].cursor_serial = c.serial; + self.slots[slot].cursor_ready = true; } Ok([c.x, c.y, cw as i32, ch as i32]) } @@ -1033,7 +1430,7 @@ impl PyroWaveEncoder { vk::AccessFlags2::SHADER_READ, )]), ); - self.cursor_ready = true; + self.slots[slot].cursor_ready = true; } Ok([0, 0, 0, 0]) } @@ -1090,12 +1487,20 @@ impl PyroWaveEncoder { } /// CPU RGB staging (software capture / smoke tests) — mirrors `vulkan_video.rs::ensure_cpu_rgb`. - unsafe fn ensure_cpu_rgb(&mut self, fmt: vk::Format, bytes: &[u8]) -> Result { + /// + /// PER SLOT since PW5 stage 4: the host writes this staging buffer, so writing it while a + /// previous frame's buffer-to-image copy is still pending would race that copy. + unsafe fn ensure_cpu_rgb( + &mut self, + slot: usize, + fmt: vk::Format, + bytes: &[u8], + ) -> Result { let dev = self.device.clone(); let (w, h) = (self.width, self.height); let need = (w * h * 4) as u64; - if self.cpu_img.map(|(_, _, _, f)| f) != Some(fmt) { - if let Some((i, m, v, _)) = self.cpu_img.take() { + if self.slots[slot].cpu_img.map(|(_, _, _, f)| f) != Some(fmt) { + if let Some((i, m, v, _)) = self.slots[slot].cpu_img.take() { dev.destroy_image_view(v, None); dev.destroy_image(i, None); dev.free_memory(m, None); @@ -1108,10 +1513,14 @@ impl PyroWaveEncoder { h, vk::ImageUsageFlags::SAMPLED | vk::ImageUsageFlags::TRANSFER_DST, )?; - self.cpu_img = Some((i, m, v, fmt)); + self.slots[slot].cpu_img = Some((i, m, v, fmt)); } - if self.cpu_stage.map(|(_, _, s)| s < need).unwrap_or(true) { - if let Some((b, m, _)) = self.cpu_stage.take() { + if self.slots[slot] + .cpu_stage + .map(|(_, _, s)| s < need) + .unwrap_or(true) + { + if let Some((b, m, _)) = self.slots[slot].cpu_stage.take() { dev.destroy_buffer(b, None); dev.free_memory(m, None); } @@ -1121,14 +1530,14 @@ impl PyroWaveEncoder { need, vk::BufferUsageFlags::TRANSFER_SRC, )?; - self.cpu_stage = Some((buf, mem, need)); + self.slots[slot].cpu_stage = Some((buf, mem, need)); } - let (_, m, _) = self.cpu_stage.unwrap(); + let (_, m, _) = self.slots[slot].cpu_stage.unwrap(); let p = dev.map_memory(m, 0, vk::WHOLE_SIZE, vk::MemoryMapFlags::empty())? as *mut u8; let n = bytes.len().min(need as usize); std::ptr::copy_nonoverlapping(bytes.as_ptr(), p, n); dev.unmap_memory(m); - Ok(self.cpu_img.unwrap().2) + Ok(self.slots[slot].cpu_img.unwrap().2) } /// The per-frame budget handed to pyrowave rate control: `frame_budget`, deflated by the @@ -1141,14 +1550,19 @@ impl PyroWaveEncoder { } } - /// One frame, synchronously: ingest → CSC → pyrowave encode (recorded into our command - /// buffer) → submit + fence wait (sub-ms) → packetize into an `EncodedFrame`. - unsafe fn encode_frame(&mut self, frame: &CapturedFrame) -> Result<()> { + /// The SUBMIT half of one frame (PW5 stage 3): ingest → CSC → pyrowave encode, recorded into + /// our command buffer → queue-submit → **return**. The fence wait and packetize moved to + /// [`wait_and_packetize`](Self::wait_and_packetize), which is where every other backend in + /// this crate has always had them. + /// + /// On success exactly one [`InFlight`] is pushed. On failure nothing is pushed and the command + /// buffer has been reset (see the error-arm note inside) — the caller may submit again. + unsafe fn submit_frame(&mut self, frame: &CapturedFrame, t0: std::time::Instant) -> Result<()> { // A failed `reset()` leaves the encoder destroyed and null. Callers today turn that into // a session error and never resubmit, but a null here would be a use-after-free inside // pyrowave rather than a clean error — so fail loudly instead of relying on that. anyhow::ensure!( - !self.pw_enc.is_null(), + self.pw_encs.iter().all(|e| !e.is_null()), "pyrowave: encode after a failed reset (encoder was destroyed and not rebuilt)" ); let dev = self.device.clone(); @@ -1198,16 +1612,23 @@ impl PyroWaveEncoder { // which the next `begin` may implicitly reset. // Resolved before the closure (which borrows `self` mutably for the recording calls). let rate_budget = self.rate_budget(); + // THE slot this frame owns for its whole life — command buffer, fence, descriptor set, + // y/uv images, cursor image and CPU staging (PW5 stage 4). `submit` guaranteed it is free + // by draining to `max_inflight - 1` before calling here. + let slot = self.next_slot; + let seq = self.wire_seq; + let cmd = self.slots[slot].cmd; + let fence = self.slots[slot].fence; let record_and_submit = (|| -> Result<()> { dev.begin_command_buffer( - self.cmd, + cmd, &vk::CommandBufferBeginInfo::default() .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), )?; // Cursor-as-metadata: refresh the cursor image (only when the bitmap changed) + get the // shader push constant. Recorded into `self.cmd` before the CSC dispatch samples binding 3. - let cursor_pc = self.prep_cursor(frame.cursor.as_ref())?; + let cursor_pc = self.prep_cursor(slot, frame.cursor.as_ref())?; // ---- ingest RGB (same barrier discipline as vulkan_video.rs) ---- let rgb_view = match &frame.payload { @@ -1234,7 +1655,7 @@ impl PyroWaveEncoder { .image(img) .subresource_range(color_range(0)); dev.cmd_pipeline_barrier2( - self.cmd, + cmd, &vk::DependencyInfo::default().image_memory_barriers(&[acq]), ); view @@ -1247,13 +1668,13 @@ impl PyroWaveEncoder { normalize_cpu_rgb(frame.format, bytes, &mut scratch, false); let fmt = pixel_to_vk(norm_fmt).context("unsupported CPU pixel format"); let view = match fmt { - Ok(f) => self.ensure_cpu_rgb(f, norm_bytes), + Ok(f) => self.ensure_cpu_rgb(slot, f, norm_bytes), Err(e) => Err(e), }; self.cpu_expand = scratch; let view = view?; - let (img, ..) = self.cpu_img.unwrap(); - let (stage, ..) = self.cpu_stage.unwrap(); + let (img, ..) = self.slots[slot].cpu_img.unwrap(); + let (stage, ..) = self.slots[slot].cpu_stage.unwrap(); let to_dst = vk::ImageMemoryBarrier2::default() .src_stage_mask(vk::PipelineStageFlags2::NONE) .src_access_mask(vk::AccessFlags2::NONE) @@ -1264,11 +1685,11 @@ impl PyroWaveEncoder { .image(img) .subresource_range(color_range(0)); dev.cmd_pipeline_barrier2( - self.cmd, + cmd, &vk::DependencyInfo::default().image_memory_barriers(&[to_dst]), ); dev.cmd_copy_buffer_to_image( - self.cmd, + cmd, stage, img, vk::ImageLayout::TRANSFER_DST_OPTIMAL, @@ -1294,18 +1715,19 @@ impl PyroWaveEncoder { .image(img) .subresource_range(color_range(0)); dev.cmd_pipeline_barrier2( - self.cmd, + cmd, &vk::DependencyInfo::default().image_memory_barriers(&[to_read]), ); view } _ => bail!("pyrowave: unsupported FramePayload (need Dmabuf or Cpu RGB)"), }; - self.bind_rgb(rgb_view); + self.bind_rgb(slot, rgb_view); - // y/uv -> GENERAL for the CSC's storage writes (discard prior contents — the previous - // frame's encode already completed under our synchronous fence, which is also the - // "execution barrier before writing to images" pyrowave's contract asks for). + // y/uv -> GENERAL for the CSC's storage writes (discard prior contents — this SLOT's + // previous frame was retired before `submit` chose it, which is also the "execution + // barrier before writing to images" pyrowave's contract asks for). + let (y_img, uv_img) = (self.slots[slot].y_img, self.slots[slot].uv_img); let to_general = |img| { vk::ImageMemoryBarrier2::default() .src_stage_mask(vk::PipelineStageFlags2::NONE) @@ -1318,17 +1740,17 @@ impl PyroWaveEncoder { .subresource_range(color_range(0)) }; dev.cmd_pipeline_barrier2( - self.cmd, + cmd, &vk::DependencyInfo::default() - .image_memory_barriers(&[to_general(self.y_img), to_general(self.uv_img)]), + .image_memory_barriers(&[to_general(y_img), to_general(uv_img)]), ); - dev.cmd_bind_pipeline(self.cmd, vk::PipelineBindPoint::COMPUTE, self.csc_pipe); + dev.cmd_bind_pipeline(cmd, vk::PipelineBindPoint::COMPUTE, self.csc_pipe); dev.cmd_bind_descriptor_sets( - self.cmd, + cmd, vk::PipelineBindPoint::COMPUTE, self.csc_layout, 0, - &[self.csc_set], + &[self.slots[slot].csc_set], &[], ); let mut pc_bytes = [0u8; 16]; @@ -1336,7 +1758,7 @@ impl PyroWaveEncoder { pc_bytes[i * 4..i * 4 + 4].copy_from_slice(&v.to_ne_bytes()); } dev.cmd_push_constants( - self.cmd, + cmd, self.csc_layout, vk::ShaderStageFlags::COMPUTE, 0, @@ -1344,9 +1766,9 @@ impl PyroWaveEncoder { ); // 4:2:0: one invocation per 2x2 luma block (per chroma sample); 4:4:4: per pixel. if self.chroma444 { - dev.cmd_dispatch(self.cmd, w.div_ceil(8), h.div_ceil(8), 1); + dev.cmd_dispatch(cmd, w.div_ceil(8), h.div_ceil(8), 1); } else { - dev.cmd_dispatch(self.cmd, (w / 2).div_ceil(8), (h / 2).div_ceil(8), 1); + dev.cmd_dispatch(cmd, (w / 2).div_ceil(8), (h / 2).div_ceil(8), 1); } // CSC storage writes -> pyrowave's sampled reads (images stay GENERAL — the layout @@ -1363,9 +1785,9 @@ impl PyroWaveEncoder { .subresource_range(color_range(0)) }; dev.cmd_pipeline_barrier2( - self.cmd, + cmd, &vk::DependencyInfo::default() - .image_memory_barriers(&[to_sampled(self.y_img), to_sampled(self.uv_img)]), + .image_memory_barriers(&[to_sampled(y_img), to_sampled(uv_img)]), ); // ---- pyrowave encode, recorded into OUR command buffer ---- @@ -1392,7 +1814,7 @@ impl PyroWaveEncoder { let buffers = pw::pyrowave_gpu_buffers { planes: [ plane( - self.y_img, + y_img, w, h, r8, @@ -1403,14 +1825,14 @@ impl PyroWaveEncoder { // The view extent is the chroma IMAGE's own mip0 extent (it's a separate // image, not a planar aspect): half-res for 4:2:0, full-res for 4:4:4. plane( - self.uv_img, + uv_img, if self.chroma444 { w } else { w / 2 }, if self.chroma444 { h } else { h / 2 }, rg8, pw::VkComponentSwizzle_VK_COMPONENT_SWIZZLE_R, ), plane( - self.uv_img, + uv_img, if self.chroma444 { w } else { w / 2 }, if self.chroma444 { h } else { h / 2 }, rg8, @@ -1423,10 +1845,22 @@ impl PyroWaveEncoder { }; pw::pyrowave_device_set_command_buffer( self.pw_dev, - self.cmd.as_raw() as usize as pw::VkCommandBuffer, + cmd.as_raw() as usize as pw::VkCommandBuffer, ); + // ⚠ THE LANDMINE (PW5 stage 5). Stamp OUR monotonic counter before the encode, or + // the two alternating handles emit 1,1,2,2,3,3… and the decoder reads each repeat as + // more blocks of the same frame — half the frames silently swallowed, on every client. + // Needs `patches/0007-encoder-sequence-override.patch`; the round-trip test + // `wire_sequence_increments_across_alternating_handles` is what keeps it honest. + pw_check( + pw::pyrowave_encoder_set_next_sequence( + self.pw_encs[slot], + seq & pw::PYROWAVE_SEQUENCE_MASK, + ), + "set_next_sequence", + )?; let enc_res = pw::pyrowave_encoder_encode_gpu_synchronous( - self.pw_enc, + self.pw_encs[slot], std::ptr::null(), std::ptr::null(), &buffers, @@ -1435,26 +1869,57 @@ impl PyroWaveEncoder { pw::pyrowave_device_set_command_buffer(self.pw_dev, std::ptr::null_mut()); pw_check(enc_res, "encode_gpu_synchronous")?; - dev.end_command_buffer(self.cmd)?; - dev.reset_fences(&[self.fence])?; - let cmds = [self.cmd]; + dev.end_command_buffer(cmd)?; + dev.reset_fences(&[fence])?; + let cmds = [cmd]; dev.queue_submit( self.queue, &[vk::SubmitInfo::default().command_buffers(&cmds)], - self.fence, + fence, )?; Ok(()) })(); if let Err(e) = record_and_submit { // SAFETY: on every closure error arm the buffer is RECORDING/INVALID/EXECUTABLE — // never PENDING (nothing was enqueued) — and the pool allows the reset. - let _ = dev.reset_command_buffer(self.cmd, vk::CommandBufferResetFlags::empty()); + let _ = dev.reset_command_buffer(cmd, vk::CommandBufferResetFlags::empty()); return Err(e); } - self.gpu_pending = true; - dev.wait_for_fences(&[self.fence], true, 5_000_000_000) + // Submitted: from here the GPU may be executing, and NOTHING may touch `cmd`, the y/uv + // images or `csc_set` until this entry is retired by `wait_and_packetize`. + self.next_slot = (slot + 1) % SLOTS; + // Advance only on SUCCESS: a frame that never reached the wire must not burn a sequence + // value (a gap reads as a restart, which is right for a DROPPED frame and wrong for one + // that was never emitted at all). + self.wire_seq = self.wire_seq.wrapping_add(1); + self.inflight.push_back(InFlight { + slot, + seq: (seq & pw::PYROWAVE_SEQUENCE_MASK) as u8, + pts_ns: frame.pts_ns, + cap: self.frame_budget + BS_SLACK, + wire_chunk: self.wire_chunk, + t0, + }); + Ok(()) + } + + /// The POLL half of one frame (PW5 stage 3): wait the oldest in-flight frame's fence, then + /// packetize its bitstream into an `EncodedFrame` on `pending`. + /// + /// ⚠ The fence wait's failure path deliberately does NOT reset the command buffer: a timeout + /// leaves it PENDING, where a reset violates VUID-vkResetCommandBuffer-commandBuffer-00045. + /// The in-flight entry is likewise NOT popped on that failure — it is what tells `reset()` + /// there is still live GPU work to re-wait before the encoder object may be destroyed. + unsafe fn wait_and_packetize(&mut self) -> Result<()> { + let Some(fr) = self.inflight.front().copied() else { + return Ok(()); + }; + let dev = self.device.clone(); + dev.wait_for_fences(&[self.slots[fr.slot].fence], true, 5_000_000_000) .context("pyrowave encode fence")?; - self.gpu_pending = false; + // Waited and signaled: the command buffer is INVALID (one-time submit), which the next + // `begin` may implicitly reset, and the GPU is done with this frame's resources. + self.inflight.pop_front(); // ---- packetize ---- // Dense (default): boundary = whole buffer → the AU is exactly one pyrowave packet. @@ -1463,23 +1928,26 @@ impl PyroWaveEncoder { // self-delimiting packets — the client windows its parse and a lost shard costs // only those blocks. Padding cost is small: the packetizer fills close to the // boundary by design. - let cap = self.frame_budget + BS_SLACK; + // `fr.cap`/`fr.wire_chunk`, NOT the live fields: `reconfigure_bitrate` and + // `set_wire_chunking` may have landed since this frame was submitted, and packetizing at a + // boundary the frame was not rate-controlled for is a spurious failure. + let cap = fr.cap; self.bitstream.resize(cap, 0); // Chunked mode reserves the 4-byte window prefix from the packetize boundary (shared helper). - let boundary = crate::pyrowave_wire::packet_boundary(self.wire_chunk, cap); + let boundary = crate::pyrowave_wire::packet_boundary(fr.wire_chunk, cap); let mut n: usize = 0; pw_check( - pw::pyrowave_encoder_compute_num_packets(self.pw_enc, boundary, &mut n), + pw::pyrowave_encoder_compute_num_packets(self.pw_encs[fr.slot], boundary, &mut n), "compute_num_packets", )?; - if n == 0 || (self.wire_chunk.is_none() && n != 1) { + if n == 0 || (fr.wire_chunk.is_none() && n != 1) { bail!("pyrowave: unexpected packet count {n} at boundary {boundary}"); } let mut packets = vec![pw::pyrowave_packet { offset: 0, size: 0 }; n]; let mut out_n: usize = 0; pw_check( pw::pyrowave_encoder_packetize( - self.pw_enc, + self.pw_encs[fr.slot], packets.as_mut_ptr(), boundary, &mut out_n, @@ -1494,40 +1962,84 @@ impl PyroWaveEncoder { // blacks. (Linux capture has no HDR path, so this side never stamps BT.2020/PQ.) if let Some(p) = packets.first() { crate::pyrowave_wire::stamp_color_bits(&mut self.bitstream, p.offset, false); + // Self-check on the ONE thing a dropped vendored patch would break silently. Without + // `0007-encoder-sequence-override.patch` the two handles count independently, the wire + // reads 1,1,2,2,3,3..., and every client's decoder folds each repeated value into the + // previous frame — half the frames gone, no error anywhere. A re-vendor that loses the + // patch would not fail to build; it would fail on glass, subtly. Two byte reads per + // frame to make that loud instead. Once per process: if it is wrong it is wrong for + // every frame. + if crate::pyrowave_wire::wire_sequence(&self.bitstream, p.offset) != Some(fr.seq) { + static WARNED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + if !WARNED.swap(true, std::sync::atomic::Ordering::Relaxed) { + tracing::error!( + expected = fr.seq, + got = ?crate::pyrowave_wire::wire_sequence(&self.bitstream, p.offset), + "pyrowave: the wire sequence counter is NOT what we stamped — \ + patches/0007-encoder-sequence-override.patch is missing or ineffective. \ + With two alternating encoder handles this silently halves the frame rate \ + on every client" + ); + } + } } // Frame into the wire AU via the shared helper (byte-identical on Linux + Windows): the dense // single packet, or the datagram-aligned windowed AU (§4.4). let pkts: Vec<(usize, usize)> = packets.iter().map(|p| (p.offset, p.size)).collect(); - let au = crate::pyrowave_wire::build_au(&pkts, &self.bitstream, self.wire_chunk); - if self.wire_chunk.is_some() { + let au = crate::pyrowave_wire::build_au(&pkts, &self.bitstream, fr.wire_chunk); + if fr.wire_chunk.is_some() { let raw: usize = pkts.iter().map(|&(_, s)| s).sum(); self.wire_budget.observe(raw, au.len()); } self.frame_count += 1; self.pending.push_back(EncodedFrame { data: au, - pts_ns: frame.pts_ns, + pts_ns: fr.pts_ns, // Every frame is independently decodable — SOF/keyframe on each AU is the codec's // whole recovery story (plan §1.2). keyframe: true, recovery_anchor: false, - chunk_aligned: self.wire_chunk.is_some(), + chunk_aligned: fr.wire_chunk.is_some(), }); + // submit→AU, the same quantity `92326312` measured before the split, so stage 0's + // baseline stays comparable. Stamped here rather than in `submit` because that is where + // the AU now becomes readable. + self.note_encode_us(fr.t0.elapsed().as_micros() as u32); + Ok(()) + } + + /// Retire in-flight frames until at most `keep` remain. Used by `submit` (make room before + /// recording) and by `flush`/`poll` (drain). + unsafe fn drain_to(&mut self, keep: usize) -> Result<()> { + while self.inflight.len() > keep { + self.wait_and_packetize()?; + } Ok(()) } } impl Encoder for PyroWaveEncoder { fn submit(&mut self, frame: &CapturedFrame) -> Result<()> { - // SAFETY: single-threaded encoder; `encode_frame` records/submits on handles this - // struct owns and waits its own fence before touching results. Command-buffer state on - // failure is `encode_frame`'s own business now: its record-and-submit closure resets the - // buffer on every pre-submit failure, and its post-submit failures (fence timeout — - // buffer possibly PENDING) deliberately do NOT reset, because that violates + // `PUNKTFUNK_PERF` encode split (kept above the SAFETY comment so that comment stays + // attached to the block it proves — the crate denies undocumented unsafe blocks). + let t0 = std::time::Instant::now(); + // SAFETY: single-threaded encoder; both halves work on handles this struct owns. + // Command-buffer state on failure is `submit_frame`'s own business: its record-and-submit + // closure resets the buffer on every pre-submit failure, and the fence wait's failure + // (buffer possibly PENDING) deliberately does NOT reset, because that violates // VUID-vkResetCommandBuffer-commandBuffer-00045; the blanket reset that used to live // here fired on exactly that path. Recovery (`reset()`/`Drop`) waits the device idle // before anything touches `cmd` again. - unsafe { self.encode_frame(frame) } + unsafe { + // Make room before recording. This is THE invariant that makes the slot + // `submit_frame` is about to pick provably free: at most `max_inflight - 1` frames may + // still be in flight, and `max_inflight <= SLOTS`, so the slot `next_slot` points at + // was retired. The host loop polls after every submit so this is normally a no-op; it + // is here for callers that do not (the `spike` subcommand, the hardware smoke tests). + self.drain_to(self.max_inflight.saturating_sub(1))?; + self.submit_frame(frame, t0) + } } fn caps(&self) -> EncoderCaps { @@ -1544,29 +2056,84 @@ impl Encoder for PyroWaveEncoder { } fn poll(&mut self) -> Result> { + // Trait contract (PW6): each AU is drained through ONE method. Erroring beats + // double-emitting the bytes the chunk cursor already handed out (which would reach the + // wire twice, under the same frame index, and fail the receiver's retro-validation). + // Checked BEFORE the fence wait below: if a chunk cursor is open this call is a caller + // bug, and completing more work first would only widen the damage. + if self.chunker.is_some() { + bail!("pyrowave: poll() on an AU already being drained through poll_chunk"); + } + // PW5 stage 3: THIS is where the fence wait now lives. `submit` returns as soon as the + // work is queued; the AU only exists once the fence signals and the bitstream is + // packetized, so poll completes the oldest in-flight frame before answering. + if self.pending.is_empty() && !self.inflight.is_empty() { + // SAFETY: single-threaded encoder, waiting its own fence and reading its own + // bitstream; the failure path leaves the entry in flight for `reset()` to re-wait. + unsafe { self.wait_and_packetize()? }; + } Ok(self.pending.pop_front()) } + // --- streamed AU (PW6) — see `pyrowave_wire::AuChunker` for what this does and does NOT buy. + fn supports_chunked_poll(&self) -> bool { + crate::pyrowave_wire::stream_chunk_step(self.wire_chunk).is_some() + } + + fn poll_chunk(&mut self) -> Result> { + // Finish the AU already in flight before opening the next one — the host's `handle_chunk` + // keys begin/finish off `first`/`last` and cannot interleave two AUs. + if let Some(c) = self.chunker.as_mut() { + if let Some(chunk) = c.next() { + return Ok(Some(chunk)); + } + self.chunker = None; + } + let Some(f) = self.pending.pop_front() else { + return Ok(None); + }; + // No blocking wait here (the trait allows one): `submit` already ran the whole encode + // synchronously, so an AU in `pending` is complete by construction. + match crate::pyrowave_wire::stream_chunk_step(self.wire_chunk) { + Some(step) => Ok(self + .chunker + .insert(crate::pyrowave_wire::AuChunker::new(f, step)) + .next()), + // Unarmed / dense: the trait's own default shape, so a host that polls chunks anyway + // still gets whole AUs. + None => Ok(Some(crate::AuChunk::whole(f))), + } + } + fn reset(&mut self) -> bool { + // A rebuild forfeits every in-flight frame — including an AU only half-handed-out through + // `poll_chunk`. Dropping the cursor here (ahead of every `pending.clear()` arm below) is + // what keeps the next `poll_chunk` from splicing the tail of a dead AU onto a fresh one; + // the host sees a `first` without the previous `last`, logs "streamed AU abandoned + // mid-flight" and lets the client age that frame out. + self.chunker = None; // Cheap in-place rebuild: recreate only the pyrowave encoder object — there is no // rate-control history or reference state worth preserving (plan §4.3). // - // Bounded wait first: the only work possibly still executing is the one submitted frame - // whose synchronous fence wait timed out (`gpu_pending`). Re-wait it under the same 5 s - // cap as `encode_frame` — an untimed `device_wait_idle` here would park the recovery - // thread on the exact device it suspects is wedged, until the kernel's GPU reset, if - // ever. If the fence still won't signal, destroying the pyrowave encoder under live GPU - // work would be a use-after-free, so report "no in-place rebuild" and let the session - // surface a real error (`Drop`'s unbounded idle covers teardown, where blocking on the - // kernel is acceptable). - if self.gpu_pending { - // SAFETY: waiting this encoder's own fence under `&mut self`. - if unsafe { - self.device - .wait_for_fences(&[self.fence], true, 5_000_000_000) - } - .is_err() - { + // Bounded wait first: the only work possibly still executing is a submitted frame whose + // fence wait has not succeeded yet (`inflight` non-empty — either never polled, or polled + // and timed out). Re-wait it under the same 5 s cap as `wait_and_packetize` — an untimed + // `device_wait_idle` here would park the recovery thread on the exact device it suspects + // is wedged, until the kernel's GPU reset, if ever. If the fence still won't signal, + // destroying the pyrowave encoder under live GPU work would be a use-after-free, so + // report "no in-place rebuild" and let the session surface a real error (`Drop`'s + // unbounded idle covers teardown, where blocking on the kernel is acceptable). + if !self.inflight.is_empty() { + // Every in-flight frame's fence, not just the oldest — at depth > 1 there may be + // several, and destroying the pyrowave encoder while ANY of them still executes is a + // use-after-free. + let fences: Vec = self + .inflight + .iter() + .map(|f| self.slots[f.slot].fence) + .collect(); + // SAFETY: waiting this encoder's own fences under `&mut self`. + if unsafe { self.device.wait_for_fences(&fences, true, 5_000_000_000) }.is_err() { tracing::error!( "pyrowave: in-flight encode did not complete within the reset budget — GPU \ or driver wedged; in-place rebuild abandoned" @@ -1574,20 +2141,15 @@ impl Encoder for PyroWaveEncoder { self.pending.clear(); return false; } - self.gpu_pending = false; + // The submitted frames are forfeit (their bitstream lives in the encoder object about + // to be destroyed), but the GPU is provably done with them. + self.inflight.clear(); } // SAFETY: the device is idle for this encoder's work (the fence wait above, or no submit // outstanding) — this sweep-up is instant — and the pyrowave device outlives the encoder // object being swapped. unsafe { self.device.device_wait_idle().ok(); - pw::pyrowave_encoder_destroy(self.pw_enc); - // Publish the null IMMEDIATELY: the create below is fallible, and its failure path - // must not leave a freed pointer in the field. `pyrowave_encoder_destroy` is a plain - // `delete` (pyrowave_c.cpp) with no null check, so `Drop` running on a stale handle - // is a double free — the exact shape this reset hits when the rebuild fails because - // the device is already lost, which is the state that made the watchdog fire. - self.pw_enc = std::ptr::null_mut(); let einfo = pw::pyrowave_encoder_create_info { device: self.pw_dev, width: self.width as i32, @@ -1598,17 +2160,32 @@ impl Encoder for PyroWaveEncoder { pw::pyrowave_chroma_subsampling_PYROWAVE_CHROMA_SUBSAMPLING_420 }, }; - let mut enc: pw::pyrowave_encoder = std::ptr::null_mut(); - let r = pw::pyrowave_encoder_create(&einfo, &mut enc); - if r != pw::pyrowave_result_PYROWAVE_SUCCESS { - tracing::error!(result = ?r, "pyrowave: encoder rebuild failed"); - // `pw_enc` stays null — `Drop` and `encode_frame` both guard on it. The queued - // AUs are forfeit either way (the caller turns a false reset into a session - // error), so drop them rather than shipping output from a dead encoder. - self.pending.clear(); - return false; + for i in 0..SLOTS { + pw::pyrowave_encoder_destroy(self.pw_encs[i]); + // Publish the null IMMEDIATELY: the create below is fallible, and its failure path + // must not leave a freed pointer in the field. `pyrowave_encoder_destroy` is a + // plain `delete` (pyrowave_c.cpp) with no null check, so `Drop` running on a stale + // handle is a double free — the exact shape this reset hits when the rebuild fails + // because the device is already lost, which is the state that made the watchdog + // fire. + self.pw_encs[i] = std::ptr::null_mut(); + let mut enc: pw::pyrowave_encoder = std::ptr::null_mut(); + let r = pw::pyrowave_encoder_create(&einfo, &mut enc); + if r != pw::pyrowave_result_PYROWAVE_SUCCESS { + tracing::error!(result = ?r, slot = i, "pyrowave: encoder rebuild failed"); + // This handle stays null — `Drop` and `submit_frame` both guard on it. The + // queued AUs are forfeit either way (the caller turns a false reset into a + // session error), so drop them rather than shipping output from a dead + // encoder. + self.pending.clear(); + return false; + } + self.pw_encs[i] = enc; } - self.pw_enc = enc; + // Fresh handles start their own counters at 0, but the CLIENT's `last_seq` does not + // reset — so keep counting from where the stream was. A rebuild loses frames, and a + // gap is exactly what tells the decoder to restart. + self.next_slot = 0; } self.pending.clear(); true @@ -1640,8 +2217,11 @@ impl Encoder for PyroWaveEncoder { } fn flush(&mut self) -> Result<()> { - // Synchronous per-frame encode: nothing buffered beyond `pending`. - Ok(()) + // Since PW5 stage 3 there IS something buffered beyond `pending`: a submitted frame whose + // fence has not been waited. Retire it so the caller's `poll`-until-`None` drain + // (the trait's contract) actually returns every AU. Bounded by the same 5 s fence cap. + // SAFETY: single-threaded encoder, waiting its own fence. + unsafe { self.drain_to(0) } } } @@ -1654,13 +2234,15 @@ impl Drop for PyroWaveEncoder { // up, so on a failed open this runs against a partial prefix. That is sound because // `pyrowave_device_destroy(null)` is a bare `delete nullptr` (pyrowave_c.cpp — safe // no-op) and every `vkDestroy*`/`vkFree*` of VK_NULL_HANDLE is the spec-defined no-op; - // `pw_enc` is the one null-UNSAFE destroy and carries its own guard below. + // `pw_encs` are the null-UNSAFE destroys and carry their own guard below. unsafe { self.device.device_wait_idle().ok(); // Null when a failed `reset()` already destroyed it — `pyrowave_encoder_destroy` // is not null-safe. - if !self.pw_enc.is_null() { - pw::pyrowave_encoder_destroy(self.pw_enc); + for &e in &self.pw_encs { + if !e.is_null() { + pw::pyrowave_encoder_destroy(e); + } } pw::pyrowave_device_destroy(self.pw_dev); for (_, _, i, m, v) in self.import_cache.drain(..) { @@ -1668,16 +2250,32 @@ impl Drop for PyroWaveEncoder { self.device.destroy_image(i, None); self.device.free_memory(m, None); } - if let Some((i, m, v, _)) = self.cpu_img.take() { - self.device.destroy_image_view(v, None); - self.device.destroy_image(i, None); - self.device.free_memory(m, None); + // Every slot, in the same all-null-tolerant way (a failed open leaves a partial + // prefix built and the rest null; `vkDestroy*(VK_NULL_HANDLE)` is a spec no-op). + for sl in std::mem::take(&mut self.slots) { + if let Some((i, m, v, _)) = sl.cpu_img { + self.device.destroy_image_view(v, None); + self.device.destroy_image(i, None); + self.device.free_memory(m, None); + } + if let Some((b, m, _)) = sl.cpu_stage { + self.device.destroy_buffer(b, None); + self.device.free_memory(m, None); + } + self.device.destroy_fence(sl.fence, None); + self.device.destroy_image_view(sl.y_view, None); + self.device.destroy_image(sl.y_img, None); + self.device.free_memory(sl.y_mem, None); + self.device.destroy_image_view(sl.uv_view, None); + self.device.destroy_image(sl.uv_img, None); + self.device.free_memory(sl.uv_mem, None); + self.device.destroy_image_view(sl.cursor_view, None); + self.device.destroy_image(sl.cursor_img, None); + self.device.free_memory(sl.cursor_mem, None); + self.device.destroy_buffer(sl.cursor_stage, None); + self.device.free_memory(sl.cursor_stage_mem, None); } - if let Some((b, m, _)) = self.cpu_stage.take() { - self.device.destroy_buffer(b, None); - self.device.free_memory(m, None); - } - self.device.destroy_fence(self.fence, None); + // Command buffers and descriptor sets are freed with their pools. self.device.destroy_command_pool(self.cmd_pool, None); self.device.destroy_descriptor_pool(self.csc_pool, None); self.device.destroy_pipeline(self.csc_pipe, None); @@ -1685,17 +2283,6 @@ impl Drop for PyroWaveEncoder { self.device .destroy_descriptor_set_layout(self.csc_dsl, None); self.device.destroy_sampler(self.sampler, None); - self.device.destroy_image_view(self.y_view, None); - self.device.destroy_image(self.y_img, None); - self.device.free_memory(self.y_mem, None); - self.device.destroy_image_view(self.uv_view, None); - self.device.destroy_image(self.uv_img, None); - self.device.free_memory(self.uv_mem, None); - self.device.destroy_image_view(self.cursor_view, None); - self.device.destroy_image(self.cursor_img, None); - self.device.free_memory(self.cursor_mem, None); - self.device.destroy_buffer(self.cursor_stage, None); - self.device.free_memory(self.cursor_stage_mem, None); self.device.destroy_device(None); self.instance.destroy_instance(None); } @@ -1988,6 +2575,52 @@ mod tests { } } + /// PW5 stage 4: what the extra per-frame resource set actually COSTS in VRAM, at the modes + /// that decide whether it is affordable. Reported from the driver's own memory requirements, + /// not estimated from the dimensions — the plan's ~25-35 MB estimate is a guess, and on an + /// iGPU at 4K/4:4:4 the real number is the one that matters. + /// + /// Prints rather than asserts a threshold: a hard limit here would be a guess about every + /// future GPU. What it DOES assert is that a slot is not free and not absurd, so a refactor + /// that accidentally allocated per-slot copies of something large fails visibly. + #[test] + #[ignore = "needs a real Vulkan 1.3 compute device (run on a GPU host, not the build box)"] + fn slot_vram_cost_is_reported() { + for (w, h, chroma, name) in [ + (1920u32, 1080u32, crate::ChromaFormat::Yuv420, "1080p 4:2:0"), + (3840, 2160, crate::ChromaFormat::Yuv420, "4K 4:2:0"), + (3840, 2160, crate::ChromaFormat::Yuv444, "4K 4:4:4"), + ] { + let enc = PyroWaveEncoder::open(w, h, 60, 40_000_000, chroma).expect("open"); + // SAFETY: plain memory-requirement queries on images this encoder owns. + let per_slot: u64 = unsafe { + [ + enc.slots[0].y_img, + enc.slots[0].uv_img, + enc.slots[0].cursor_img, + ] + .iter() + .map(|&i| enc.device.get_image_memory_requirements(i).size) + .sum::() + + enc + .device + .get_buffer_memory_requirements(enc.slots[0].cursor_stage) + .size + }; + eprintln!( + "{name}: {} KiB per slot, {SLOTS} slots = {} KiB total", + per_slot / 1024, + per_slot * SLOTS as u64 / 1024 + ); + assert!(per_slot > 0, "{name}: a slot must own real memory"); + assert!( + per_slot < 512 * 1024 * 1024, + "{name}: {per_slot} bytes per slot — something large became per-slot that should \ + not have (bitstream? import cache?)" + ); + } + } + /// WP4.5: a frame that is not the session's mode must be REFUSED, not encoded. PyroWave /// applies no alignment, so a mismatch can only be a stale frame from a renegotiated mode — /// and the failure is silent without this check (`rgb2yuv.comp` clamps its fetches and the CPU @@ -2262,4 +2895,558 @@ mod tests { dump("ref-dense444-cb.bin", &cb); dump("ref-dense444-cr.bin", &cr); } + + // ---- WP14 step 4: the global-priority grammar -------------------------------------------- + // + // The device-create ladder itself has NO unit test and cannot have one — it needs a real + // Vulkan device. Its coverage is clippy plus the on-glass log line. What IS testable, and what + // actually drifts, is the grammar: it must stay character-identical to the vendored C patch, + // because the SAME env var drives the Windows path (where the patch is live) and this one. + // These are device-free by construction — that is why `queue_priority_candidates` takes the + // raw string instead of reading the environment itself (env-var tests race). + + /// Unset means `realtime`, which means "try REALTIME, then fall back to HIGH" — the ladder, + /// not a single class. Windows parity: the C patch defaults the same way. + #[test] + fn unset_requests_the_realtime_ladder() { + assert_eq!( + queue_priority_candidates(None), + vec![ + vk::QueueGlobalPriorityKHR::REALTIME, + vk::QueueGlobalPriorityKHR::HIGH + ] + ); + } + + /// `off` is the ONLY spelling that disables it, and it is case-insensitive because the C patch + /// lowercases first. Note `0` is deliberately NOT off — the C side does not accept it either, + /// and two grammars for one variable is the trap this wiring closes. + #[test] + fn only_off_disables_and_it_is_case_insensitive() { + assert!(queue_priority_candidates(Some("off")).is_empty()); + assert!(queue_priority_candidates(Some("OFF")).is_empty()); + assert!(queue_priority_candidates(Some("Off")).is_empty()); + assert!(!queue_priority_candidates(Some("0")).is_empty()); + } + + /// `high` asks for HIGH ONLY — it must not silently try REALTIME first, or the knob would be + /// unable to express "elevated, but not realtime" (the thing an operator reaches for after a + /// compositor-jank report). + #[test] + fn high_asks_for_high_alone() { + assert_eq!( + queue_priority_candidates(Some("high")), + vec![vk::QueueGlobalPriorityKHR::HIGH] + ); + assert_eq!( + queue_priority_candidates(Some("HIGH")), + vec![vk::QueueGlobalPriorityKHR::HIGH] + ); + } + + /// Junk falls back to the default ladder rather than to "off": an unparseable value must never + /// silently disable a performance lever the operator was trying to tune. + #[test] + fn junk_falls_back_to_the_default_ladder() { + for raw in ["", "realtime", "REALTIME", "yes", "1", "medium", " high"] { + assert!( + !queue_priority_candidates(Some(raw)).is_empty(), + "{raw:?} must not disable the priority request" + ); + } + // ...and specifically the full ladder, not HIGH alone. `" high"` is in the list above on + // purpose: the C patch does NOT trim, so neither do we — a space-padded value is junk to + // both, and the two must agree even in how they are wrong. + assert_eq!(queue_priority_candidates(Some(" high")).len(), 2); + } + + /// A refused class must walk the ladder down, never fail the open. `NOT_PERMITTED` is the + /// specified refusal; `INITIALIZATION_FAILED` is accepted too, matching pf-zerocopy's shipped + /// VkBridge ladder. Anything else is a real error and must propagate — a driver that is out of + /// memory should not be silently retried at a lower priority and reported as a success. + #[test] + fn only_refusals_walk_the_ladder_down() { + assert!(priority_refused(vk::Result::ERROR_NOT_PERMITTED_KHR)); + assert!(priority_refused(vk::Result::ERROR_INITIALIZATION_FAILED)); + assert!(!priority_refused(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY)); + assert!(!priority_refused(vk::Result::ERROR_EXTENSION_NOT_PRESENT)); + assert!(!priority_refused(vk::Result::SUCCESS)); + } + + // ---- PW6: the streamed-AU cut, on real GPU output ---------------------------------------- + // Appended at module END per the wave plan's ownership rule. + + /// Walk a windowed AU back into the flat codec-packet stream — the clients' parse + /// (`video_pyrowave.rs::push_window`, Apple's `MetalWaveletDecoder`), so upstream's decoder + /// sees exactly what a real client would feed it. + fn walk_windows(au: &[u8], window: usize) -> Vec { + let mut stream = Vec::new(); + let mut frag: Vec = Vec::new(); + for win in au.chunks(window) { + let used = u16::from_le_bytes([win[0], win[1]]) as usize; + let kind = u16::from_le_bytes([win[2], win[3]]); + let body = &win[4..4 + used]; + match kind { + 0 => stream.extend_from_slice(body), + 1 => frag = body.to_vec(), + 2 => frag.extend_from_slice(body), + 3 => { + frag.extend_from_slice(body); + stream.extend_from_slice(&frag); + frag.clear(); + } + k => panic!("unknown window kind {k}"), + } + } + stream + } + + /// Luma PSNR (dB) of a decoded Y plane against the BT.709 limited-range luma of the source + /// BGRA — the same math `rgb2yuv.comp` runs on the GPU. Luma only: chroma is subsampled on + /// the 4:2:0 path, and luma is where wavelet quantisation shows. + fn luma_psnr(src_bgra: &[u8], decoded_y: &[u8]) -> f64 { + assert_eq!(src_bgra.len(), decoded_y.len() * 4); + let mut sse = 0.0f64; + for (px, &got) in src_bgra.chunks_exact(4).zip(decoded_y) { + let (b, g, r) = (px[0] as f64, px[1] as f64, px[2] as f64); + let want = 16.0 + 0.1826 * r + 0.6142 * g + 0.0620 * b; + let d = want - got as f64; + sse += d * d; + } + let mse = sse / decoded_y.len() as f64; + if mse <= f64::EPSILON { + return f64::INFINITY; + } + 10.0 * (255.0f64 * 255.0 / mse).log10() + } + + /// PW6 on-glass: with `PUNKTFUNK_PYROWAVE_STREAMED_AU=1` armed, a real GPU encode of a BUSY + /// test card must come out of `poll_chunk` in several window-aligned pieces that concatenate + /// to a decodable AU — and the picture must survive, verified by PSNR against the CSC's own + /// BT.709 math rather than by "it ran". + /// + /// Flat fills are useless here (they false-greened the Windows bring-up): a solid colour + /// reassembles convincingly even when whole subbands are missing. The busy card puts energy + /// in every subband, so a cut that lost or reordered a window shows up as a PSNR collapse. + /// + /// `#[ignore]`d: needs a real Vulkan 1.3 GPU. + /// cargo test -p pf-encode --features pyrowave --no-run + /// PUNKTFUNK_PYROWAVE_STREAMED_AU=1 --ignored --nocapture pyrowave_streamed_chunks + #[test] + #[ignore = "needs a real Vulkan 1.3 compute device (run on a GPU host, not the build box)"] + fn pyrowave_streamed_chunks_reassemble_and_keep_the_picture() { + const WINDOW: usize = 1408; + // 1280x720 at 60 Mb/s ≈ 125 KB/AU — comfortably several 256 KiB-target chunks' worth of + // windows at the default step once the step is clamped to the AU, and big enough that the + // AU spans many windows. + let (w, h) = (1280u32, 720u32); + let mut enc = PyroWaveEncoder::open(w, h, 60, 200_000_000, crate::ChromaFormat::Yuv420) + .expect("open pyrowave encoder"); + enc.set_wire_chunking(WINDOW); + + assert!( + enc.supports_chunked_poll(), + "PUNKTFUNK_PYROWAVE_STREAMED_AU=1 must be set in the ENVIRONMENT of this test binary \ + — without it PW6 is off by design and there is nothing to verify" + ); + + for seed in [7u32, 11, 13] { + let frame = test_card(w, h, seed); + let FramePayload::Cpu(ref src) = frame.payload else { + panic!("test card is a CPU frame") + }; + let src = src.clone(); + enc.submit(&frame).expect("submit"); + + // Drain the AU through the chunked poll, exactly as the native pump does. + let mut au = Vec::new(); + let (mut chunks, mut firsts, mut lasts) = (0u32, 0u32, 0u32); + loop { + let c = enc + .poll_chunk() + .expect("poll_chunk") + .expect("an AU is in flight"); + assert!(c.chunk_aligned, "wire chunking is on"); + assert!(c.keyframe, "every pyrowave AU is a keyframe"); + assert_eq!( + c.data.len() % WINDOW, + 0, + "every chunk is a whole number of windows — a cut inside a window would \ + split the 4-byte framing prefix from its body" + ); + chunks += 1; + firsts += u32::from(c.first); + lasts += u32::from(c.last); + au.extend_from_slice(&c.data); + if c.last { + break; + } + } + assert_eq!(firsts, 1, "exactly one opening chunk"); + assert_eq!(lasts, 1, "exactly one closing chunk"); + assert!( + chunks > 1, + "seed {seed}: the AU came out in ONE piece ({} B) — the cut never engaged, so \ + this run proves nothing about PW6", + au.len() + ); + assert_eq!(au.len() % WINDOW, 0, "the AU is a whole number of windows"); + + // A second `poll_chunk` must report the AU is done, not dribble more bytes. + assert!( + enc.poll_chunk().expect("poll_chunk after last").is_none(), + "no AU is in flight once `last` was handed out" + ); + + // The picture: window-walk (the client's parse) → upstream's own decoder → PSNR. + let stream = walk_windows(&au, WINDOW); + // SAFETY: test-only FFI into the vendored decoder with locally-owned buffers. + let (y, _cb, _cr) = unsafe { decode_planes(w, h, &stream) }; + let psnr = luma_psnr(&src, &y); + eprintln!( + "seed {seed}: {chunks} chunks, {} B AU ({} windows), luma PSNR {psnr:.2} dB", + au.len(), + au.len() / WINDOW + ); + assert!( + psnr > 30.0, + "seed {seed}: luma PSNR {psnr:.2} dB — the streamed reassembly lost or reordered \ + picture data (a flat-fill test would NOT have caught this)" + ); + } + } + + // ---- PW5 stage 5: the alternating-handle sequence gate ------------------------------------ + + /// **THE gate for the second encoder handle.** Two `pyrowave_encoder` objects each keep their + /// OWN 3-bit `sequence_count`, so alternating them emits `1,1,2,2,3,3…` on the wire. The + /// decoder restarts a frame only when the value CHANGES + /// (`diff = (hdr.sequence - last_seq) & 0x7; restart = diff != 0`), so every repeat reads as + /// "more blocks of the same frame": `clear()` never runs and the second frame of each pair is + /// silently swallowed. Half frame rate, occasional mixed-frame blocks, no error anywhere — a + /// failure that passes a smoke test, on every client. + /// + /// `patches/0007-encoder-sequence-override.patch` exists solely to make that impossible, and + /// this test is what proves it, three ways over 20 frames (well past the 3-bit wrap at 8): + /// + /// 1. the wire counter advances by exactly +1 mod 8 per AU, read straight out of the block + /// header the decoder reads; + /// 2. ONE persistent decoder — `last_seq` carried across every push, exactly as a client's is — + /// reports ready for every single AU, so nothing is swallowed; + /// 3. consecutive decoded pictures DIFFER. Content moves every frame (`test_card` reseeded per + /// frame; flat fills are the documented false-green trap here), so a swallowed frame would + /// show up as a repeat, and this catches it even if 1 and 2 somehow both passed. + #[test] + #[ignore = "needs a real Vulkan 1.3 compute device (run on a GPU host, not the build box)"] + fn wire_sequence_increments_across_alternating_handles() { + const FRAMES: u32 = 20; + let (w, h) = (256u32, 256u32); + let mut enc = + PyroWaveEncoder::open(w, h, 60, 40_000_000, crate::ChromaFormat::Yuv420).expect("open"); + // This gate is meaningless with a single encoder handle. + const { assert!(SLOTS >= 2) }; + + let mut aus: Vec> = Vec::new(); + for i in 0..FRAMES { + // Content MOVES every frame — a repeated picture is the symptom being hunted, and a + // static card would hide it. ODD seeds only: `test_card` starts its LCG at `seed | 1`, + // so 2 and 3 produce a byte-identical card and consecutive even/odd seeds would fake + // the very repeat this test looks for (it did, on the first run). + enc.submit(&test_card(w, h, 2 * i + 1)).expect("submit"); + let au = enc.poll().expect("poll").expect("one AU per frame"); + aus.push(au.data); + } + + // (1) the wire counter, read from the header the decoder parses. + let seqs: Vec = aus + .iter() + .map(|au| { + crate::pyrowave_wire::wire_sequence(au, 0).expect("AU carries a block header") + }) + .collect(); + for (i, pair) in seqs.windows(2).enumerate() { + assert_eq!( + pair[1], + (pair[0] + 1) & 7, + "frame {} -> {}: wire sequence went {} -> {} (all: {seqs:?}). Two encoder handles \ + each counting alone produce repeats, which the decoder reads as more blocks of \ + the same frame — check that patch 0007 is applied and set_next_sequence is called", + i, + i + 1, + pair[0], + pair[1] + ); + } + + // (2) + (3) ONE decoder for the whole run — a fresh decoder per AU would reset `last_seq` + // and hide the exact bug this exists to catch. + // SAFETY: test-only FFI into the vendored decoder with locally-owned buffers. + unsafe { + let mut dev: pw::pyrowave_device = std::ptr::null_mut(); + assert_eq!( + pw::pyrowave_create_default_device(&mut dev), + pw::pyrowave_result_PYROWAVE_SUCCESS + ); + let dinfo = pw::pyrowave_decoder_create_info { + device: dev, + width: w as i32, + height: h as i32, + chroma: pw::pyrowave_chroma_subsampling_PYROWAVE_CHROMA_SUBSAMPLING_420, + fragment_path: false, + }; + let mut dec: pw::pyrowave_decoder = std::ptr::null_mut(); + assert_eq!( + pw::pyrowave_decoder_create(&dinfo, &mut dec), + pw::pyrowave_result_PYROWAVE_SUCCESS + ); + let mut last_y: Option> = None; + for (i, au) in aus.iter().enumerate() { + assert_eq!( + pw::pyrowave_decoder_push_packet(dec, au.as_ptr() as *const _, au.len()), + pw::pyrowave_result_PYROWAVE_SUCCESS, + "frame {i} was rejected by the decoder" + ); + assert!( + pw::pyrowave_decoder_decode_is_ready(dec, false), + "frame {i} never became decodable — the decoder is still accumulating it into \ + the PREVIOUS frame, which is exactly the repeated-sequence failure" + ); + let mut y = vec![0u8; (w * h) as usize]; + let mut cb = vec![0u8; (w * h / 4) as usize]; + let mut cr = vec![0u8; (w * h / 4) as usize]; + let mut buf: pw::pyrowave_cpu_buffer = std::mem::zeroed(); + buf.format = pw::pyrowave_cpu_buffer_format_PYROWAVE_CPU_BUFFER_FORMAT_YUV420P; + buf.width = w as i32; + buf.height = h as i32; + buf.data = [ + y.as_mut_ptr() as *mut _, + cb.as_mut_ptr() as *mut _, + cr.as_mut_ptr() as *mut _, + ]; + buf.row_stride_in_bytes = [w as usize, (w / 2) as usize, (w / 2) as usize]; + buf.plane_size_in_bytes = [y.len(), cb.len(), cr.len()]; + assert_eq!( + pw::pyrowave_decoder_decode_cpu_buffer_synchronous(dec, &buf), + pw::pyrowave_result_PYROWAVE_SUCCESS, + "frame {i} failed to decode" + ); + if let Some(prev) = &last_y { + assert_ne!( + prev, + &y, + "frame {i} decoded to the SAME picture as frame {} — a swallowed frame", + i - 1 + ); + } + last_y = Some(y); + } + pw::pyrowave_decoder_destroy(dec); + pw::pyrowave_device_destroy(dev); + } + } + + /// Decode a whole AU stream through ONE decoder (a client's `last_seq` is not reset per frame) + /// and return each frame's luma plane. + /// + /// # Safety + /// Test-only FFI into the vendored decoder with locally-owned buffers. + unsafe fn decode_stream_luma(w: u32, h: u32, aus: &[Vec]) -> Vec> { + let mut dev: pw::pyrowave_device = std::ptr::null_mut(); + assert_eq!( + pw::pyrowave_create_default_device(&mut dev), + pw::pyrowave_result_PYROWAVE_SUCCESS + ); + let dinfo = pw::pyrowave_decoder_create_info { + device: dev, + width: w as i32, + height: h as i32, + chroma: pw::pyrowave_chroma_subsampling_PYROWAVE_CHROMA_SUBSAMPLING_420, + fragment_path: false, + }; + let mut dec: pw::pyrowave_decoder = std::ptr::null_mut(); + assert_eq!( + pw::pyrowave_decoder_create(&dinfo, &mut dec), + pw::pyrowave_result_PYROWAVE_SUCCESS + ); + let mut out = Vec::with_capacity(aus.len()); + for (i, au) in aus.iter().enumerate() { + assert_eq!( + pw::pyrowave_decoder_push_packet(dec, au.as_ptr() as *const _, au.len()), + pw::pyrowave_result_PYROWAVE_SUCCESS, + "frame {i} rejected" + ); + assert!( + pw::pyrowave_decoder_decode_is_ready(dec, false), + "frame {i} never became decodable" + ); + let mut y = vec![0u8; (w * h) as usize]; + let mut cb = vec![0u8; (w * h / 4) as usize]; + let mut cr = vec![0u8; (w * h / 4) as usize]; + let mut buf: pw::pyrowave_cpu_buffer = std::mem::zeroed(); + buf.format = pw::pyrowave_cpu_buffer_format_PYROWAVE_CPU_BUFFER_FORMAT_YUV420P; + buf.width = w as i32; + buf.height = h as i32; + buf.data = [ + y.as_mut_ptr() as *mut _, + cb.as_mut_ptr() as *mut _, + cr.as_mut_ptr() as *mut _, + ]; + buf.row_stride_in_bytes = [w as usize, (w / 2) as usize, (w / 2) as usize]; + buf.plane_size_in_bytes = [y.len(), cb.len(), cr.len()]; + assert_eq!( + pw::pyrowave_decoder_decode_cpu_buffer_synchronous(dec, &buf), + pw::pyrowave_result_PYROWAVE_SUCCESS, + "frame {i} failed to decode" + ); + out.push(y); + } + pw::pyrowave_decoder_destroy(dec); + pw::pyrowave_device_destroy(dev); + out + } + + /// PSNR (dB) between two equal-sized 8-bit planes; `f64::INFINITY` when identical. + fn psnr(a: &[u8], b: &[u8]) -> f64 { + assert_eq!(a.len(), b.len()); + let mse = a + .iter() + .zip(b) + .map(|(&x, &y)| { + let d = x as f64 - y as f64; + d * d + }) + .sum::() + / a.len() as f64; + if mse == 0.0 { + f64::INFINITY + } else { + 10.0 * (255.0 * 255.0 / mse).log10() + } + } + + /// **PW5 stage 6's ENCODER-side gate.** Two frames genuinely in flight at once must produce the + /// same pictures, in the same order, as the synchronous depth-1 path. + /// + /// This is the half of the depth-2 risk that lives in THIS crate: the slot resources + /// (`cmd`/`fence`/`csc_set`/y/uv/cursor) and the alternating encoder handles. Ground truth is + /// the encoder's OWN depth-1 output over the same frames, which is the honest reference — + /// pyrowave's raw AU bytes are not reproducible run-to-run (see the stage-3 commit), but its + /// DECODED planes are. + /// + /// Content moves every frame. Flat fills are the documented false-green trap here: a torn frame + /// assembled from two halves of a static card is invisible, and a gray fill once green-lit a + /// broken import. + /// + /// ⚠ WHAT THIS DOES **NOT** COVER, and no in-tree test can: the CAPTURE side. `.process` + /// requeues the SPA buffer to the compositor at callback return while the encode thread still + /// holds only a dup of its fd, so a second frame in flight widens the window in which the + /// producer may overwrite a buffer we are still reading by a full frame period. That needs a + /// live compositor, a real client and a long moving-content session — the on-glass tear-hunt + /// PW5 stage 6 is gated on. This test passing is necessary, not sufficient. + /// + /// Drives the backend at depth 2 by setting `max_inflight` directly rather than through a + /// shipped knob: the shipped value is 1 and this must not change that. + #[test] + #[ignore = "needs a real Vulkan 1.3 compute device (run on a GPU host, not the build box)"] + fn overlapping_two_frames_reproduces_the_synchronous_picture() { + const FRAMES: u32 = 16; + let (w, h) = (256u32, 256u32); + // Odd seeds: `test_card` starts its LCG at `seed | 1`, so 2 and 3 build the same card. + let cards: Vec = (0..FRAMES).map(|i| test_card(w, h, 2 * i + 1)).collect(); + let open = || { + PyroWaveEncoder::open(w, h, 60, 40_000_000, crate::ChromaFormat::Yuv420).expect("open") + }; + + // --- reference: strictly synchronous, one frame at a time --- + let mut enc = open(); + let sync: Vec> = cards + .iter() + .map(|c| { + enc.submit(c).expect("sync submit"); + assert_eq!( + enc.inflight.len(), + 1, + "submit must leave exactly one in flight" + ); + enc.poll() + .expect("sync poll") + .expect("one AU per frame") + .data + }) + .collect(); + drop(enc); + + // --- overlapped: submit N+1 before polling N --- + let mut enc = open(); + enc.max_inflight = SLOTS; + let mut overlapped: Vec> = Vec::new(); + let mut saw_two_in_flight = false; + for c in &cards { + enc.submit(c).expect("overlapped submit"); + saw_two_in_flight |= enc.inflight.len() == 2; + if enc.inflight.len() >= SLOTS { + overlapped.push( + enc.poll() + .expect("overlapped poll") + .expect("an AU once the pipeline is full") + .data, + ); + } + } + enc.flush().expect("flush drains the tail"); + while let Some(au) = enc.poll().expect("tail poll") { + overlapped.push(au.data); + } + drop(enc); + assert!( + saw_two_in_flight, + "two frames were never actually in flight — this test proved nothing" + ); + assert_eq!( + overlapped.len(), + sync.len(), + "the overlapped run emitted a different number of AUs — a frame was lost" + ); + + // --- decode both streams and compare, frame by frame --- + // SAFETY: test-only FFI into the vendored decoder with locally-owned buffers. + let (sy, oy) = unsafe { + ( + decode_stream_luma(w, h, &sync), + decode_stream_luma(w, h, &overlapped), + ) + }; + let mut worst = f64::INFINITY; + for i in 0..sy.len() { + let p = psnr(&sy[i], &oy[i]); + worst = worst.min(p); + // 45 dB is far above "looks the same" — a torn frame stitched from two moving cards + // lands in the teens. Not an equality assert only because the wavelet RDO is not + // bit-reproducible; the printed worst-case is the number to read. + assert!( + p > 45.0, + "frame {i}: overlapped decode is {p:.1} dB from the synchronous one — the pipelined \ + path changed the picture" + ); + // The discriminator that PSNR alone can miss: a frame delivered ONE POSITION OFF still + // scores well against a similar neighbour. It must match its OWN reference best. + if i > 0 { + let prev = psnr(&sy[i - 1], &oy[i]); + assert!( + p > prev, + "frame {i} matches the PREVIOUS reference better ({prev:.1} dB) than its own \ + ({p:.1} dB) — the pipeline is off by one" + ); + } + } + eprintln!( + "depth-2 vs depth-1 over {} frames: worst-case PSNR {}", + sy.len(), + if worst.is_infinite() { + "identical (inf)".to_string() + } else { + format!("{worst:.1} dB") + } + ); + } } diff --git a/crates/pf-encode/src/enc/pyrowave_wire.rs b/crates/pf-encode/src/enc/pyrowave_wire.rs index c7e02653..6ac4691d 100644 --- a/crates/pf-encode/src/enc/pyrowave_wire.rs +++ b/crates/pf-encode/src/enc/pyrowave_wire.rs @@ -53,6 +53,24 @@ pub(crate) fn stamp_color_bits(bitstream: &mut [u8], seq_offset: usize, bt2020_p } } +/// Read the 3-bit wire sequence counter out of a pyrowave block header. +/// +/// Every block header is `{ u16 ballot; u16 payload_words:12, sequence:3, extended:1; u32 ... }` +/// (`pyrowave_common.hpp`, `static_assert(sizeof == 8)`), so the counter is bits 12..14 of the +/// little-endian half-word at `packet_offset + 2` — the same word `stamp_color_bits` reaches into +/// from the other end. +/// +/// This field is the entire frame-boundary signal on the wire: the decoder restarts a frame only +/// when the value CHANGES (`diff = (hdr.sequence - last_seq) & 0x7; restart = diff != 0`), so a +/// repeated value is read as more blocks of the same frame. That is why PW5's alternating encoder +/// handles need `pyrowave_encoder_set_next_sequence`, and why a test asserts this reader sees +/// +1 mod 8 across the pair. +pub(crate) fn wire_sequence(bitstream: &[u8], packet_offset: usize) -> Option { + let lo = *bitstream.get(packet_offset + 2)?; + let hi = *bitstream.get(packet_offset + 3)?; + Some(((u16::from_le_bytes([lo, hi]) >> 12) & 0x7) as u8) +} + /// The wavelet block space's total 32x32-block count for a mode — the exact counting walk of /// upstream `WaveletBuffers::init_block_meta` (also ported to the Apple `WaveletLayout`, whose /// golden tests pin it against real host AUs). Needed because the vendored RDO pass packs the @@ -201,6 +219,193 @@ pub(crate) fn build_au( au } +// --------------------------------------------------------------------------- +// Streamed-AU chunk cutting (PW6 — latency plan §T3.4, wave-2 plan PW6) +// --------------------------------------------------------------------------- + +/// Default per-chunk target — ~3–4 chunks for a 400 Mb/s 60 fps AU (~833 KB). Deliberately coarse, +/// because the SEALER, not this size, sets how early bytes actually leave: +/// +/// * Toward a plain `VIDEO_CAP_STREAMED_AU` client, `Packetizer::push_streamed` flushes only when +/// its pending buffer exceeds one FEC block — `fec.max_data_per_block × shard_payload`, which is +/// 200 × 1408 = 281 600 B on the shipped 1500-MTU IPv4 geometry. Anything smaller than that is +/// simply buffered. (256 KiB sits just under one block, so the first flush lands on the SECOND +/// chunk; the win is intact either way — the whole-AU path seals all ~3 blocks before its first +/// datagram may leave.) Only a client that ALSO negotiated `VIDEO_CAP_MULTI_SLICE` gets the +/// finer `MIN_STREAM_BLOCK_SHARDS` floor (16 shards ≈ 22 KB), where the chunk size does set the +/// flush granularity directly. pf-encode is not told the session's FEC geometry, so this is a +/// fixed byte target rather than a block-derived one. +/// * Chunks are not free: the send thread paces each sealed batch on its own +/// (`stream.rs::pace_sealed`), and every call grants a fresh `max(bytes/4, 128 KiB)` microburst +/// allowance. Cutting an AU into dozens of chunks therefore erodes the pacing this host does to +/// stop line-rate bursts from overrunning the NIC — the failure mode the pacer exists for. +const STREAM_CHUNK_TARGET_BYTES: usize = 256 * 1024; +/// Clamp on the `PUNKTFUNK_PYROWAVE_CHUNK_KIB` override (see [`stream_chunk_step`]). +const STREAM_CHUNK_MIN_KIB: usize = 4; +const STREAM_CHUNK_MAX_KIB: usize = 8192; + +/// Whether streamed-AU output is armed for this host process. +/// +/// **Default OFF, and deliberately so.** The streamed wire shape costs one PyroWave-specific +/// regression that has not been measured: an UNPINNED streamed frame (its final block never +/// arrived, so `frame_bytes` is still the 0 sentinel) is excluded from partial delivery +/// (`reassemble.rs`, 2026-07 security-review finding 10) — where today's whole-AU path hands the +/// consumer a usable blurred partial, a streamed frame that loses its final block delivers +/// NOTHING. PyroWave clients opt into partial delivery unconditionally +/// (`client/pump/handshake.rs`), so this is a live behaviour change for every one of them. The +/// netem loss-harness leg (2 % on `lo`, FEC pinned off — the Phase-4 recipe) comparing +/// partial-delivery rates streamed vs whole-AU is the prerequisite for flipping the default; +/// until it has run, `PUNKTFUNK_PYROWAVE_STREAMED_AU=1` is how you get it. +/// +/// The client's `VIDEO_CAP_STREAMED_AU` and the host's `PUNKTFUNK_STREAMED_AU` remain the outer +/// gates (`stream.rs`) — this only decides whether the ENCODER offers chunks at all. +fn stream_armed() -> bool { + static ARMED: std::sync::OnceLock = std::sync::OnceLock::new(); + // Latched once: `supports_chunked_poll` is re-queried per AU, and a knob that could change + // mid-session would flip the wire shape under an open `StreamedAu`. + *ARMED.get_or_init(|| { + matches!( + std::env::var("PUNKTFUNK_PYROWAVE_STREAMED_AU").as_deref(), + Ok("1") + ) + }) +} + +/// Bytes per streamed chunk, rounded DOWN to a whole number of `window`-sized windows (never +/// below one). The rounding is the whole point — see [`AuChunker`]. +fn chunk_step(window: usize, target: usize) -> usize { + (target / window.max(1)).max(1) * window.max(1) +} + +/// The streamed-AU chunk size for a backend whose wire chunking is `wire_chunk`, or `None` when +/// this session must stay on the whole-AU path — which is the answer whenever the feature is not +/// armed ([`stream_armed`]) or the encoder is in DENSE mode. +/// +/// Dense mode is excluded on purpose: there the AU is ONE atomic pyrowave packet with no window +/// framing, so a cut is neither shard-aligned nor a framing boundary. Every real PyroWave session +/// runs datagram-aligned (`stream.rs` sets `plan.wire_chunk = Some(session.shard_payload())`), so +/// nothing is lost — but the invariant this file promises stays true instead of nearly true. +/// +/// `PUNKTFUNK_PYROWAVE_CHUNK_KIB` overrides the target (clamped to +/// [`STREAM_CHUNK_MIN_KIB`]..=[`STREAM_CHUNK_MAX_KIB`]); garbage falls back to the default. +pub(crate) fn stream_chunk_step(wire_chunk: Option) -> Option { + let window = wire_chunk.filter(|&w| w > 0)?; + if !stream_armed() { + return None; + } + static TARGET: std::sync::OnceLock = std::sync::OnceLock::new(); + let target = *TARGET.get_or_init(|| { + std::env::var("PUNKTFUNK_PYROWAVE_CHUNK_KIB") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|k| (STREAM_CHUNK_MIN_KIB..=STREAM_CHUNK_MAX_KIB).contains(k)) + .map(|k| k * 1024) + .unwrap_or(STREAM_CHUNK_TARGET_BYTES) + }); + Some(chunk_step(window, target)) +} + +/// Hands a **finished** datagram-aligned AU out in window-aligned pieces for the streamed-AU wire +/// ([`crate::Encoder::poll_chunk`], `punktfunk_core::quic::VIDEO_CAP_STREAMED_AU`). Shared by both +/// pyrowave backends so the cut rule cannot drift between Linux and Windows — the Windows backend +/// cannot even be compiled from a Linux/macOS dev box, so logic written into it directly ships +/// unverified. +/// +/// ## What this does NOT buy (read before quoting PW6 as a latency win) +/// +/// pyrowave's `encode_frame` is **synchronous**: `submit` returns only once the whole AU sits in +/// `pending`, so by the time the host can poll a chunk the encode is over. `poll_chunk` is +/// therefore NOT "emit slices as the encoder produces them" — it is "hand the finished AU out in +/// pieces so the wire work pipelines with itself". Concretely, what moves: +/// +/// * whole-AU path: `Session::seal_frame_at` FEC-protects, packetizes and AEAD-seals the ENTIRE +/// ~830 KB AU before its first datagram may leave the socket; +/// * streamed path: each FEC block seals and paces as it completes, so the first byte reaches the +/// wire after one block's seal, and the remaining seal work overlaps its own transmission. +/// +/// There is NO encode/send overlap here — unlike the H.26x sub-frame slice path, where chunks +/// genuinely appear while the encoder is still working. PW6 and PW5 (encode overlap) are +/// independent packages, not sequential ones. +/// +/// It also does **not** give the client decode-while-arriving: the reassembler completes a +/// streamed AU exactly like a whole one (`reassemble.rs` — `block_count != 0 && blocks_ok == +/// block_count`) and hands up ONE `Frame`. Client-side prefix decode is the separate +/// `Session::set_deliver_frame_parts` opt-in, which PyroWave's newest-wins frame channel cannot +/// take — see the PW6 section of `design/linux-host-performance-wave2-pyrowave.md`. +/// +/// ## The cut rule +/// +/// A chunk is a whole number of `chunk`-sized WINDOWS. [`build_au`] gives every window exactly ONE +/// `kind` in its 4-byte prefix (`WIN_PACKED` or one link of a `WIN_FRAG_*` chain), so a cut inside +/// a window would split a unit the clients parse atomically. Whole windows are `shard_payload` +/// multiples by construction, which is what makes the sealer's sentinel block bases shard-aligned +/// for free (plan §4.4) — the streamed path's placement contract. +pub(crate) struct AuChunker { + au: Vec, + /// Bytes already handed out. + cursor: usize, + /// Bytes per chunk — a whole number of windows ([`chunk_step`]). + step: usize, + pts_ns: u64, + keyframe: bool, + recovery_anchor: bool, + chunk_aligned: bool, + /// Set once anything has been emitted, so the degenerate EMPTY AU still owes exactly one + /// chunk and not an infinite stream of them. + emitted: bool, +} + +impl AuChunker { + pub(crate) fn new(frame: crate::EncodedFrame, step: usize) -> AuChunker { + AuChunker { + au: frame.data, + cursor: 0, + step: step.max(1), + pts_ns: frame.pts_ns, + keyframe: frame.keyframe, + recovery_anchor: frame.recovery_anchor, + chunk_aligned: frame.chunk_aligned, + emitted: false, + } + } + + /// The next piece, or `None` once the AU is spent. The pieces concatenate to exactly the bytes + /// [`crate::Encoder::poll`] would have returned; `first` opens the wire frame and `last` closes + /// it (the host's `handle_chunk` keys its `begin`/`finish` off precisely those two). + pub(crate) fn next(&mut self) -> Option { + if self.cursor >= self.au.len() { + // A zero-byte AU is not reachable through `build_au` (it always emits at least one + // window), but the host would leak its open `StreamedAu` if a chunked poll returned + // nothing at all — so the degenerate case still owes one self-closing chunk. + if self.emitted { + return None; + } + self.emitted = true; + return Some(self.chunk(Vec::new(), true, true)); + } + let first = self.cursor == 0; + let end = (self.cursor + self.step).min(self.au.len()); + let data = self.au[self.cursor..end].to_vec(); + self.cursor = end; + self.emitted = true; + Some(self.chunk(data, first, end == self.au.len())) + } + + /// AU-level metadata rides every chunk (the `AuChunk` contract only makes it authoritative on + /// `first`, but a truthful copy on each one costs nothing and keeps a mid-AU log honest). + fn chunk(&self, data: Vec, first: bool, last: bool) -> crate::AuChunk { + crate::AuChunk { + data, + pts_ns: self.pts_ns, + keyframe: self.keyframe, + recovery_anchor: self.recovery_anchor, + chunk_aligned: self.chunk_aligned, + first, + last, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -362,4 +567,119 @@ mod tests { stamp_color_bits(&mut bs, 0, true); assert_eq!(bs[7], 0x78); } + + // --- streamed-AU chunk cutting (PW6) ------------------------------------ + // Appended at module END per the wave plan's ownership rule. + + fn frame(data: Vec) -> crate::EncodedFrame { + crate::EncodedFrame { + data, + pts_ns: 1_234_567, + keyframe: true, + recovery_anchor: false, + chunk_aligned: true, + } + } + + /// Drain a chunker into `(concatenated bytes, per-chunk lengths, first flags, last flags)`. + fn drain(mut c: AuChunker) -> (Vec, Vec, Vec, Vec) { + let (mut bytes, mut lens, mut firsts, mut lasts) = (Vec::new(), Vec::new(), vec![], vec![]); + while let Some(ch) = c.next() { + lens.push(ch.data.len()); + firsts.push(ch.first); + lasts.push(ch.last); + bytes.extend_from_slice(&ch.data); + assert_eq!(ch.pts_ns, 1_234_567, "AU metadata rides every chunk"); + assert!(ch.keyframe && ch.chunk_aligned && !ch.recovery_anchor); + } + (bytes, lens, firsts, lasts) + } + + /// The invariant PW6 rests on: chunks concatenate to EXACTLY the AU, every cut lands on a + /// whole-window boundary (so no window's single `kind` is split across two wire frames), and + /// the reassembled stream still walks back to the same codec packets. A cut inside a window + /// would hand the client a 4-byte prefix whose body arrives in a different chunk — the + /// framing is one-kind-per-window, so there is no way to express that. + #[test] + fn stream_chunks_tile_the_au_on_window_boundaries() { + let bs: Vec = (0..4000u32).map(|i| (i % 251) as u8).collect(); + let packets = [(0, 20), (20, 300), (320, 55), (375, 900), (1275, 40)]; + let chunk = 64; + let au = build_au(&packets, &bs, Some(chunk)); + assert!(au.len() / chunk > 4, "need several windows to cut between"); + let step = chunk_step(chunk, 3 * chunk); + assert_eq!(step, 3 * chunk); + let (bytes, lens, firsts, lasts) = drain(AuChunker::new(frame(au.clone()), step)); + assert_eq!(bytes, au, "chunks concatenate to exactly the AU"); + assert!( + lens.iter().all(|l| l % chunk == 0), + "every chunk is a whole number of windows: {lens:?}" + ); + assert!( + lens[..lens.len() - 1].iter().all(|&l| l == step), + "only the tail chunk may be short: {lens:?}" + ); + assert_eq!( + firsts, + (0..lens.len()).map(|i| i == 0).collect::>(), + "exactly one opening chunk" + ); + assert_eq!( + lasts, + (0..lens.len()) + .map(|i| i + 1 == lens.len()) + .collect::>(), + "exactly one closing chunk" + ); + // And the client's parse is unchanged by the cutting. + let mut expect = Vec::new(); + for &(o, s) in &packets { + expect.extend_from_slice(&bs[o..o + s]); + } + assert_eq!(walk(&bytes, chunk), expect); + } + + /// The step always rounds DOWN to whole windows and never to zero — a target below one window + /// degenerates to one window per chunk rather than an empty chunk (which would spin forever). + #[test] + fn chunk_step_rounds_down_to_whole_windows() { + // 262144 / 1408 = 186.2 → 186 whole windows (261 888 B), never the 262 144 asked for. + assert_eq!(chunk_step(1408, 256 * 1024), 186 * 1408); + assert_eq!(chunk_step(1408, 1408), 1408); + assert_eq!(chunk_step(1408, 1407), 1408); // below one window → one window + assert_eq!(chunk_step(1408, 0), 1408); + assert_eq!(chunk_step(0, 4096), 4096); // defensive: never divides by zero + } + + /// An AU that fits one chunk is a single `first && last` piece — the shape the host's + /// `handle_chunk` turns into begin+finish on one message, and byte-identical on the wire to + /// what the whole-AU path would have sealed. + #[test] + fn single_chunk_au_opens_and_closes_itself() { + let au = vec![7u8; 512]; + let (bytes, lens, firsts, lasts) = drain(AuChunker::new(frame(au.clone()), 4096)); + assert_eq!(bytes, au); + assert_eq!(lens, vec![512]); + assert_eq!(firsts, vec![true]); + assert_eq!(lasts, vec![true]); + } + + /// The degenerate empty AU still owes exactly ONE self-closing chunk: a chunked poll that + /// returned nothing would leave the host's `StreamedAu` open forever (its `begin` fires on + /// `first`, its `finish` on `last`). + #[test] + fn empty_au_still_emits_one_self_closing_chunk() { + let mut c = AuChunker::new(frame(Vec::new()), 4096); + let ch = c.next().expect("one chunk"); + assert!(ch.first && ch.last && ch.data.is_empty()); + assert!(c.next().is_none(), "and never a second one"); + } + + /// Dense (non-windowed) AUs never stream: there is no window framing to cut on, so a chunk + /// boundary would be neither shard-aligned nor a parse boundary. + #[test] + fn dense_mode_never_streams() { + assert!(stream_chunk_step(None).is_none()); + assert!(stream_chunk_step(Some(0)).is_none()); + } } diff --git a/crates/pf-encode/src/enc/windows/pyrowave.rs b/crates/pf-encode/src/enc/windows/pyrowave.rs index 67a2f271..18504751 100644 --- a/crates/pf-encode/src/enc/windows/pyrowave.rs +++ b/crates/pf-encode/src/enc/windows/pyrowave.rs @@ -128,6 +128,11 @@ pub struct PyroWaveEncoder { wire_budget: pyrowave_wire::WireBudget, bitstream: Vec, pending: VecDeque, + /// The AU currently being handed out in streamed chunks (PW6 — `Some` strictly between a + /// `first` chunk and its `last`). See [`pyrowave_wire::AuChunker`]: this backend's encode is + /// synchronous, so the AU is COMPLETE before the first chunk leaves — the split is for the + /// send side, never an encode/send overlap. + chunker: Option, } // SAFETY: used only from the single encode thread; the pyrowave handles are owned and only touched @@ -255,6 +260,7 @@ impl PyroWaveEncoder { wire_budget: pyrowave_wire::WireBudget::new(), bitstream: Vec::new(), pending: VecDeque::new(), + chunker: None, }) } } @@ -676,10 +682,55 @@ impl Encoder for PyroWaveEncoder { } fn poll(&mut self) -> Result> { + // Trait contract: each AU is drained through ONE method. Erroring beats double-emitting + // the bytes the chunk cursor already handed out (which would reach the wire twice, under + // the same frame index, and fail the receiver's retro-validation). + if self.chunker.is_some() { + bail!("pyrowave: poll() on an AU already being drained through poll_chunk"); + } Ok(self.pending.pop_front()) } + // --- streamed AU (PW6) — see `pyrowave_wire::AuChunker` for what this does and does NOT buy. + // Byte-identical to the Linux twin BY CONSTRUCTION: all of the cutting lives in the shared + // helper, which compiles and unit-tests on every platform. This file cannot be compiled from + // a Linux/macOS dev box, so anything written here directly would ship unverified. + fn supports_chunked_poll(&self) -> bool { + pyrowave_wire::stream_chunk_step(self.wire_chunk).is_some() + } + + fn poll_chunk(&mut self) -> Result> { + // Finish the AU already in flight before opening the next one — the host's `handle_chunk` + // keys begin/finish off `first`/`last` and cannot interleave two AUs. + if let Some(c) = self.chunker.as_mut() { + if let Some(chunk) = c.next() { + return Ok(Some(chunk)); + } + self.chunker = None; + } + let Some(f) = self.pending.pop_front() else { + return Ok(None); + }; + // No blocking wait here (the trait allows one): `submit` already ran the whole encode + // synchronously, so an AU in `pending` is complete by construction. + match pyrowave_wire::stream_chunk_step(self.wire_chunk) { + Some(step) => Ok(self + .chunker + .insert(pyrowave_wire::AuChunker::new(f, step)) + .next()), + // Unarmed / dense: the trait's own default shape, so a host that polls chunks anyway + // still gets whole AUs. + None => Ok(Some(crate::AuChunk::whole(f))), + } + } + fn reset(&mut self) -> bool { + // A rebuild forfeits every in-flight frame — including an AU only half-handed-out through + // `poll_chunk`. Dropping the cursor here (ahead of every `pending.clear()` arm below) is + // what keeps the next `poll_chunk` from splicing the tail of a dead AU onto a fresh one; + // the host sees a `first` without the previous `last`, logs "streamed AU abandoned + // mid-flight" and lets the client age that frame out. + self.chunker = None; // Cheap in-place rebuild: recreate only the pyrowave encoder object (no rate-control / // reference state to preserve). The device, imported textures and fence survive. // SAFETY: encode is synchronous (no work in flight); the device outlives the swapped encoder. diff --git a/crates/pf-zerocopy/src/imp/mod.rs b/crates/pf-zerocopy/src/imp/mod.rs index f3a4a4d9..f6d38174 100644 --- a/crates/pf-zerocopy/src/imp/mod.rs +++ b/crates/pf-zerocopy/src/imp/mod.rs @@ -19,7 +19,7 @@ pub mod vkslot; pub mod vulkan; pub mod worker; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; pub use cuda::DeviceBuffer; pub use egl::{DmabufPlane, EglImporter}; @@ -261,56 +261,223 @@ pub fn gpu_import_disabled() -> bool { /// operator found `PUNKTFUNK_ZEROCOPY=0` by hand. The host already knows how to encode that /// machine — capture just has to stop handing it dmabufs. Latching here is what makes the next /// session negotiate CPU frames on its own. -static RAW_DMABUF_FAILURE_STREAK: AtomicU32 = AtomicU32::new(0); -static RAW_DMABUF_DISABLED: AtomicBool = AtomicBool::new(false); /// Below the encoder's own rebuild budget, so the latch is set before the session it doomed ends. const RAW_DMABUF_FAILURE_LATCH: u32 = 3; -/// Record an encoder-side raw-dmabuf import failure. Latches the process-wide disable after -/// `RAW_DMABUF_FAILURE_LATCH` consecutive failures. +/// Consecutive capture rebuilds whose dmabuf-only offer never negotiated before the passthrough is +/// latched off. **2 = one retry**, deliberately: each failed negotiation costs a ~10 s stall, so a +/// larger budget is paid by the user in dead air. One retry is enough to survive a compositor +/// caught mid-restart, which is the transient this exists for; a compositor that genuinely never +/// accepts keeps the same capture identity, so its streak accumulates and it latches on the second +/// try — one extra stall versus the old behaviour, once per host lifetime. +const RAW_DMABUF_NEGOTIATION_LATCH: u32 = 2; + +/// The raw-dmabuf passthrough's off-switch — **two causes with two different lifetimes**, which is +/// the whole point of this type. +/// +/// They used to share one `AtomicBool`, so the cheap recoverable cause (a negotiation that timed +/// out, possibly because the compositor was mid-restart) was as permanent as the expensive +/// unrecoverable one (an encoder that cannot import what this compositor allocates). Once either +/// fired, EVERY later session on the host captured CPU frames until the process was restarted — +/// including sessions against a completely different compositor and node, which had never failed +/// at anything. +/// +/// * **Import failures stay sticky.** A driver that will not take what the compositor allocates +/// refuses identically on every retry, and the encode-stall recovery above cannot tell that from +/// a transient — it rebuilt the same failing encoder five times and then ended the session, on +/// every connection, forever. That is what this latch was born to stop, and it must keep +/// stopping it. +/// * **Negotiation timeouts get a retry budget** ([`RAW_DMABUF_NEGOTIATION_LATCH`]). +/// * **Both are keyed to a capture identity.** A new node id — a fresh virtual output, the +/// Bazzite Gaming↔Desktop switch, a compositor restart — is a genuinely different question, so +/// it earns a fresh dmabuf attempt instead of inheriting a verdict about something else. +/// +/// Atomics rather than a lock because [`note_import_ok`](Self::note_import_ok) is on the per-frame +/// import path; everything else here runs at pipeline build or on failure. +#[derive(Debug)] +pub struct RawDmabufLatch { + import_streak: AtomicU32, + import_latched: AtomicBool, + negotiation_streak: AtomicU32, + negotiation_latched: AtomicBool, + /// The capture identity the counters above describe. `u64::MAX` = nothing observed yet (a real + /// identity is a node id, so it can never collide with the sentinel). + identity: AtomicU64, +} + +/// Nothing observed yet — distinct from any real capture identity. +const NO_IDENTITY: u64 = u64::MAX; + +impl RawDmabufLatch { + pub const fn new() -> Self { + RawDmabufLatch { + import_streak: AtomicU32::new(0), + import_latched: AtomicBool::new(false), + negotiation_streak: AtomicU32::new(0), + negotiation_latched: AtomicBool::new(false), + identity: AtomicU64::new(NO_IDENTITY), + } + } + + /// Whether the raw-dmabuf passthrough is currently off, for either cause. + pub fn disabled(&self) -> bool { + self.import_latched.load(Ordering::Relaxed) + || self.negotiation_latched.load(Ordering::Relaxed) + } + + /// Tell the latch which capture is about to be built. A DIFFERENT capture from the one the + /// current verdict was formed against clears every counter and both latches, so the new + /// pipeline earns a fresh dmabuf attempt. + /// + /// Returns `true` only when that clear actually **re-armed something** — i.e. the identity + /// changed *and* a latch was set. Deliberately not "the identity changed": every session on a + /// fresh virtual output changes it, and a caller that logged on that would print a re-arm line + /// on every healthy session open, which is noise. `true` means "this capture would have been + /// forced to CPU by an earlier capture's verdict, and no longer is". + /// + /// Call this BEFORE reading [`disabled`](Self::disabled) for a negotiation decision, or the + /// decision is made against the previous capture's verdict. + pub fn observe_capture(&self, identity: u64) -> bool { + if self.identity.swap(identity, Ordering::Relaxed) == identity { + return false; + } + let was_latched = self.disabled(); + self.import_streak.store(0, Ordering::Relaxed); + self.import_latched.store(false, Ordering::Relaxed); + self.negotiation_streak.store(0, Ordering::Relaxed); + self.negotiation_latched.store(false, Ordering::Relaxed); + was_latched + } + + /// Record an encoder-side raw-dmabuf import failure. Returns `true` if this failure is the one + /// that latched the passthrough off. + pub fn note_import_failure(&self) -> Option { + let streak = self.import_streak.fetch_add(1, Ordering::Relaxed) + 1; + (streak >= RAW_DMABUF_FAILURE_LATCH && !self.import_latched.swap(true, Ordering::Relaxed)) + .then_some(streak) + } + + /// Record a raw dmabuf that imported and encoded — resets the failure streak. The per-frame + /// hot path, hence a single relaxed store. + /// + /// Deliberately does NOT clear `import_latched`: once the latch fires, capture has already + /// moved to CPU frames, so there are no more dmabuf imports to succeed. Only a new capture + /// identity clears it. + pub fn note_import_ok(&self) { + self.import_streak.store(0, Ordering::Relaxed); + } + + /// Record a capture rebuild whose dmabuf-only offer never negotiated. Returns `Some(streak)` + /// if this is the failure that latched the passthrough off, `None` while retries remain. + pub fn note_negotiation_timeout(&self) -> Option { + let streak = self.negotiation_streak.fetch_add(1, Ordering::Relaxed) + 1; + (streak >= RAW_DMABUF_NEGOTIATION_LATCH + && !self.negotiation_latched.swap(true, Ordering::Relaxed)) + .then_some(streak) + } + + /// Record a capture whose dmabuf offer DID negotiate — the retry budget is per consecutive + /// run of failures, so a success spends none of it. + pub fn note_negotiation_ok(&self) { + self.negotiation_streak.store(0, Ordering::Relaxed); + } + + /// Diagnostic for the session-open line: which cause (if any) currently holds it off. + pub fn state(&self) -> &'static str { + match ( + self.import_latched.load(Ordering::Relaxed), + self.negotiation_latched.load(Ordering::Relaxed), + ) { + (true, true) => "latched: encoder-import + negotiation", + (true, false) => "latched: encoder-import failures", + (false, true) => "latched: negotiation timeouts", + (false, false) => "live", + } + } +} + +impl Default for RawDmabufLatch { + fn default() -> Self { + Self::new() + } +} + +static RAW_DMABUF: RawDmabufLatch = RawDmabufLatch::new(); + +/// Record an encoder-side raw-dmabuf import failure. Latches the passthrough off after +/// `RAW_DMABUF_FAILURE_LATCH` consecutive failures, until the capture identity changes. pub fn note_raw_dmabuf_import_failure(reason: &str) { - let streak = RAW_DMABUF_FAILURE_STREAK.fetch_add(1, Ordering::Relaxed) + 1; - if streak >= RAW_DMABUF_FAILURE_LATCH && !RAW_DMABUF_DISABLED.swap(true, Ordering::Relaxed) { + if let Some(streak) = RAW_DMABUF.note_import_failure() { tracing::error!( streak, reason, - "zero-copy raw-dmabuf passthrough disabled for this host process: the encoder failed \ - to import the compositor's dmabuf {streak} times in a row — captures fall back to the \ - CPU path (slower, but this host could not stream at all otherwise)" + "zero-copy raw-dmabuf passthrough disabled: the encoder failed to import the \ + compositor's dmabuf {streak} times in a row — captures fall back to the CPU path \ + (slower, but this host could not stream at all otherwise). A new capture (different \ + node / compositor) clears this." ); } } /// Record a raw dmabuf that imported and encoded — resets the failure streak. pub fn note_raw_dmabuf_import_ok() { - RAW_DMABUF_FAILURE_STREAK.store(0, Ordering::Relaxed); + RAW_DMABUF.note_import_ok(); } /// Latch the raw-dmabuf passthrough off because its dmabuf-only *offer never negotiated* — the -/// CAPTURE-side counterpart to [`note_raw_dmabuf_import_failure`]'s encoder-side streak. One -/// timeout is conclusive for this offer (a compositor that cannot allocate the requested -/// LINEAR/modifier BGRx dmabuf refuses it identically on every retry), so there is no streak to -/// count: the next capture skips the passthrough and negotiates SHM/CPU instead of re-running the -/// same 10 s timeout on every reconnect. +/// CAPTURE-side counterpart to [`note_raw_dmabuf_import_failure`]'s encoder-side streak. +/// +/// Unlike the import streak this gets a retry budget: the offer can time out because the +/// compositor was mid-restart rather than because it will never accept, and the old behaviour +/// (one timeout = CPU capture for the rest of the host's life, for every compositor and every +/// node) turned a transient into a permanent downgrade nobody could see. /// /// Scoped deliberately. This used to be `note_vaapi_dmabuf_failed`, which fed [`enabled`] and so -/// disabled ALL zero-copy host-wide — see [`enabled`]. `RAW_DMABUF_DISABLED` gates only the -/// raw-passthrough decision, so the EGL→CUDA importer that a later NVENC session builds is -/// untouched. +/// disabled ALL zero-copy host-wide — see [`enabled`]. It gates only the raw-passthrough decision, +/// so the EGL→CUDA importer that a later NVENC session builds is untouched. pub fn note_raw_dmabuf_negotiation_failed() { - if !RAW_DMABUF_DISABLED.swap(true, Ordering::Relaxed) { - tracing::warn!( - "zero-copy raw-dmabuf passthrough disabled for this host process: the compositor never \ - accepted the dmabuf-only capture offer, so later captures negotiate the CPU path \ - instead of repeating that timeout (the EGL→CUDA import path is NOT affected)" - ); + match RAW_DMABUF.note_negotiation_timeout() { + Some(streak) => tracing::warn!( + streak, + "zero-copy raw-dmabuf passthrough disabled: the compositor did not accept the \ + dmabuf-only capture offer {streak} builds in a row, so later captures negotiate the \ + CPU path instead of repeating that timeout (the EGL→CUDA import path is NOT \ + affected). A new capture (different node / compositor) clears this." + ), + None => tracing::warn!( + "the compositor did not accept the dmabuf-only capture offer — retrying dmabuf on the \ + next capture build before giving up on it" + ), } } -/// True once repeated encoder import failures latched the raw-dmabuf passthrough off (see -/// [`note_raw_dmabuf_import_failure`]). +/// Record a capture whose dmabuf offer negotiated — spends none of the retry budget. +pub fn note_raw_dmabuf_negotiation_ok() { + RAW_DMABUF.note_negotiation_ok(); +} + +/// Tell the latch which capture is about to be built, so a verdict formed against a DIFFERENT +/// compositor/node is not inherited. Returns `true` if a latch was cleared by the change. +pub fn note_raw_dmabuf_capture(identity: u64) -> bool { + let cleared = RAW_DMABUF.observe_capture(identity); + if cleared { + tracing::info!( + identity, + "zero-copy raw-dmabuf passthrough re-armed: this is a different capture from the one \ + that failed, so it gets a fresh dmabuf attempt" + ); + } + cleared +} + +/// True while either cause holds the raw-dmabuf passthrough off (see [`RawDmabufLatch`]). pub fn raw_dmabuf_import_disabled() -> bool { - RAW_DMABUF_DISABLED.load(Ordering::Relaxed) + RAW_DMABUF.disabled() +} + +/// Which cause holds the passthrough off, for the session-open diagnostic line. +pub fn raw_dmabuf_latch_state() -> &'static str { + RAW_DMABUF.state() } /// The EGL→CUDA twin of the raw-passthrough negotiation latch: the capture advertised the GPU @@ -564,4 +731,131 @@ mod tests { note_gpu_import_death(); // third consecutive death assert!(gpu_import_disabled()); } + + // ---- PW3: the raw-dmabuf latch's two lifetimes ------------------------------------------ + // + // Against a LOCAL `RawDmabufLatch`, never the process-wide static: these assertions are about + // the state machine, and sharing one global across a test binary's threads is how a latch test + // becomes order-dependent. + + /// The expensive cause stays sticky. A driver that cannot import what this compositor + /// allocates refuses identically every time, and the encode-stall recovery cannot tell that + /// from a transient — this latch is what stops it rebuilding the same doomed encoder forever. + #[test] + fn import_failures_latch_and_stay_latched() { + let l = RawDmabufLatch::new(); + assert!(!l.disabled()); + assert_eq!(l.note_import_failure(), None); // 1 + assert_eq!(l.note_import_failure(), None); // 2 + assert!(!l.disabled(), "must not latch before the streak completes"); + assert_eq!(l.note_import_failure(), Some(3)); + assert!(l.disabled()); + // Only the FIRST crossing reports, so the error line cannot repeat per frame. + assert_eq!(l.note_import_failure(), None); + // A success resets the streak but must NOT unlatch: once capture moved to CPU frames there + // are no more dmabuf imports, so an "ok" here would be about something else entirely. + l.note_import_ok(); + assert!(l.disabled()); + } + + /// A run of failures broken by a success spends none of the budget — the streak is + /// consecutive-only, which is what makes an occasional failure survivable. + #[test] + fn a_success_breaks_the_import_streak() { + let l = RawDmabufLatch::new(); + l.note_import_failure(); + l.note_import_failure(); + l.note_import_ok(); + assert_eq!(l.note_import_failure(), None, "streak restarted at 1"); + assert_eq!(l.note_import_failure(), None); + assert!(!l.disabled()); + assert_eq!(l.note_import_failure(), Some(3)); + } + + /// The cheap cause gets a retry. This is the behaviour change PW3 exists for: one timeout used + /// to mean CPU capture for the rest of the host's life, on every compositor and every node. + #[test] + fn a_negotiation_timeout_is_retried_before_it_latches() { + let l = RawDmabufLatch::new(); + assert_eq!(l.note_negotiation_timeout(), None, "first one retries"); + assert!( + !l.disabled(), + "the next capture build must still be allowed to try dmabuf" + ); + assert_eq!(l.note_negotiation_timeout(), Some(2)); + assert!(l.disabled()); + assert_eq!(l.note_negotiation_timeout(), None, "reports once"); + } + + /// A capture that negotiates credits the budget back, so a compositor that fails once and then + /// works never accumulates its way to a latch across an evening of reconnects. + #[test] + fn a_negotiated_capture_credits_the_retry_budget() { + let l = RawDmabufLatch::new(); + for _ in 0..10 { + assert_eq!(l.note_negotiation_timeout(), None); + l.note_negotiation_ok(); + } + assert!(!l.disabled()); + } + + /// A different capture is a different question. New node id (fresh virtual output, compositor + /// restart, the Bazzite Gaming↔Desktop switch) clears BOTH causes — the same capture does not. + #[test] + fn a_new_capture_identity_clears_the_latch_and_the_same_one_does_not() { + let l = RawDmabufLatch::new(); + // Nothing is latched yet, so observing a new capture re-arms NOTHING — that is what the + // return value means, and it is why a healthy session open logs no re-arm line. + assert!( + !l.observe_capture(7), + "nothing was latched, nothing re-armed" + ); + assert!(!l.observe_capture(7), "same capture, no clear"); + for _ in 0..RAW_DMABUF_FAILURE_LATCH { + l.note_import_failure(); + } + assert!(l.disabled()); + assert!( + !l.observe_capture(7), + "the SAME capture must keep its verdict — this is the 10s-stall hazard the latch exists for" + ); + assert!(l.disabled()); + assert!(l.observe_capture(9), "a different node re-arms it"); + assert!(!l.disabled()); + // ...and the streaks reset with it, so the fresh attempt gets a full budget. + assert_eq!(l.note_import_failure(), None); + } + + /// The negotiation latch is keyed the same way — a compositor restart must not inherit the + /// previous one's timeout verdict. + #[test] + fn a_new_capture_identity_clears_the_negotiation_latch_too() { + let l = RawDmabufLatch::new(); + l.observe_capture(1); + l.note_negotiation_timeout(); + l.note_negotiation_timeout(); + assert!(l.disabled()); + assert!(l.observe_capture(2)); + assert!(!l.disabled()); + } + + /// The session-open line has to name WHICH cause holds it off — "cpu because nothing here + /// does dmabuf" and "cpu because something failed earlier" are different bugs. + #[test] + fn latch_state_names_the_cause() { + let l = RawDmabufLatch::new(); + assert_eq!(l.state(), "live"); + l.note_negotiation_timeout(); + l.note_negotiation_timeout(); + assert_eq!(l.state(), "latched: negotiation timeouts"); + let l = RawDmabufLatch::new(); + for _ in 0..RAW_DMABUF_FAILURE_LATCH { + l.note_import_failure(); + } + assert_eq!(l.state(), "latched: encoder-import failures"); + for _ in 0..RAW_DMABUF_NEGOTIATION_LATCH { + l.note_negotiation_timeout(); + } + assert_eq!(l.state(), "latched: encoder-import + negotiation"); + } } diff --git a/crates/punktfunk-core/src/client/frame_channel.rs b/crates/punktfunk-core/src/client/frame_channel.rs index 4d5c3dad..f8c20b6e 100644 --- a/crates/punktfunk-core/src/client/frame_channel.rs +++ b/crates/punktfunk-core/src/client/frame_channel.rs @@ -353,6 +353,18 @@ impl FrameChannel { /// all-intra stream ([`Self::set_all_intra`]) a multi-deep queue drains to the NEWEST AU /// instead — the skipped ones are already superseded and decode independently, so showing /// them only adds latency. + /// + /// ⚠ **The all-intra drain counts QUEUE ENTRIES and assumes one entry == one AU.** That holds + /// today only because slice-progressive delivery is refused on PyroWave + /// (`client/pump/handshake.rs`; see [`crate::session::Session::set_deliver_frame_parts`]). + /// Turn parts on for an all-intra stream and one AU pushes several entries, at which point + /// `len > 1` no longer means "the consumer is behind": this fires mid-AU, hands back a SUFFIX + /// and `clear()`s that AU's own prefixes — a headerless frame, every frame. Anyone making the + /// two composable must skip whole SUPERSEDED AUs (drop up to the newest entry whose + /// `part.first` is set, never split an AU), give `push`'s `FRAME_QUEUE_HARD_CAP` eviction the + /// same rule, and count `skipped_total` in AUs. Host-side streamed AUs + /// ([`crate::quic::VIDEO_CAP_STREAMED_AU`]) are NOT affected — they still arrive as one + /// completed `Frame` per AU. pub(crate) fn pop(&self, timeout: Duration) -> FramePop { let mut st = self.inner.lock().unwrap(); if st.q.is_empty() && !st.closed { diff --git a/crates/punktfunk-core/src/client/pump/handshake.rs b/crates/punktfunk-core/src/client/pump/handshake.rs index 57485d5a..62b1b27a 100644 --- a/crates/punktfunk-core/src/client/pump/handshake.rs +++ b/crates/punktfunk-core/src/client/pump/handshake.rs @@ -229,7 +229,10 @@ pub(super) async fn connect_and_handshake(args: &WorkerArgs) -> Result Arc 1500) because it is not free: quinn sizes +/// its endpoint receive buffer as `max_udp_payload_size × max_receive_segments × BATCH_SIZE`, +/// which on a GRO-capable Linux/Android client is 64 × 32 segments — ~2.9 MiB at the 1472 +/// default, ~18 MiB at jumbo. A jumbo LAN is a deliberate deployment; every other client keeps +/// today's buffer to the byte. Without the opt-in this returns the stock config, so the +/// advertisement, the wire, and the memory are all unchanged. +fn endpoint_config() -> quinn::EndpointConfig { + let mut cfg = quinn::EndpointConfig::default(); + if let Some(mtu) = crate::config::jumbo_wire_mtu() { + // Derived exactly like the probe ceiling above (IPv4 overhead — a v6 peer's sealed + // target is smaller, so this covers it), and clamped into quinn's accepted range. + let shard = crate::config::jumbo_shard_payload_for( + mtu, + std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED), + ); + let accept = crate::config::sealed_datagram_bytes(shard).clamp(1200, 65_527) as u16; + if cfg.max_udp_payload_size(accept).is_ok() { + tracing::info!( + max_udp_payload_size = accept, + wire_mtu = mtu, + "jumbo opt-in: this endpoint advertises a jumbo QUIC receive ceiling, so the \ + peer's MTU discovery can prove a jumbo path (it is capped by this value)" + ); + } + } + cfg +} + /// Server endpoint with a fresh self-signed certificate (tests/dev — production hosts /// persist an identity and use [`server_with_identity`] so clients can pin it). pub fn server(addr: std::net::SocketAddr) -> anyhow_result::Result { @@ -238,7 +281,15 @@ pub fn client_pinned_with_identity( .map_err(|e| anyhow_result::Error::msg(format!("quic client config: {e}")))?; let mut client_cfg = quinn::ClientConfig::new(Arc::new(quic_cfg)); client_cfg.transport_config(stream_transport()); // keep-alive — see stream_transport - let mut ep = quinn::Endpoint::client("0.0.0.0:0".parse().unwrap())?; + + // `Endpoint::client` hardcodes `EndpointConfig::default()`, whose 1472-byte + // `max_udp_payload_size` caps the HOST's MTU discovery (see `endpoint_config`), so the + // endpoint is built by hand to carry the jumbo opt-in. Same bind as before + // (`0.0.0.0:0`, v4 — no dual-stack flag to reproduce) and the same default runtime. + let socket = std::net::UdpSocket::bind("0.0.0.0:0")?; + let runtime = quinn::default_runtime() + .ok_or_else(|| anyhow_result::Error::msg("no async runtime found".into()))?; + let mut ep = quinn::Endpoint::new(endpoint_config(), None, socket, runtime)?; ep.set_default_client_config(client_cfg); Ok(ep) })(); @@ -348,4 +399,80 @@ mod tests { let _ = super::stream_transport_idle(std::time::Duration::MAX); let _ = super::stream_transport_idle(std::time::Duration::ZERO); } + + /// Where a connection's MTU discovery is allowed to climb to, measured rather than argued + /// (PW7a). Loopback's own MTU is 64 KiB, so the ONLY thing that can stop the search here is + /// configuration — which makes this a clean instrument for the two ceilings: + /// + /// * **leg A** — server opted in, client NOT: the search stalls at the client's default + /// `max_udp_payload_size` advertisement (1472) no matter how high the server's probe + /// ceiling is. This is why the shipped jumbo grow could never fire: `wire_mtu.rs` waits + /// for a settle at the sealed jumbo size and the peer's transport parameter forbids it. + /// * **leg B** — both opted in: the search reaches the sealed jumbo datagram, and the + /// elapsed time is what the `Welcome`'s bounded proof-wait has to cover. + /// + /// `#[ignore]`d: it sets process-wide env (each endpoint reads the opt-in at construction, + /// which is exactly how the two legs are built) and spends seconds of wall clock. + /// Run it alone: `cargo test -p punktfunk-core --features quic mtu_discovery -- --ignored + /// --nocapture --test-threads=1`. + #[tokio::test] + #[ignore = "measurement: sets process env and takes ~15 s of wall clock"] + async fn mtu_discovery_climbs_only_as_high_as_the_peer_advertises() { + async fn climb(server_jumbo: bool, client_jumbo: bool) -> (u16, u128) { + let set = |on: bool| { + if on { + std::env::set_var("PUNKTFUNK_JUMBO", "1"); + } else { + std::env::remove_var("PUNKTFUNK_JUMBO"); + } + }; + set(server_jumbo); + let server = endpoint::server("127.0.0.1:0".parse().unwrap()).unwrap(); + let addr = server.local_addr().unwrap(); + set(client_jumbo); + let client = endpoint::client_insecure().unwrap(); + set(false); + let accept = tokio::spawn(async move { + let incoming = server.accept().await.expect("incoming"); + let conn = incoming.await.expect("host side connects"); + (server, conn) + }); + let client_conn = client.connect(addr, "punktfunk").unwrap().await.unwrap(); + let (_server_ep, host_conn) = accept.await.unwrap(); + // A stream write gives the driver something to transmit, which is what starts the + // search (probes ride `poll_transmit`); after that each probe's ack drives the next. + let mut s = host_conn.open_uni().await.unwrap(); + s.write_all(b"go").await.unwrap(); + let want = crate::config::sealed_datagram_bytes(crate::config::jumbo_shard_payload_for( + 9000, + std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED), + )) as u16; + let t0 = std::time::Instant::now(); + let mut mtu = host_conn.stats().path.current_mtu; + while t0.elapsed() < std::time::Duration::from_secs(6) && mtu < want { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + mtu = host_conn.stats().path.current_mtu; + } + let elapsed = t0.elapsed().as_millis(); + drop(client_conn); + drop(client); + (mtu, elapsed) + } + + let (capped, _) = climb(true, false).await; + println!("leg A (server opted in, client not): settled at {capped} B UDP payload"); + assert_eq!( + capped, 1472, + "a peer that advertises the stock max_udp_payload_size caps the search at 1472 — \ + the whole point of raising it on the client endpoint" + ); + + let (grown, ms) = climb(true, true).await; + println!("leg B (both opted in): reached {grown} B UDP payload in {ms} ms"); + assert!( + grown >= 8972, + "both sides opted in, loopback MTU is 64 KiB — discovery should reach the sealed \ + jumbo datagram, got {grown}" + ); + } } diff --git a/crates/punktfunk-core/src/session.rs b/crates/punktfunk-core/src/session.rs index a95d6ff5..398871c2 100644 --- a/crates/punktfunk-core/src/session.rs +++ b/crates/punktfunk-core/src/session.rs @@ -677,8 +677,21 @@ impl Session { /// [`Frame::part`]` = Some` while the rest is still on the wire, instead of one whole-AU /// delivery (the slice-progressive decode path — [`crate::packet::USER_FLAG_SLICE_STREAM`]). /// With it on, EVERY video frame delivery carries `part: Some` (a frame with no early - /// parts arrives as the degenerate `{offset: 0, first, last}` whole). Do not combine with - /// an all-intra (PyroWave) stream: its newest-wins draining assumes whole AUs. + /// parts arrives as the degenerate `{offset: 0, first, last}` whole). + /// + /// **Do not combine with an all-intra (PyroWave) stream**, and the reason is sharper than + /// "newest-wins draining assumes whole AUs" (2026-08-08, PW6): the drain + /// (`client::frame_channel::FrameChannel::pop`) counts QUEUE ENTRIES and takes one entry to be + /// one AU. With parts on, a single AU pushes K entries, so `len > 1` stops meaning "the consumer + /// is behind" — the drain fires mid-AU, returns the newest entry (a SUFFIX) and clears that + /// same AU's prefixes. For PyroWave that is unrecoverable rather than lossy: the sequence + /// header lives in window 0 of every AU, so every frame would arrive headerless. Making the + /// two composable means teaching the drain to skip whole superseded AUs (never to split one) + /// — see the PW6 section of `design/linux-host-performance-wave2-pyrowave.md`. + /// + /// Note this is a DIFFERENT axis from the host's streamed-AU wire + /// ([`crate::quic::VIDEO_CAP_STREAMED_AU`]): a streamed AU still completes as ONE `Frame` + /// here, so it is unaffected by any of the above. pub fn set_deliver_frame_parts(&mut self, on: bool) { self.reassembler.set_deliver_parts(on); } diff --git a/crates/punktfunk-host/src/main.rs b/crates/punktfunk-host/src/main.rs index ed28763d..209e7e88 100644 --- a/crates/punktfunk-host/src/main.rs +++ b/crates/punktfunk-host/src/main.rs @@ -844,6 +844,7 @@ fn parse_spike(args: &[String]) -> Result { let mut bitrate_mbps = 20u64; let mut out: Option = None; let mut loopback = true; + let mut wire_chunk: Option = None; let mut i = 0; while i < args.len() { @@ -890,7 +891,13 @@ fn parse_spike(args: &[String]) -> Result { "h264" => Codec::H264, "h265" | "hevc" => Codec::H265, "av1" => Codec::Av1, - other => bail!("unknown --codec '{other}' (h264|h265|av1)"), + // The spike is the only way to drive a PyroWave capture→encode pass without + // a client, which is what the Linux-host PyroWave work measures against. + // Needs the `pyrowave` feature (default-on) and pairs with + // `PUNKTFUNK_ENCODER=pyrowave`, which is what puts the CAPTURE side on the + // raw-dmabuf passthrough. + "pyrowave" => Codec::PyroWave, + other => bail!("unknown --codec '{other}' (h264|h265|av1|pyrowave)"), } } "--bitrate" => { @@ -900,6 +907,12 @@ fn parse_spike(args: &[String]) -> Result { } "--out" => out = Some(PathBuf::from(next()?)), "--no-loopback" => loopback = false, + "--wire-chunk" => { + let v: usize = next()? + .parse() + .map_err(|_| anyhow::anyhow!("bad --wire-chunk (bytes)"))?; + wire_chunk = (v > 0).then_some(v); + } "-h" | "--help" => { print_usage(); std::process::exit(0); @@ -934,6 +947,7 @@ fn parse_spike(args: &[String]) -> Result { bitrate_bps: bitrate_mbps.saturating_mul(1_000_000), out, loopback, + wire_chunk, }) } @@ -1007,11 +1021,18 @@ SPIKE OPTIONS: KWin virtual output at --width x --height and captures it --seconds capture duration in seconds (default: 5) --fps target frame rate (default: 60) - --codec NVENC codec (default: h265) + --codec + encode codec (default: h265). 'pyrowave' also wants + PUNKTFUNK_ENCODER=pyrowave so capture takes the passthrough --bitrate target bitrate in Mbps (default: 20) --width --height synthetic source size (default: 1920x1080) --out raw Annex-B output (default: /tmp/punktfunk-spike.) --no-loopback skip the punktfunk_core round-trip verification + --wire-chunk PyroWave datagram-aligned packetization at this shard payload + (a real session passes its negotiated shard_payload, e.g. 1408). + With PUNKTFUNK_PYROWAVE_STREAMED_AU=1 also armed, the AU is + drained through poll_chunk and sealed as a STREAMED wire frame + (VIDEO_CAP_STREAMED_AU), then byte-verified by the loopback -h, --help this help NOTES: diff --git a/crates/punktfunk-host/src/native.rs b/crates/punktfunk-host/src/native.rs index 122de3f3..be01a1f9 100644 --- a/crates/punktfunk-host/src/native.rs +++ b/crates/punktfunk-host/src/native.rs @@ -1148,7 +1148,12 @@ async fn serve_session( // path verdict (WARN + learned clamp for the next session on a constrained path; clears // a stale clamp on a healthy one) — and, with the driver above, heal or grow THIS // session mid-stream. Bounded ~10 s task unless a jumbo grow leaves it as revert guard. - wire_mtu::spawn_watch(conn.clone(), welcome.shard_payload as usize, shard_reneg); + wire_mtu::spawn_watch( + conn.clone(), + welcome.shard_payload as usize, + hello.max_shard_payload, + shard_reneg, + ); // Negotiated cursor forwarding: the HOST_CAP_CURSOR bit the Welcome advertised, read back // rather than recomputed (`handshake::cursor_forward` computed it once, with the encoder // blend-capability gate — re-running it here could drift, and would re-probe). diff --git a/crates/punktfunk-host/src/native/handshake.rs b/crates/punktfunk-host/src/native/handshake.rs index 2eae0786..60afb95b 100644 --- a/crates/punktfunk-host/src/native/handshake.rs +++ b/crates/punktfunk-host/src/native/handshake.rs @@ -148,7 +148,6 @@ pub(super) async fn negotiate( Option, Option, )> { - let peer = conn.remote_address(); let mut hello = Hello::decode(first).map_err(|e| anyhow!("Hello decode: {e:?}"))?; if hello.abi_version != punktfunk_core::WIRE_VERSION { close_rejected( @@ -497,6 +496,11 @@ pub(super) async fn negotiate( let (data_sock, direct) = bind_data_socket(data_port)?; let udp_port = data_sock.local_addr()?.port(); + // The session's video geometry (see the `shard_payload` field below). Resolved before the + // Welcome struct because a path a previous session proved jumbo is given a bounded moment + // to re-prove itself live on THIS connection — the awaited part of `negotiated_shard_payload`. + let shard_payload = wire_mtu::negotiated_shard_payload(conn, hello.max_shard_payload).await; + let mut key = [0u8; 16]; rand::thread_rng().fill_bytes(&mut key); // Fresh per-session salt alongside the fresh key. GCM nonce uniqueness only *requires* one @@ -548,14 +552,15 @@ pub(super) async fn negotiate( // hardcoded 1452 overshot the v4 ceiling (its math forgot the header/crypto ride // inside the UDP payload) and silently IP-fragmented EVERY video datagram, doubling // per-datagram loss on Wi-Fi — the "100 Mbps badly fails on the phone" root cause. - // Negotiated, so the client follows. Jumbo (≈8900) is a future negotiated bump (needs - // MAX_DATAGRAM_BYTES raised + end-to-end 9000 MTU). - // Resolution order (wire_mtu.rs): `PUNKTFUNK_WIRE_MTU` operator override, then a path - // budget learned from a prior session whose QUIC MTU discovery settled below the - // video-datagram ceiling (the "VPN on the host blackholes every video packet" field - // shape — small flows pass, the stream is an endless black screen), then this family - // default. Healthy paths take the default branch and are byte-identical to before. - shard_payload: wire_mtu::negotiated_shard_payload(peer.ip()) as u16, + // Negotiated, so the client follows. + // Resolution order (wire_mtu.rs): a JUMBO start (≈8900) on a path a previous session + // proved AND this connection has just re-proved live, then the `PUNKTFUNK_WIRE_MTU` + // operator override, then a path budget learned from a prior session whose QUIC MTU + // discovery settled below the video-datagram ceiling (the "VPN on the host blackholes + // every video packet" field shape — small flows pass, the stream is an endless black + // screen), then this family default. Healthy paths take the default branch and are + // byte-identical to before. + shard_payload: shard_payload as u16, encrypt: true, key, salt, diff --git a/crates/punktfunk-host/src/native/wire_mtu.rs b/crates/punktfunk-host/src/native/wire_mtu.rs index 408e19b7..9b963b4f 100644 --- a/crates/punktfunk-host/src/native/wire_mtu.rs +++ b/crates/punktfunk-host/src/native/wire_mtu.rs @@ -24,6 +24,14 @@ //! - **Heal** — the next handshake from that peer clamps `shard_payload` to the recorded //! budget, so a reconnect fixes the stream. A later session that reaches the ceiling erases //! the record (the learn/heal loop is self-correcting in both directions). +//! - **Grow** (PW7a) — the mirror image, for the jumbo half: a connection whose discovery +//! settles at the sealed JUMBO size has proven the path carries ~8.9 KB video datagrams, and +//! the next session on that same path *starts* there instead of at the 1500-byte default. +//! PyroWave sessions cannot be re-keyed mid-stream (the client's parse window is the +//! `Welcome` value, read once over the C ABI), so the session-start value is the ONLY way +//! they ever reach jumbo — and it is exactly where ~6× fewer datagrams per frame is worth +//! the most. See [`jumbo_session_start`] for why a remembered verdict alone is never +//! allowed to seal one byte above the default. use std::collections::HashMap; use std::net::IpAddr; @@ -63,10 +71,155 @@ fn learned() -> &'static Mutex> { LEARNED.get_or_init(|| Mutex::new(HashMap::new())) } -/// The shard payload for a new session to `peer`: `PUNKTFUNK_WIRE_MTU` override, else the -/// peer's learned path budget, else the family default (today's exact behavior). Logs whenever -/// the result differs from the default. -pub(super) fn negotiated_shard_payload(peer: IpAddr) -> usize { +/// Identity of a PATH, not of a peer — the key the jumbo verdict is filed under. +/// +/// The clamp above is keyed by peer IP alone, and that is safe *because being wrong is benign*: +/// a stale clamp only makes video datagrams smaller than they had to be. A stale GROW is the +/// opposite — one oversized datagram on a 1500-byte path is silently dropped, which is the +/// "connects fine, black screen forever" shape this whole module exists to kill. So the grow +/// keys strictly: a verdict earned over the host's 10 GbE NIC does not apply to the same peer +/// IP reached over the host's Wi-Fi or a VPN adapter, because those are different routes with +/// different MTUs. +/// +/// `local` is `Connection::local_ip()` (the address the connection was actually received on); +/// `None` where the platform can't report it, which degrades this key to the clamp's — safely, +/// because the live re-proof in [`jumbo_session_start`] is what actually protects the grow. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +struct PathKey { + local: Option, + peer: IpAddr, +} + +/// A path that a completed MTU-discovery search proved carries jumbo video datagrams. +#[derive(Clone, Copy, Debug)] +struct JumboVerdict { + /// The settled UDP-payload budget the proof measured. + udp_budget: u16, + /// The operator's jumbo target when the proof was taken. A changed `PUNKTFUNK_JUMBO` / + /// `PUNKTFUNK_WIRE_MTU` invalidates it rather than being silently reinterpreted. + target_wire_mtu: usize, + /// When it was taken ([`JUMBO_VERDICT_TTL`]). + at: std::time::Instant, +} + +/// How long a jumbo verdict may be redeemed for. Contrary evidence erases it long before this +/// (any settle below the sealed target, on any later session over the same path — the same +/// self-correction the clamp has), so the TTL is not the safety mechanism; it is a bound on how +/// stale an *unrefreshed* memory can get, for the case where the path changes while no session +/// is running. +const JUMBO_VERDICT_TTL: std::time::Duration = std::time::Duration::from_secs(6 * 3600); + +/// How long the `Welcome` may wait for THIS connection's MTU discovery to re-prove a jumbo +/// path. +/// +/// The wait is structural, not laziness: every connection restarts discovery from ~1200 bytes, +/// so the live proof the grow requires does not exist yet when the `Welcome` is built — and the +/// binary search up to sealed-jumbo needs an ACKED probe per step, each of which a peer may sit +/// on for its ack delay. Without a wait the gate would never pass and the feature would be dead. +/// +/// It is honestly on the bring-up critical path (`handshake.rs` sends the `Welcome` and only +/// THEN kicks the display prep), so it is bounded, returns the instant the proof lands, and is +/// entered ONLY for a path a previous session already proved jumbo — i.e. an opted-in operator +/// on a jumbo LAN, never anyone else. The worst case (the full wait, no proof) is the moved +/// laptop, and it is self-limiting: that session's watcher erases the verdict, so the next +/// connect doesn't wait at all. +const JUMBO_PROOF_WAIT: std::time::Duration = std::time::Duration::from_millis(300); +const JUMBO_PROOF_POLL: std::time::Duration = std::time::Duration::from_millis(10); + +/// Proven-jumbo paths. Same lifetime rules as [`learned`] — in-memory, re-earned in one session +/// after a host restart. +fn jumbo_verdicts() -> &'static Mutex> { + static JUMBO: OnceLock>> = OnceLock::new(); + JUMBO.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn path_key(conn: &quinn::Connection) -> PathKey { + PathKey { + local: conn.local_ip(), + peer: conn.remote_address().ip(), + } +} + +/// Everything the session-start jumbo decision reads. Every field but `proven_udp_budget` is +/// observed on THIS connection during THIS handshake — which is the point (see +/// [`jumbo_session_start`]). +#[derive(Clone, Copy, Debug)] +struct JumboStart { + /// The host operator's opt-in ([`jumbo_wire_mtu`]) — `None` = no jumbo, ever. + target_wire_mtu: Option, + /// `Hello::max_shard_payload`: the client's own receive ceiling (0 = legacy client, which + /// never gets a geometry it didn't ask for). + client_ceiling: u16, + /// `conn.stats().path.current_mtu` right now: the largest UDP payload quinn has had ACKED + /// on this connection. + live_udp_mtu: u16, + /// What a previous session over this same [`PathKey`] settled at, if any. + proven_udp_budget: Option, + /// The constrained-path clamp [`learned`] for this peer, if any. Contradictory evidence + /// (this peer black-screened on a small MTU recently) vetoes the grow — the two memories + /// are keyed differently and the safe one wins. + clamped_udp_budget: Option, +} + +/// The jumbo shard payload a session to `peer` could use, or `None` when there is nothing to +/// gain (no opt-in, a legacy/low client ceiling, or a target that isn't bigger than the family +/// default). Shared by the decision, the wait, and the watcher so all three agree on the number. +fn jumbo_target( + target_wire_mtu: Option, + client_ceiling: u16, + peer: IpAddr, +) -> Option { + let mtu = target_wire_mtu?; + let t = jumbo_shard_payload_for(mtu, peer).min(client_ceiling as usize); + let t = t - t % 2; // FEC requires even shards + (t > mtu1500_shard_payload_for(peer)).then_some(t) +} + +/// The session-START jumbo decision: `Some(shard_payload)` only when every gate below holds. +/// +/// **Why a remembered verdict is never enough.** A laptop that proved jumbo on the wired LAN +/// and comes back on Wi-Fi, a switch that lost its jumbo config, a client IP recycled by DHCP — +/// all of them present a path that cannot carry an 8.9 KB datagram, and a PyroWave session +/// sealed at that size cannot be re-keyed mid-stream, so it would black-screen for its whole +/// life. The memory therefore only decides whether it is worth WAITING for a proof; what +/// actually authorises the grow is `live_udp_mtu` — a datagram of exactly that size, acked by +/// this client, on this connection, seconds ago. That is why this is as safe as the clamp +/// despite the failure modes being opposite: a wrong memory cannot produce a jumbo `Welcome`, +/// only a live measurement can. +/// +/// The gates, in order: the host operator opted in; the client advertised enough receive +/// headroom; the target beats the family default (nothing to gain otherwise); no constrained-path +/// clamp contradicts it; a prior session over this exact path settled at or above the sealed +/// target; and this connection has re-proven it live. +fn jumbo_session_start(i: JumboStart, peer: IpAddr) -> Option { + let target = jumbo_target(i.target_wire_mtu, i.client_ceiling, peer)?; + let sealed = sealed_datagram_bytes(target); + if let Some(clamp) = i.clamped_udp_budget { + if (clamp as usize) < sealed { + return None; + } + } + if (i.proven_udp_budget? as usize) < sealed { + return None; + } + if (i.live_udp_mtu as usize) < sealed { + return None; + } + Some(target) +} + +/// The shard payload for a new session on `conn`: a proven-jumbo grow, else the +/// `PUNKTFUNK_WIRE_MTU` override, else the peer's learned path budget, else the family default +/// (today's exact behavior). Logs whenever the result differs from the default. +/// +/// `client_ceiling` is the client's `Hello::max_shard_payload`. Async only for the bounded +/// [`JUMBO_PROOF_WAIT`], which is entered *only* on a path a previous session already proved +/// jumbo — every other session resolves without awaiting anything. +pub(super) async fn negotiated_shard_payload( + conn: &quinn::Connection, + client_ceiling: u16, +) -> usize { + let peer = conn.remote_address().ip(); let env = match std::env::var("PUNKTFUNK_WIRE_MTU") { Ok(v) => match v.trim().parse::() { Ok(mtu) => Some(mtu), @@ -78,13 +231,80 @@ pub(super) fn negotiated_shard_payload(peer: IpAddr) -> usize { Err(_) => None, }; let learned_budget = learned().lock().unwrap().get(&peer).copied(); - resolve(env, learned_budget, peer) + let target_wire_mtu = jumbo_wire_mtu(); + let proven_udp_budget = fresh_verdict(path_key(conn), target_wire_mtu); + let mut jumbo = JumboStart { + target_wire_mtu, + client_ceiling, + live_udp_mtu: conn.stats().path.current_mtu, + proven_udp_budget, + clamped_udp_budget: learned_budget, + }; + // A proven path is worth waiting a moment for: MTU discovery starts when the handshake + // completes and needs an acked probe per binary-search step, so at `Welcome` time it may + // simply not have got there yet. Bounded, and only on paths that already proved it once. + let awaited_proof = proven_udp_budget + .and_then(|_| jumbo_target(target_wire_mtu, client_ceiling, peer)) + .map(|t| sealed_datagram_bytes(t) as u16); + if let Some(sealed) = awaited_proof { + if jumbo.live_udp_mtu < sealed { + let t0 = std::time::Instant::now(); + while t0.elapsed() < JUMBO_PROOF_WAIT { + tokio::time::sleep(JUMBO_PROOF_POLL).await; + jumbo.live_udp_mtu = conn.stats().path.current_mtu; + if jumbo.live_udp_mtu >= sealed { + break; + } + } + tracing::debug!( + peer = %peer, + waited_ms = t0.elapsed().as_millis() as u64, + live_udp_mtu = jumbo.live_udp_mtu, + needed = sealed, + "wire MTU: waited for this connection to re-prove its jumbo path" + ); + } + } + resolve(env, learned_budget, jumbo, peer) } -/// Pure resolution (env override > learned budget > family default) — the tested core of -/// [`negotiated_shard_payload`]. -fn resolve(env_wire_mtu: Option, learned_udp_budget: Option, peer: IpAddr) -> usize { +/// The peer's jumbo verdict if it is still redeemable: same operator target, inside the TTL. +/// A verdict that fails either test is dropped on the spot rather than left to rot. +fn fresh_verdict(key: PathKey, target_wire_mtu: Option) -> Option { + let target = target_wire_mtu?; + let mut map = jumbo_verdicts().lock().unwrap(); + let v = *map.get(&key)?; + if v.target_wire_mtu != target || v.at.elapsed() > JUMBO_VERDICT_TTL { + map.remove(&key); + return None; + } + Some(v.udp_budget) +} + +/// Pure resolution (proven jumbo > env override > learned budget > family default) — the tested +/// core of [`negotiated_shard_payload`]. +fn resolve( + env_wire_mtu: Option, + learned_udp_budget: Option, + jumbo: JumboStart, + peer: IpAddr, +) -> usize { let default = mtu1500_shard_payload_for(peer); + // First, because the two are mutually exclusive by construction: `jumbo_wire_mtu()` only + // fires above 1500, and the env branch below CLAMPS to the family default, so a + // `PUNKTFUNK_WIRE_MTU=9000` operator would otherwise get 1408 and never a jumbo start. + if let Some(p) = jumbo_session_start(jumbo, peer) { + tracing::info!( + peer = %peer, + shard_payload = p, + default, + live_udp_mtu = jumbo.live_udp_mtu, + proven_udp_budget = jumbo.proven_udp_budget, + "wire MTU: session starts at the JUMBO shard — this path proved it in a previous \ + session AND re-proved it live on this connection (~6× fewer datagrams per frame)" + ); + return p; + } if let Some(mtu) = env_wire_mtu { let p = shard_payload_for_wire_mtu(mtu, peer); if p != default { @@ -119,34 +339,73 @@ fn resolve(env_wire_mtu: Option, learned_udp_budget: Option, peer: I /// into a verdict — and, with a [`ShardReneg`] driver, act on it MID-SESSION /// (design/shard-payload-reneg.md Phase 2): a below-ceiling verdict shrinks the live wire at /// the ~3–10 s mark (session 1 heals instead of staying black), and a settled-at-jumbo -/// verdict grows it, ack-gated, when the operator opted in. Spawned once per negotiated -/// session; without a grow the task ends after the final sample (bounded ~10 s lifetime, -/// holding only a cheap `Connection` handle) — after a grow it stays as the revert guard -/// until the connection closes. +/// verdict grows it, ack-gated, when the operator opted in. The same settled-at-jumbo reading +/// also writes this path's next-session verdict (PW7a) — `client_ceiling` is the client's +/// `Hello::max_shard_payload`, which decides what "jumbo" is worth proving for this peer. +/// Spawned once per negotiated session; without a grow the task ends after the final sample +/// (bounded ~10 s lifetime, holding only a cheap `Connection` handle) — after a grow, or on a +/// session that STARTED jumbo, it stays as the revert guard until the connection closes. pub(super) fn spawn_watch( conn: quinn::Connection, session_shard_payload: usize, + client_ceiling: u16, reneg: Option, ) { tokio::spawn(async move { let peer = conn.remote_address().ip(); let ceiling = video_datagram_udp_ceiling() as u16; + // The sealed size a JUMBO proof has to reach on this path (PW7a) — `None` unless the + // operator opted in AND this client advertised the headroom. Read once: the verdict + // records the target it was proven under, and the two must be the same number. + let target_wire_mtu = jumbo_wire_mtu(); + let jumbo_proof = + jumbo_target(target_wire_mtu, client_ceiling, peer).map(sealed_datagram_bytes); // Discovery finishes in a handful of RTTs on a LAN (well under the first sample) but // needs a loss timeout per failed probe on a constrained path — the second sample // covers that with margin. Max, because discovery only ever raises `current_mtu` // (the post-grow revert guard below re-reads it live, where blackhole detection CAN - // lower it again). + // lower it again). Stop early only once nothing more is expected: with a jumbo opt-in + // the search keeps climbing past the 1500-byte ceiling, and stopping there would throw + // away the very measurement the proof needs. + let goal = jumbo_proof + .unwrap_or(ceiling as usize) + .max(ceiling as usize) as u16; let mut settled = 0u16; for wait_s in [3u64, 7] { tokio::time::sleep(std::time::Duration::from_secs(wait_s)).await; settled = settled.max(conn.stats().path.current_mtu); - if settled >= ceiling { + if settled >= goal { break; } } // The wire this session is CURRENTLY sealed at — moves on a mid-session shrink/grow. let mut current = session_shard_payload; let mut reneg = reneg; + // PW7a bookkeeping, before anything else can return: this is where a jumbo path earns + // its next-session verdict — and, far more importantly, where it LOSES it. Recording + // needs a live connection that reached the sealed target; anything else (a lower + // settle, a connection that died before the window closed, i.e. exactly what a client + // staring at a black screen does) erases, so the next session falls back to the + // 1500-byte default and has to prove itself again from scratch. + if let Some(need) = jumbo_proof { + let key = path_key(&conn); + if settled as usize >= need && conn.close_reason().is_none() { + jumbo_verdicts().lock().unwrap().insert( + key, + JumboVerdict { + udp_budget: settled, + target_wire_mtu: target_wire_mtu.unwrap_or_default(), + at: std::time::Instant::now(), + }, + ); + tracing::info!(peer = %peer, discovered_udp_mtu = settled, needed = need, + "wire MTU: this path carries JUMBO video datagrams — the next session over \ + it starts at the big shard (it still has to re-prove the path live)"); + } else if jumbo_verdicts().lock().unwrap().remove(&key).is_some() { + tracing::info!(peer = %peer, discovered_udp_mtu = settled, needed = need, + "wire MTU: jumbo verdict cleared — this path no longer proves it"); + } + } if settled >= ceiling { // The path carries full-size video datagrams — erase any stale learned clamp so // the next session returns to the default wire. @@ -154,6 +413,34 @@ pub(super) fn spawn_watch( tracing::info!(peer = %peer, "wire MTU: path re-measured at full size — learned clamp cleared"); } + // …but "full size" is the 1500-byte ceiling, and this session may have STARTED + // above it (a PW7a jumbo start whose path changed since the proof, or a client + // that roamed onto a 1500-MTU link). Then every video datagram is dying right now. + // The verdict is already erased above; heal the live wire if this session can be + // re-keyed at all — a PyroWave client cannot (its parse window is the `Welcome` + // value), so for those the WARN plus a corrected next session is all there is. + if sealed_datagram_bytes(current) > settled as usize { + tracing::warn!( + peer = %peer, + discovered_udp_mtu = settled, + shard_payload = current, + "wire MTU: this session started at a JUMBO shard but the path does not \ + carry it — video datagrams are oversized for a hop, which streams as a \ + black screen with zero reported loss. The jumbo verdict for this path is \ + cleared: the next connect starts at the standard 1500-byte wire." + ); + if let Some(r) = reneg.as_ref() { + let back = shard_payload_for_udp_budget(settled as usize, peer); + if back < current + && r.change_tx.send(back as u16).is_ok() + && r.apply_tx.send(back).is_ok() + { + tracing::info!(peer = %peer, shard_payload = back, was = current, + "wire MTU: video re-keyed mid-session back to the standard wire"); + current = back; + } + } + } } else { // A closed connection stops discovering, so a session that ended before the final // sample proves nothing (a healthy high-RTT path could still be mid-search): learn @@ -203,12 +490,41 @@ pub(super) fn spawn_watch( } } } + // PW7a revert guard for a session that STARTED jumbo and has no re-key channel (the + // PyroWave case, and the only reason the session-start grow exists). Nothing can save + // this session if the path stops fitting mid-stream — but the NEXT one must not repeat + // it, so keep sampling and drop the verdict the moment quinn's blackhole detection or + // a re-search says the path shrank. Cheap: one `Connection` handle, one sample per 5 s. + // Only for a session that is currently FITTING — one that already failed the check + // above has been warned about and had its verdict erased there. + if current > mtu1500_shard_payload_for(peer) + && reneg.is_none() + && sealed_datagram_bytes(current) <= settled as usize + { + loop { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + if conn.close_reason().is_some() { + return; + } + let mtu_now = conn.stats().path.current_mtu; + if (mtu_now as usize) < sealed_datagram_bytes(current) { + jumbo_verdicts().lock().unwrap().remove(&path_key(&conn)); + tracing::warn!(peer = %peer, discovered_udp_mtu = mtu_now, + shard_payload = current, + "wire MTU: the jumbo path this session started on stopped fitting — this \ + session cannot be re-keyed (chunk-aligned client parse window), so it \ + will not recover, but the verdict is cleared and the next connect \ + starts at the standard wire"); + return; + } + } + } // Phase 2 up-leg: jumbo grow — operator opt-in (PUNKTFUNK_JUMBO / PUNKTFUNK_WIRE_MTU // > 1500, which also raised the endpoint's probe ceiling so `settled` can even reach // here), client-advertised headroom, and a settled-at-jumbo proof. The grow is // ACK-GATED: not one sealed datagram above the old size leaves before the client's // ack, even though its buffers are statically sized — the rule must not erode. - let (Some(mtu), Some(r)) = (jumbo_wire_mtu(), reneg.as_mut()) else { + let (Some(mtu), Some(r)) = (target_wire_mtu, reneg.as_mut()) else { return; }; let target = jumbo_shard_payload_for(mtu, peer).min(r.client_ceiling as usize); @@ -275,34 +591,196 @@ mod tests { const V4: IpAddr = IpAddr::V4(Ipv4Addr::new(192, 168, 1, 2)); const V6: IpAddr = IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1)); + /// No jumbo anywhere — what every session that isn't on an opted-in jumbo LAN passes. + const NO_JUMBO: JumboStart = JumboStart { + target_wire_mtu: None, + client_ceiling: 0, + live_udp_mtu: 0, + proven_udp_budget: None, + clamped_udp_budget: None, + }; + /// A 9000-MTU LAN, a modern client, a path proven last session and re-proven live now. + fn proven_jumbo() -> JumboStart { + JumboStart { + target_wire_mtu: Some(9000), + client_ceiling: punktfunk_core::config::max_shard_payload() as u16, + live_udp_mtu: 8972, + proven_udp_budget: Some(8972), + clamped_udp_budget: None, + } + } #[test] fn default_when_nothing_known() { - assert_eq!(resolve(None, None, V4), mtu1500_shard_payload_for(V4)); - assert_eq!(resolve(None, None, V6), mtu1500_shard_payload_for(V6)); + assert_eq!( + resolve(None, None, NO_JUMBO, V4), + mtu1500_shard_payload_for(V4) + ); + assert_eq!( + resolve(None, None, NO_JUMBO, V6), + mtu1500_shard_payload_for(V6) + ); } #[test] fn env_override_beats_learned() { // 1280 wire − 28 IP/UDP − 64 header/crypto = 1188. - assert_eq!(resolve(Some(1280), Some(1472), V4), 1188); + assert_eq!(resolve(Some(1280), Some(1472), NO_JUMBO, V4), 1188); } #[test] fn learned_budget_clamps() { // A WARP-shaped path: 1280-byte UDP budget → 1280 − 64 = 1216. - assert_eq!(resolve(None, Some(1280), V4), 1216); + assert_eq!(resolve(None, Some(1280), NO_JUMBO, V4), 1216); } #[test] fn learned_at_or_above_ceiling_is_the_default_wire() { - assert_eq!(resolve(None, Some(1472), V4), mtu1500_shard_payload_for(V4)); - assert_eq!(resolve(None, Some(2000), V4), mtu1500_shard_payload_for(V4)); + assert_eq!( + resolve(None, Some(1472), NO_JUMBO, V4), + mtu1500_shard_payload_for(V4) + ); + assert_eq!( + resolve(None, Some(2000), NO_JUMBO, V4), + mtu1500_shard_payload_for(V4) + ); } #[test] fn env_full_mtu_is_the_default_wire_both_families() { - assert_eq!(resolve(Some(1500), None, V4), mtu1500_shard_payload_for(V4)); - assert_eq!(resolve(Some(1500), None, V6), mtu1500_shard_payload_for(V6)); + assert_eq!( + resolve(Some(1500), None, NO_JUMBO, V4), + mtu1500_shard_payload_for(V4) + ); + assert_eq!( + resolve(Some(1500), None, NO_JUMBO, V6), + mtu1500_shard_payload_for(V6) + ); + } + + /// The happy path, both families: 9000 − 28 (IPv4) − 64 = 8908, and 9000 − 48 − 64 = 8888. + #[test] + fn proven_and_reproven_path_starts_jumbo() { + assert_eq!(jumbo_session_start(proven_jumbo(), V4), Some(8908)); + let mut v6 = proven_jumbo(); + v6.live_udp_mtu = 8952; + v6.proven_udp_budget = Some(8952); + assert_eq!(jumbo_session_start(v6, V6), Some(8888)); + // …and it is what `resolve` returns, ahead of the env branch that would clamp a + // >1500 `PUNKTFUNK_WIRE_MTU` back down to the family default. + assert_eq!(resolve(Some(9000), None, proven_jumbo(), V4), 8908); + } + + /// THE guard: the laptop that proved jumbo on the wired LAN and came back on a 1500-MTU + /// link. The memory still says jumbo; the live connection says otherwise; the live one + /// wins, every time. This is what makes the grow as safe as the clamp. + #[test] + fn a_remembered_verdict_never_grows_without_a_live_reproof() { + let mut moved = proven_jumbo(); + moved.live_udp_mtu = 1472; // a clean 1500-MTU path, freshly measured + assert_eq!(jumbo_session_start(moved, V4), None); + assert_eq!( + resolve(None, None, moved, V4), + mtu1500_shard_payload_for(V4) + ); + // Not even one byte of headroom short of the sealed target is enough. + let mut nearly = proven_jumbo(); + nearly.live_udp_mtu = 8971; + assert_eq!(jumbo_session_start(nearly, V4), None); + } + + /// …and the mirror: a live-proven path with no prior verdict still starts at the default. + /// Both halves are required, so a single fluke on either side cannot seal a jumbo wire. + #[test] + fn a_live_proof_alone_does_not_grow() { + let mut first_ever = proven_jumbo(); + first_ever.proven_udp_budget = None; + assert_eq!(jumbo_session_start(first_ever, V4), None); + let mut weak_memory = proven_jumbo(); + weak_memory.proven_udp_budget = Some(1472); + assert_eq!(jumbo_session_start(weak_memory, V4), None); + } + + /// The two memories are keyed differently (clamp: peer; verdict: route), so they can + /// disagree. When they do, the one that keeps datagrams small wins. + #[test] + fn a_constrained_path_clamp_vetoes_the_grow() { + let mut contradicted = proven_jumbo(); + contradicted.clamped_udp_budget = Some(1280); + assert_eq!(jumbo_session_start(contradicted, V4), None); + // A clamp that is itself at or above the sealed target isn't contrary evidence. + let mut roomy = proven_jumbo(); + roomy.clamped_udp_budget = Some(8972); + assert_eq!(jumbo_session_start(roomy, V4), Some(8908)); + } + + #[test] + fn without_the_operator_opt_in_nothing_grows() { + let mut no_optin = proven_jumbo(); + no_optin.target_wire_mtu = None; + assert_eq!(jumbo_session_start(no_optin, V4), None); + } + + /// A legacy client (no `Hello::max_shard_payload`) is never handed a geometry it did not + /// advertise, and a client whose ceiling lands under the family default is left alone + /// rather than being "grown" to something smaller. + #[test] + fn the_client_ceiling_is_binding() { + let mut legacy = proven_jumbo(); + legacy.client_ceiling = 0; + assert_eq!(jumbo_session_start(legacy, V4), None); + let mut small = proven_jumbo(); + small.client_ceiling = 1408; + assert_eq!(jumbo_session_start(small, V4), None); + // A ceiling between the default and the path target caps the grow — and the proof + // then only has to cover the SMALLER sealed size. + let mut capped = proven_jumbo(); + capped.client_ceiling = 4000; + assert_eq!(jumbo_session_start(capped, V4), Some(4000)); + } + + /// Every shard payload the grow can produce is even (Leopard FEC splits shards in halves) + /// and fits the receive ceiling every client sizes its buffers from. + #[test] + fn grown_shards_stay_even_and_inside_the_receive_ceiling() { + for mtu in [2000usize, 4000, 4001, 9000, 9216, 64000] { + for peer in [V4, V6] { + let Some(t) = jumbo_target(Some(mtu), u16::MAX, peer) else { + continue; + }; + assert_eq!(t % 2, 0, "odd shard for mtu {mtu}"); + assert!(t <= punktfunk_core::config::max_shard_payload()); + assert!(t > mtu1500_shard_payload_for(peer)); + assert!( + sealed_datagram_bytes(t) <= punktfunk_core::packet::MAX_DATAGRAM_BYTES, + "sealed datagram overflows the receive ceiling at mtu {mtu}" + ); + } + } + // Below the family default there is nothing to grow to. + assert_eq!(jumbo_target(Some(1500), u16::MAX, V4), None); + assert_eq!(jumbo_target(None, u16::MAX, V4), None); + } + + /// A path is a (local interface, peer) pair, not a peer: the same client reached over the + /// host's other NIC is a different route with a different MTU. + #[test] + fn the_verdict_key_separates_routes_to_the_same_peer() { + let over_10g = PathKey { + local: Some(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))), + peer: V4, + }; + let over_wifi = PathKey { + local: Some(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1))), + peer: V4, + }; + assert_ne!(over_10g, over_wifi); + assert_ne!( + over_10g, + PathKey { + local: None, + peer: V4 + } + ); } } diff --git a/crates/punktfunk-host/src/session_plan.rs b/crates/punktfunk-host/src/session_plan.rs index 0709015d..9926bf42 100644 --- a/crates/punktfunk-host/src/session_plan.rs +++ b/crates/punktfunk-host/src/session_plan.rs @@ -193,12 +193,29 @@ impl SessionPlan { // Surface the trade loudly: this is the single biggest per-frame cost a 4:4:4 // session adds (full-res CPU readback + swscale RGB→YUV444P every frame), and // it looks like an unexplained fps ceiling if you don't know it happened. - tracing::warn!( - "4:4:4 session on the NVENC path without PUNKTFUNK_ZEROCOPY: zero-copy GPU \ - capture DISABLED — every frame is CPU RGB + swscale RGB→YUV444P; expect a \ - lower fps ceiling than 4:2:0 at this mode (set PUNKTFUNK_ZEROCOPY=1 for the \ - GPU 4:4:4 convert)" - ); + // + // Name the SESSION's codec, not the backend the gate is named after. The gate + // keys on `linux_zero_copy_is_vaapi()`, which reads the host-global encoder pref + // — so a per-session PyroWave negotiation on an NVENC/auto host lands here and + // was told it was "on the NVENC path", which is false in every particular: the + // wavelet encoder never touches NVENC, never swscales to YUV444P, and what it + // actually loses is the raw-dmabuf passthrough its whole design assumes. + if self.codec == crate::encode::Codec::PyroWave { + tracing::warn!( + "4:4:4 PyroWave session with PUNKTFUNK_ZEROCOPY off: zero-copy GPU \ + capture DISABLED — the wavelet encoder loses its raw-dmabuf passthrough \ + and every frame becomes a full-resolution CPU readback plus an upload \ + into its own Vulkan device; expect a materially lower fps ceiling (set \ + PUNKTFUNK_ZEROCOPY=1 to restore the passthrough)" + ); + } else { + tracing::warn!( + "4:4:4 session on the NVENC path without PUNKTFUNK_ZEROCOPY: zero-copy \ + GPU capture DISABLED — every frame is CPU RGB + swscale RGB→YUV444P; \ + expect a lower fps ceiling than 4:2:0 at this mode (set \ + PUNKTFUNK_ZEROCOPY=1 for the GPU 4:4:4 convert)" + ); + } } gpu && !force_cpu_for_nvenc_444 }; diff --git a/crates/punktfunk-host/src/spike.rs b/crates/punktfunk-host/src/spike.rs index 33b1169f..d1e922a0 100644 --- a/crates/punktfunk-host/src/spike.rs +++ b/crates/punktfunk-host/src/spike.rs @@ -48,6 +48,17 @@ pub struct Options { pub out: PathBuf, /// Also round-trip every AU through a `punktfunk_core` host→client loopback and verify. pub loopback: bool, + /// PyroWave datagram-aligned packetization at this shard payload + /// ([`Encoder::set_wire_chunking`], plan §4.4) — what a real session passes from its + /// negotiated `shard_payload`. `None` = the dense one-packet-per-AU shape. + /// + /// This is also the switch that makes the STREAMED-AU wire reachable from the spike: with + /// it set and `PUNKTFUNK_PYROWAVE_STREAMED_AU=1` armed, the encoder's `poll_chunk` hands the + /// AU out in window-aligned pieces and the loopback seals them through + /// `begin_streamed_frame_at`/`seal_streamed_chunk`/`seal_streamed_finish` — the same path a + /// `VIDEO_CAP_STREAMED_AU` client drives. Without it there is no way to exercise PW6 end to + /// end outside a real client session. + pub wire_chunk: Option, } pub fn run(opts: Options) -> Result<()> { @@ -114,9 +125,21 @@ pub fn run(opts: Options) -> Result<()> { refresh_hz: opts.fps, }) .context("create virtual output")?; + // `resolve` is the shared GameStream/spike constructor and hard-codes `pyrowave: false` + // (GameStream never negotiates it). The spike DOES know its codec, and on Linux that + // flag is what puts the capture on the raw-dmabuf passthrough + // (`ZeroCopyPolicy::pyrowave_session`, set from the same comparison in + // `session_plan::output_format`). Left false, `--codec pyrowave` encoded PyroWave off a + // capture negotiated for somebody else, and the only way to exercise the real path was + // the host-global `PUNKTFUNK_ENCODER=pyrowave` lever — which ALSO flips + // `backend_is_vaapi`, so it cannot reproduce a per-session PyroWave negotiation on an + // auto/NVENC host at all. That is precisely the configuration PW2 exists for. + let mut want = + capture::OutputFormat::resolve(false, crate::encode::resolved_backend_is_gpu()); + want.pyrowave = opts.codec == Codec::PyroWave; capture::capture_virtual_output( vout, - capture::OutputFormat::resolve(false, crate::encode::resolved_backend_is_gpu()), + want, crate::session_plan::CaptureBackend::resolve(), compositor == crate::vdisplay::Compositor::Kwin, ) @@ -155,6 +178,18 @@ pub fn run(opts: Options) -> Result<()> { ) .context("open encoder")?; + // Datagram-aligned packetization (§4.4) — and, with the PW6 knob armed, the gate that makes + // `supports_chunked_poll()` true so the drain below takes the streamed-AU path. + if let Some(c) = opts.wire_chunk { + encoder.set_wire_chunking(c); + tracing::info!( + shard_payload = c, + chunked_poll = encoder.supports_chunked_poll(), + "spike: wire chunking on (chunked_poll=false means PUNKTFUNK_PYROWAVE_STREAMED_AU \ + is not armed — the AU still goes out whole)" + ); + } + let mut sink = BufWriter::new( File::create(&opts.out).with_context(|| format!("create {}", opts.out.display()))?, ); @@ -194,6 +229,12 @@ pub fn run(opts: Options) -> Result<()> { out = %opts.out.display(), elapsed_s = format!("{elapsed:.2}"), encode_fps = format!("{:.1}", stats.encoded as f64 / elapsed.max(1e-9)), + // 0 = the whole-AU drain; > encoded = the streamed drain actually cut AUs into pieces. + chunks = stats.chunks, + chunks_per_au = format!( + "{:.1}", + stats.chunks as f64 / (stats.encoded.max(1)) as f64 + ), "spike capture→encode→file complete" ); @@ -217,6 +258,9 @@ struct Stats { encoded: u64, keyframes: u64, bytes_out: u64, + /// Streamed-AU drain only: total chunks polled across all AUs (1 per AU means the cut never + /// engaged — the knob is off or the AU fits one chunk). + chunks: u64, } fn drain_encoder( @@ -225,6 +269,12 @@ fn drain_encoder( mut lb: Option<&mut Loopback>, stats: &mut Stats, ) -> Result<()> { + // Streamed-AU drain (PW6): the encoder hands the finished AU out in shard-aligned pieces and + // the loopback seals each piece as it arrives, exactly as the native host's send thread does. + // Re-queried per drain, never cached — the trait's contract. + if encoder.supports_chunked_poll() { + return drain_encoder_chunked(encoder, sink, lb, stats); + } while let Some(au) = encoder.poll().context("encoder poll")? { sink.write_all(&au.data).context("write AU to file")?; stats.encoded += 1; @@ -239,6 +289,49 @@ fn drain_encoder( Ok(()) } +/// The streamed-AU drain. Each chunk is sealed into the open wire frame the moment it is polled; +/// the concatenation is kept only so the completed AU can still be written to the file sink and +/// byte-compared against what the client reassembled — which is the point of the leg: it proves +/// the chunks the encoder cut, sealed through the sentinel-block wire, reassemble to EXACTLY the +/// AU `poll()` would have produced. +fn drain_encoder_chunked( + encoder: &mut dyn Encoder, + sink: &mut impl Write, + mut lb: Option<&mut Loopback>, + stats: &mut Stats, +) -> Result<()> { + let mut whole: Vec = Vec::new(); + let mut chunks = 0u32; + while let Some(c) = encoder.poll_chunk().context("encoder poll_chunk")? { + if c.first { + whole.clear(); + chunks = 0; + if let Some(lb) = lb.as_deref_mut() { + lb.streamed_begin(c.pts_ns, c.keyframe)?; + } + } + whole.extend_from_slice(&c.data); + chunks += 1; + if let Some(lb) = lb.as_deref_mut() { + lb.streamed_chunk(&c.data)?; + } + if !c.last { + continue; + } + sink.write_all(&whole).context("write AU to file")?; + stats.encoded += 1; + stats.bytes_out += whole.len() as u64; + stats.chunks += chunks as u64; + if c.keyframe { + stats.keyframes += 1; + } + if let Some(lb) = lb.as_deref_mut() { + lb.streamed_finish(&whole)?; + } + } + Ok(()) +} + /// A host↔client `punktfunk_core` pair over a lossless in-process loopback. Each encoded AU is /// FEC-protected, packetized, sent, then reassembled on the client and byte-compared to the /// original — exercising the core on real encoder output (the spike "feed into a Session" goal). @@ -249,6 +342,14 @@ struct Loopback { recovered: u64, mismatches: u64, bytes: u64, + /// The streamed AU currently open (PW6). `Some` strictly between `streamed_begin` and + /// `streamed_finish`, mirroring the native send thread's `StreamedOpen`. + open: Option, + /// Wire frame index for the streamed path. `submit_frame` uses the packetizer's internal + /// counter and `begin_streamed_frame_at` takes an explicit one; a session must use ONE + /// numbering style, and the spike never mixes them (`supports_chunked_poll()` is constant + /// for a PyroWave session, so every AU takes the same route). + next_index: u32, } impl Loopback { @@ -265,9 +366,101 @@ impl Loopback { recovered: 0, mismatches: 0, bytes: 0, + open: None, + next_index: 0, }) } + /// Open a streamed AU on the wire (PW6). The client side needs no opt-in: a streamed frame + /// completes exactly like a whole one and is handed up as a single `Frame` — which is the + /// finding this leg exists to demonstrate rather than assert. + fn streamed_begin(&mut self, pts_ns: u64, keyframe: bool) -> Result<()> { + if self.open.is_some() { + return Err(anyhow!( + "streamed AU still open at begin — a previous AU never sent its `last` chunk" + )); + } + let mut flags = FLAG_PIC as u32; + if keyframe { + flags |= FLAG_SOF as u32; + } + let idx = self.next_index; + self.next_index = self.next_index.wrapping_add(1); + self.open = Some( + self.host + .begin_streamed_frame_at(pts_ns, flags, idx) + .map_err(|e| anyhow!("begin_streamed_frame_at: {e:?}"))?, + ); + Ok(()) + } + + /// Seal + send one encoder chunk. The returned batch is often EMPTY (the sealer buffers + /// until a whole FEC block accumulates) — that is the normal case, not an error. + fn streamed_chunk(&mut self, data: &[u8]) -> Result<()> { + let au = self + .open + .as_mut() + .ok_or_else(|| anyhow!("streamed chunk with no open AU"))?; + let wires = self + .host + .seal_streamed_chunk(au, data, false) + .map_err(|e| anyhow!("seal_streamed_chunk: {e:?}"))?; + self.send(wires) + } + + /// Close the AU (final block carries the real totals) and verify what the client got. + fn streamed_finish(&mut self, expect: &[u8]) -> Result<()> { + let au = self + .open + .take() + .ok_or_else(|| anyhow!("streamed finish with no open AU"))?; + let wires = self + .host + .seal_streamed_finish(au) + .map_err(|e| anyhow!("seal_streamed_finish: {e:?}"))?; + self.send(wires)?; + self.submitted += 1; + self.bytes += expect.len() as u64; + self.verify(expect) + } + + fn send(&mut self, wires: Vec>) -> Result<()> { + if wires.is_empty() { + return Ok(()); + } + let refs: Vec<&[u8]> = wires.iter().map(|w| w.as_slice()).collect(); + self.host + .send_sealed(&refs) + .map_err(|e| anyhow!("send_sealed: {e:?}"))?; + drop(refs); + self.host.reclaim_wires(wires); + Ok(()) + } + + /// Drain whatever the client can now reassemble and byte-compare it to `expect`. + fn verify(&mut self, expect: &[u8]) -> Result<()> { + loop { + match self.client.poll_frame() { + Ok(frame) => { + self.recovered += 1; + if frame.data != expect { + self.mismatches += 1; + tracing::warn!( + recovered = self.recovered, + got = frame.data.len(), + expected = expect.len(), + complete = frame.complete, + "loopback AU mismatch" + ); + } + } + Err(punktfunk_core::PunktfunkError::NoFrame) => break, + Err(e) => return Err(anyhow!("client poll_frame: {e:?}")), + } + } + Ok(()) + } + fn submit(&mut self, au: &EncodedFrame) -> Result<()> { let mut flags = FLAG_PIC as u32; if au.keyframe { diff --git a/crates/pyrowave-sys/patches/0005-global-priority-queue.patch b/crates/pyrowave-sys/patches/0005-global-priority-queue.patch index 81e95632..5609243c 100644 --- a/crates/pyrowave-sys/patches/0005-global-priority-queue.patch +++ b/crates/pyrowave-sys/patches/0005-global-priority-queue.patch @@ -17,7 +17,38 @@ VK_ERROR_NOT_PERMITTED_KHR so a refused class NEVER regresses the encoder. Gated NOTE: on an RTX 4090 / Windows / WDDM this did not reduce the spikes (the graphics-vs-compute preemption granularity is the wall) — kept because it is correct, harmless (graceful fallback), and may help other GPUs/drivers. Reduce the encode's GPU cost (4:2:0/8-bit) or use H.265 for a -GPU-saturated game. +GPU-saturated game. **That measurement is Windows/WDDM and does NOT transfer to Linux** — a +different driver stack with a different preemption model. + +MEASURED ON LINUX/NVIDIA 2026-08-08, and it comes out the OTHER WAY: the elevated queue DOES cut +the tail. RTX 5070 Ti (driver 610.57.04), GRID 2 benchmark loop saturating the GPU at 54-87 %, +PyroWave 1080p, same binary in both arms (the only difference is CAP_SYS_NICE, i.e. whether the +class is granted at all), steady-state windows of 30 frames: + + arm p50 p99 worst frame + default priority (refused) ~2.6 ms ~6.4 ms 9.5 ms + REALTIME granted ~3.2 ms ~4.4 ms 5.4 ms (repeat: p50 ~3.35, p99 ~4.8) + +So on this stack the priority class buys a materially tighter TAIL — p99 down ~30 %, worst frame +roughly halved — at the cost of ~0.6 ms on the median. For a streaming encoder that is the right +side of the trade: the tail is what shows up as a visible hitch. Do NOT delete this patch on the +strength of the RTX 4090/WDDM result above; the two stacks disagree. + +Caveats, so the number is not over-read: the arms were not interleaved and the background game +load drifted between them, capture was frame-starved (~2.5 fps) so this measures encode latency +under contention rather than a full-rate stream, and it is two granted runs against one refused +run. The direction was consistent across all 25 measurement windows. + +NOTE 2 — WHERE THIS PATCH IS ACTUALLY LIVE. It is gated `if (!inherit_info)`, and only the WINDOWS +path leaves `inherit_info` null: `crates/pf-encode/src/enc/windows/pyrowave.rs` calls +`pyrowave_create_device_by_compat`, so Granite builds the device itself and this block runs. +**On LINUX it has never done anything.** `crates/pf-encode/src/enc/linux/pyrowave.rs::open_inner` +passes its own instance/device create-infos into `pyrowave_device_create_info`, Granite's +`MyDeviceFactory::get_existing_create_info()` returns them, `create_device` takes the inherit +branch, and the whole block above is skipped. The Linux request is therefore wired natively in +**`crates/pf-encode/src/enc/linux/pyrowave.rs`** (search `queue_priority_candidates`), which +implements the SAME env grammar and the SAME downgrade ladder so one knob means one thing on both +platforms. If you change the grammar here, change it there in the same commit. diff --git a/crates/pyrowave-sys/vendor/pyrowave/Granite/vulkan/context.cpp b/crates/pyrowave-sys/vendor/pyrowave/Granite/vulkan/context.cpp index 5257fc33..479eeded 100644 diff --git a/crates/pyrowave-sys/patches/0007-encoder-sequence-override.patch b/crates/pyrowave-sys/patches/0007-encoder-sequence-override.patch new file mode 100644 index 00000000..d69514b6 --- /dev/null +++ b/crates/pyrowave-sys/patches/0007-encoder-sequence-override.patch @@ -0,0 +1,123 @@ +Encoder wire-sequence override — PUNKTFUNK LOCAL PATCH. + +Not upstream. Exposes `Encoder::set_next_sequence(uint32_t)` (and a +`pyrowave_encoder_set_next_sequence` C entry) so the caller can stamp the 3-bit wire sequence +counter itself instead of relying on the encoder object's private one. + +WHY IT EXISTS. PyroWave's `Encoder` structurally cannot hold two frames in flight: `Encoder::Impl` +owns ONE each of `wavelet_img_high_res`, `bucket_buffer`, `meta_buffer`, `block_stat_buffer`, +`payload_data` and `quant_buffer`, and `Impl::encode` OPENS by discarding them — an image barrier +with `VK_IMAGE_LAYOUT_UNDEFINED` as the old layout (a written promise nothing else is reading it) +plus three `fill_buffer` clears. Two `encode()` calls recorded into two command buffers and +submitted to the same queue have no execution dependency in Vulkan, so encode N+1's DWT would +overwrite the bands and zero the RDO buckets while encode N's block packing still reads them. + +So overlapping frames means TWO encoder handles on one device, alternated — which is fine for +every resource above, because each handle gets its own. It is NOT fine for `sequence_count`, which +also lives on `Impl` and is stamped into every block header (pyrowave_encoder.cpp `packing_push`). +Two alternating handles each count 1,2,3... independently, so the wire sees 1,1,2,2,3,3... + +That is silently fatal on the decode side. `pyrowave_decoder.cpp` computes +`diff = (hdr.sequence - last_seq) & 0x7` and treats `restart = diff != 0`, so a REPEATED value +reads as "more blocks of the same frame": `clear()` never runs, `decoded_frame_for_current_sequence` +stays true, and every second frame is swallowed. The symptom is "it works, just at half rate, with +occasional mixed-frame blocks" — the kind of failure that passes a smoke test. It would hit every +client, since pf-client-core and the Apple Metal hand-port parse the same field. + +WHAT IT DOES. `set_next_sequence(seq)` stores `(seq - 1) & SequenceCountMask`, because +`Impl::encode` pre-increments before stamping — the setter's contract is about the next ENCODE, not +the next store. The Rust side keeps one monotonic counter across both handles and calls this before +each encode, so the wire sequence increments by exactly 1 mod 8 regardless of which handle produced +the frame. + +INERT WHEN UNUSED. Nothing calls it unless the caller does, so the single-handle paths — including +the whole Windows backend — behave exactly as before. No `.def` change is needed: the C API is +built as a static archive (crates/pyrowave-sys/CMakeLists.txt). + +Upstream status: not reported. It is a hook for a use case upstream explicitly designed against +("For low-latency use cases, overlapping frames in encode is meaningless due to latency and the +encoder is so fast anyway" — pyrowave.h). That reasoning holds at 1080p60 and stops holding at 4K +or under a GPU-bound game, which is what PW5 measured. + +diff --git a/crates/pyrowave-sys/vendor/pyrowave/pyrowave.h b/crates/pyrowave-sys/vendor/pyrowave/pyrowave.h +index fc0d5834..aeb22ffc 100644 +--- a/crates/pyrowave-sys/vendor/pyrowave/pyrowave.h ++++ b/crates/pyrowave-sys/vendor/pyrowave/pyrowave.h +@@ -476,6 +476,19 @@ PYROWAVE_PUBLIC_API pyrowave_result + pyrowave_encoder_packetize(pyrowave_encoder encoder, pyrowave_packet *packets, size_t packet_boundary, + size_t *out_packets, void *bitstream, size_t size); + ++// PUNKTFUNK LOCAL EXTENSION (patches/0007-encoder-sequence-override.patch), not upstream. ++// The wire sequence counter is 3 bits (PyroWave::SequenceCountMask, pyrowave_common.hpp); ++// exported here so callers mask with the codec's own value instead of a copied literal. ++#define PYROWAVE_SEQUENCE_MASK 0x7u ++ ++// Overrides the 3-bit wire sequence counter the NEXT encode will stamp into every block header. ++// The counter lives on the encoder object, so a caller that alternates TWO encoders to overlap ++// frames emits 1,1,2,2,3,3... and the decoder — which restarts a frame only when the value ++// CHANGES — reads the repeat as more blocks of the same frame and silently swallows every second ++// frame. Stamp a single monotonic counter across the handles with this. Value is masked to 3 bits. ++PYROWAVE_PUBLIC_API pyrowave_result ++pyrowave_encoder_set_next_sequence(pyrowave_encoder encoder, uint32_t sequence); ++ + // Implementation ensures GPU is idle before destroying objects. + PYROWAVE_PUBLIC_API void + pyrowave_encoder_destroy(pyrowave_encoder encoder); +diff --git a/crates/pyrowave-sys/vendor/pyrowave/pyrowave_c.cpp b/crates/pyrowave-sys/vendor/pyrowave/pyrowave_c.cpp +index 985cd0a9..fcd7d6f8 100644 +--- a/crates/pyrowave-sys/vendor/pyrowave/pyrowave_c.cpp ++++ b/crates/pyrowave-sys/vendor/pyrowave/pyrowave_c.cpp +@@ -1196,6 +1196,17 @@ pyrowave_encoder_packetize(pyrowave_encoder encoder, pyrowave_packet *packets, s + return PYROWAVE_SUCCESS; + } + ++// PUNKTFUNK LOCAL EXTENSION (patches/0007-encoder-sequence-override.patch), not upstream. ++pyrowave_result ++pyrowave_encoder_set_next_sequence(pyrowave_encoder encoder, uint32_t sequence) ++{ ++ Util::set_thread_logging_interface(&null_logger); ++ if (!encoder) ++ return PYROWAVE_ERROR_GENERIC; ++ encoder->encoder.set_next_sequence(sequence); ++ return PYROWAVE_SUCCESS; ++} ++ + void pyrowave_encoder_destroy(pyrowave_encoder encoder) + { + auto *device = encoder->device; +diff --git a/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.cpp b/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.cpp +index ad4e9746..f23717f3 100644 +--- a/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.cpp ++++ b/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.cpp +@@ -1230,6 +1230,14 @@ bool Encoder::encode(CommandBuffer &cmd, const ViewBuffers &views, const Bitstre + return impl->encode(cmd, views, buffers); + } + ++// PUNKTFUNK: see the declaration in pyrowave_encoder.hpp. Impl::encode PRE-increments ++// (sequence_count = (sequence_count + 1) & mask before stamping), so store one less than the value ++// the caller wants stamped — the setter's contract is about the next ENCODE, not the next store. ++void Encoder::set_next_sequence(uint32_t sequence) ++{ ++ impl->sequence_count = (sequence - 1) & SequenceCountMask; ++} ++ + const Vulkan::ImageView &Encoder::get_wavelet_band(int component, int level) + { + return *impl->component_layer_views[component][level]; +diff --git a/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.hpp b/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.hpp +index a65447d5..8c0ef0d0 100644 +--- a/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.hpp ++++ b/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.hpp +@@ -37,6 +37,12 @@ public: + bool init(Vulkan::Device *device, int width, int height, ChromaSubsampling chroma); + bool encode(Vulkan::CommandBuffer &cmd, const ViewBuffers &views, const BitstreamBuffers &buffers); + ++ // PUNKTFUNK: override the 3-bit wire sequence counter the NEXT encode will stamp. ++ // The counter is per-Encoder, so alternating two encoder objects to overlap frames emits ++ // 1,1,2,2,3,3... and the decoder reads a repeated value as "more blocks of the same frame". ++ // See crates/pyrowave-sys/patches/0007-encoder-sequence-override.patch. ++ void set_next_sequence(uint32_t sequence); ++ + // Debug hackery + const Vulkan::ImageView &get_wavelet_band(int component, int level); + bool encode_pre_transformed(Vulkan::CommandBuffer &cmd, const BitstreamBuffers &buffers, float quant_scale); diff --git a/crates/pyrowave-sys/vendor/pyrowave/PUNKTFUNK-VENDOR.txt b/crates/pyrowave-sys/vendor/pyrowave/PUNKTFUNK-VENDOR.txt index d145f8e0..3bfe2b24 100644 --- a/crates/pyrowave-sys/vendor/pyrowave/PUNKTFUNK-VENDOR.txt +++ b/crates/pyrowave-sys/vendor/pyrowave/PUNKTFUNK-VENDOR.txt @@ -46,4 +46,9 @@ upstream: realtime) so the wavelet encode can preempt a GPU-bound game on the shared shader cores. A create loop downgrades on NOT_PERMITTED so a refused class never regresses the encoder. Did not overcome the graphics-vs-compute preemption wall on an RTX 4090 (kept: correct + harmless, - may help other HW/drivers). + may help other HW/drivers) — that measurement is Windows/WDDM and does not transfer to Linux. + GATED ON !inherit_info, so it is LIVE ONLY ON THE WINDOWS PATH (pyrowave_create_device_by_compat, + where Granite builds its own device). Linux passes its own create-infos and takes the inherit + branch, so this patch is inert there; the Linux request lives natively in + crates/pf-encode/src/enc/linux/pyrowave.rs (queue_priority_candidates), with the same grammar + and the same downgrade ladder. Change one, change both. diff --git a/crates/pyrowave-sys/vendor/pyrowave/pyrowave.h b/crates/pyrowave-sys/vendor/pyrowave/pyrowave.h index fc0d5834..aeb22ffc 100644 --- a/crates/pyrowave-sys/vendor/pyrowave/pyrowave.h +++ b/crates/pyrowave-sys/vendor/pyrowave/pyrowave.h @@ -476,6 +476,19 @@ PYROWAVE_PUBLIC_API pyrowave_result pyrowave_encoder_packetize(pyrowave_encoder encoder, pyrowave_packet *packets, size_t packet_boundary, size_t *out_packets, void *bitstream, size_t size); +// PUNKTFUNK LOCAL EXTENSION (patches/0007-encoder-sequence-override.patch), not upstream. +// The wire sequence counter is 3 bits (PyroWave::SequenceCountMask, pyrowave_common.hpp); +// exported here so callers mask with the codec's own value instead of a copied literal. +#define PYROWAVE_SEQUENCE_MASK 0x7u + +// Overrides the 3-bit wire sequence counter the NEXT encode will stamp into every block header. +// The counter lives on the encoder object, so a caller that alternates TWO encoders to overlap +// frames emits 1,1,2,2,3,3... and the decoder — which restarts a frame only when the value +// CHANGES — reads the repeat as more blocks of the same frame and silently swallows every second +// frame. Stamp a single monotonic counter across the handles with this. Value is masked to 3 bits. +PYROWAVE_PUBLIC_API pyrowave_result +pyrowave_encoder_set_next_sequence(pyrowave_encoder encoder, uint32_t sequence); + // Implementation ensures GPU is idle before destroying objects. PYROWAVE_PUBLIC_API void pyrowave_encoder_destroy(pyrowave_encoder encoder); diff --git a/crates/pyrowave-sys/vendor/pyrowave/pyrowave_c.cpp b/crates/pyrowave-sys/vendor/pyrowave/pyrowave_c.cpp index 985cd0a9..fcd7d6f8 100644 --- a/crates/pyrowave-sys/vendor/pyrowave/pyrowave_c.cpp +++ b/crates/pyrowave-sys/vendor/pyrowave/pyrowave_c.cpp @@ -1196,6 +1196,17 @@ pyrowave_encoder_packetize(pyrowave_encoder encoder, pyrowave_packet *packets, s return PYROWAVE_SUCCESS; } +// PUNKTFUNK LOCAL EXTENSION (patches/0007-encoder-sequence-override.patch), not upstream. +pyrowave_result +pyrowave_encoder_set_next_sequence(pyrowave_encoder encoder, uint32_t sequence) +{ + Util::set_thread_logging_interface(&null_logger); + if (!encoder) + return PYROWAVE_ERROR_GENERIC; + encoder->encoder.set_next_sequence(sequence); + return PYROWAVE_SUCCESS; +} + void pyrowave_encoder_destroy(pyrowave_encoder encoder) { auto *device = encoder->device; diff --git a/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.cpp b/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.cpp index ad4e9746..f23717f3 100644 --- a/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.cpp +++ b/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.cpp @@ -1230,6 +1230,14 @@ bool Encoder::encode(CommandBuffer &cmd, const ViewBuffers &views, const Bitstre return impl->encode(cmd, views, buffers); } +// PUNKTFUNK: see the declaration in pyrowave_encoder.hpp. Impl::encode PRE-increments +// (sequence_count = (sequence_count + 1) & mask before stamping), so store one less than the value +// the caller wants stamped — the setter's contract is about the next ENCODE, not the next store. +void Encoder::set_next_sequence(uint32_t sequence) +{ + impl->sequence_count = (sequence - 1) & SequenceCountMask; +} + const Vulkan::ImageView &Encoder::get_wavelet_band(int component, int level) { return *impl->component_layer_views[component][level]; diff --git a/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.hpp b/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.hpp index a65447d5..8c0ef0d0 100644 --- a/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.hpp +++ b/crates/pyrowave-sys/vendor/pyrowave/pyrowave_encoder.hpp @@ -37,6 +37,12 @@ public: bool init(Vulkan::Device *device, int width, int height, ChromaSubsampling chroma); bool encode(Vulkan::CommandBuffer &cmd, const ViewBuffers &views, const BitstreamBuffers &buffers); + // PUNKTFUNK: override the 3-bit wire sequence counter the NEXT encode will stamp. + // The counter is per-Encoder, so alternating two encoder objects to overlap frames emits + // 1,1,2,2,3,3... and the decoder reads a repeated value as "more blocks of the same frame". + // See crates/pyrowave-sys/patches/0007-encoder-sequence-override.patch. + void set_next_sequence(uint32_t sequence); + // Debug hackery const Vulkan::ImageView &get_wavelet_band(int component, int level); bool encode_pre_transformed(Vulkan::CommandBuffer &cmd, const BitstreamBuffers &buffers, float quant_scale); diff --git a/docs-site/content/docs/configuration.md b/docs-site/content/docs/configuration.md index 7b305475..e401b8cc 100644 --- a/docs-site/content/docs/configuration.md +++ b/docs-site/content/docs/configuration.md @@ -241,6 +241,7 @@ notes for context. | `PUNKTFUNK_NVENC_SPLIT_ARBITRATE` | `1` | Opt-in: let the host change its split-encode decision **live**, mid-session, as the pixel rate moves, instead of only choosing once at session start. Currently wired on the Linux direct-NVENC path. Only interesting alongside `PUNKTFUNK_SPLIT_ENCODE=auto` at very high pixel rates. | | `PUNKTFUNK_GPU_PRIORITY_CLASS` | `off` · `normal` · `high` · `realtime` · `auto` | **(Windows)** GPU scheduling priority for capture/encode under a GPU-saturating game. Default `auto` (starts `high`, upgrades to `realtime` when it's safe — e.g. HAGS off); `high` pins the static pre-gate behaviour; `realtime` is the strongest lever but can freeze NVENC on some setups. | | `PUNKTFUNK_IDD_DEPTH` | `N` (default `2`) | **(Windows)** IDD-push pipeline depth. `1` cuts latency once GPU priority is raised; higher smooths a contended GPU. | +| `PYROWAVE_QUEUE_PRIORITY` | `realtime` *(default)* · `high` · `off` | [PyroWave](/docs/pyrowave) sessions only. PyroWave encodes on the same GPU shader cores a game uses, so a demanding game can starve it and the frame rate drops. This asks the driver to schedule the encode ahead of the game. `realtime` tries the strongest class and falls back to `high`; `high` asks only for the middle one; `off` disables the request. A driver that refuses simply encodes at normal priority — it can never stop a session starting. The packages grant the host the `CAP_SYS_NICE` capability this needs; on a host built or installed by hand it will be refused, and the host says so once at session start. Set `off` if you see the desktop stutter while streaming. | ## Diagnostics diff --git a/docs-site/content/docs/pyrowave.mdx b/docs-site/content/docs/pyrowave.mdx index ee37c63c..8fcbb745 100644 --- a/docs-site/content/docs/pyrowave.mdx +++ b/docs-site/content/docs/pyrowave.mdx @@ -101,6 +101,27 @@ dropped packets. The stats overlay shows `pyrowave` as the decode path when the mode is active. +## Checking the host is really zero-copy + +On a Linux host the CPU fallback mentioned above is not an error — the session still streams, it +just pays a full-resolution copy of every frame, which shows up as a lower frame-rate ceiling and +higher CPU use rather than as anything obviously broken. The host log states which path a session +took, once, when the capture starts: + +``` +capture pipeline resolved: dmabuf-passthrough → pyrowave +``` + +`dmabuf-passthrough` is the good one: the compositor's buffer goes straight into the wavelet +encoder. `cpu` means the copy is happening, and a second line says why — a compositor that would +not allocate a dmabuf, `PUNKTFUNK_ZEROCOPY` set to `0`, or a per-frame fall-through such as the +compositor serving shared memory after agreeing to dmabufs. Each distinct reason is logged once per +session with a running count, so a persistent downgrade is easy to tell from a hiccup while the +display mode settles. + +If you see `cpu` and did not ask for it, check that `PUNKTFUNK_ZEROCOPY` is unset (it defaults to +on) and read the accompanying line — it names the cause and the fix. + ## Current limits - Linux and Windows hosts; Linux clients (the GTK desktop app and the session client, including diff --git a/docs-site/content/docs/running-as-a-service.md b/docs-site/content/docs/running-as-a-service.md index 750b6857..f510e8fd 100644 --- a/docs-site/content/docs/running-as-a-service.md +++ b/docs-site/content/docs/running-as-a-service.md @@ -205,6 +205,36 @@ the host. If the host answers, it's up. If not, check `journalctl --user -u punktfunk-host` on the host — on a Windows host, run `punktfunk-host service status` from an elevated prompt on the machine itself. +## GPU scheduling priority + +The Linux packages give the host binary one Linux capability, `CAP_SYS_NICE`, and it is worth +knowing why it is there and how to take it away. + +The [PyroWave](/docs/pyrowave) codec encodes on the same GPU shader cores your game is using, so a +demanding game can crowd it out and the stream's frame rate drops with it. The fix is to ask the +driver to schedule the encode ahead of the game, and every driver we tested gates that request on +this capability: without it the request is simply refused and nothing changes. The other codecs use +a separate video engine on the GPU and are unaffected either way. + +`CAP_SYS_NICE` lets a process raise its own scheduling priority. It grants no access to files, +the network or other users' processes, and it is **not** the same as running as root — the host +still runs as you, under your user session. + +To check, or to take it away: + +```sh +getcap /usr/bin/punktfunk-host # shows cap_sys_nice=ep when granted +sudo setcap -r /usr/bin/punktfunk-host # remove it; streaming still works +``` + +Removing it costs you nothing unless you stream PyroWave, and you can also just set +`PYROWAVE_QUEUE_PRIORITY=off` to stop the host asking. Note that a package update replaces the +binary and re-applies the capability. + +Two side effects, if you are debugging the host: a binary carrying a capability is treated as +security-sensitive by the dynamic loader, so `LD_LIBRARY_PATH` and `LD_PRELOAD` are ignored for it, +and it does not write core dumps by default. + ## Stopping and removing After a Linux package update the user service keeps running the old binary until it's restarted, and diff --git a/packaging/arch/punktfunk-host.install b/packaging/arch/punktfunk-host.install index 894da933..2774590e 100644 --- a/packaging/arch/punktfunk-host.install +++ b/packaging/arch/punktfunk-host.install @@ -12,9 +12,33 @@ _ensure_punktfunk_group() { getent group punktfunk >/dev/null 2>&1 || groupadd --system punktfunk 2>/dev/null || true } +# CAP_SYS_NICE on the host binary — the GPU-scheduling grant. +# +# WHY: PyroWave encodes on the GPU's shader cores, so a GPU-bound game starves it (measured: the +# encode dispatch goes from ~2 ms to 15-18 ms at 95 % game load). The fix is an elevated +# global-priority Vulkan queue, which the driver gates on CAP_SYS_NICE — measured 2026-08-08 on an +# RTX 5070 Ti: WITHOUT the capability every priority class is refused, WITH it the encoder is +# granted REALTIME on the first attempt. RADV is the same. Without this line the knob exists and +# does nothing. Same capability, same mechanism, as our gamescope package sets on its own binary. +# +# NARROW: CAP_SYS_NICE only permits raising scheduling priority (nice/ioprio/affinity/RT class). It +# grants no filesystem, network or user-switching privilege, and it is NOT setuid. +# +# TWO CONSEQUENCES worth knowing before you debug something odd on this host: +# * a file capability makes the process AT_SECURE, so the dynamic loader IGNORES LD_LIBRARY_PATH +# and LD_PRELOAD for it. A library-path shim that used to work will silently stop. +# * core dumps are suppressed for capability-carrying binaries by default (fs.suid_dumpable). +# +# Never fails the install: a box without libcap, or a filesystem that cannot store capabilities +# (some overlay/NFS setups), just runs at default priority exactly as before. +_grant_sched_capability() { + setcap 'cap_sys_nice=ep' usr/bin/punktfunk-host 2>/dev/null || true +} + post_install() { _ensure_update_group _ensure_punktfunk_group + _grant_sched_capability udevadm control --reload-rules 2>/dev/null || true udevadm trigger --subsystem-match=misc 2>/dev/null || true # Apply the UDP socket-buffer tuning now (also auto-applied at boot by systemd-sysctl). @@ -73,6 +97,8 @@ post_upgrade() { # root-only, and the virtual Steam Deck pad silently unable to attach. groupadd is idempotent, so # this is a no-op on boxes that installed fresh. _ensure_punktfunk_group + # A replaced binary is a NEW inode — file capabilities do not survive the upgrade, so re-grant. + _grant_sched_capability udevadm control --reload-rules 2>/dev/null || true sysctl -p /usr/lib/sysctl.d/99-punktfunk-net.conf >/dev/null 2>&1 || true _warn_stale_firewall_ports diff --git a/packaging/bazzite/build-sysext.sh b/packaging/bazzite/build-sysext.sh index c88acd62..32c2861b 100644 --- a/packaging/bazzite/build-sysext.sh +++ b/packaging/bazzite/build-sysext.sh @@ -130,6 +130,31 @@ SYSEXT_VERSION_ID=$PF_VR EXTENSION_RELOAD_MANAGER=1 EOF +# CAP_SYS_NICE on the host binary — the GPU-scheduling grant. PyroWave encodes on the GPU shader +# cores a game saturates, and the driver gates the elevated global-priority Vulkan queue that fixes +# it on this capability (measured 2026-08-08 on an RTX 5070 Ti: refused without it, granted REALTIME +# with it; RADV the same). Narrow — scheduling priority only, no filesystem/network privilege, not +# setuid. +# +# It has to be applied HERE, not in the merge hook: a merged sysext's /usr is a read-only squashfs, +# so nothing can setcap it afterwards. And it cannot ride in from the RPM either — the spec declares +# it with %caps, but rpm stores capabilities in its own header and `rpm2cpio | cpio` carries only +# the payload, so the staged file arrives with no capability at all. mksquashfs DOES record +# security.capability (only security.selinux is excluded below), so a setcap on the staging tree is +# what ends up in the image. +# +# Needs CAP_SETFCAP, i.e. root (or fakeroot) — a plain-user CI build cannot do it. That is not fatal: +# the image just ships as it does today and the encode runs at default GPU priority, so warn and +# carry on rather than fail a release build over a performance lever. +if [ -f "$STAGE/usr/bin/punktfunk-host" ]; then + if setcap 'cap_sys_nice=ep' "$STAGE/usr/bin/punktfunk-host" 2>/dev/null; then + echo "granted CAP_SYS_NICE to usr/bin/punktfunk-host (GPU-priority lever active)" + else + echo "WARNING: could not setcap CAP_SYS_NICE (need root/CAP_SETFCAP) — the image will ship" >&2 + echo " without it and PyroWave will encode at default GPU priority." >&2 + fi +fi + # SELinux labels as pseudo-xattrs (see header). matchpathcon resolves each target path against # the targeted policy's file_contexts; <> means "no specific entry" — skip those (the # handful of matches all resolve to real contexts for our payload). diff --git a/packaging/debian/build-deb.sh b/packaging/debian/build-deb.sh index 8ebdaac5..0485d930 100755 --- a/packaging/debian/build-deb.sh +++ b/packaging/debian/build-deb.sh @@ -294,6 +294,16 @@ if [ "$1" = "configure" ]; then # primitive that must not ride on the group users are told to join for gamepads # (security-review 2026-08-05 M-4). getent group punktfunk >/dev/null 2>&1 || addgroup --system punktfunk 2>/dev/null || true + # CAP_SYS_NICE — the GPU-scheduling grant. PyroWave encodes on the shader cores a game + # saturates, and the driver gates the elevated global-priority Vulkan queue that fixes it on + # this capability: measured 2026-08-08 on an RTX 5070 Ti, WITHOUT it every priority class is + # refused and WITH it the encoder is granted REALTIME first try (RADV behaves the same). + # Without this line the knob exists and does nothing. Narrow: it permits raising scheduling + # priority only — no filesystem, network or user-switching privilege, and no setuid. Note a + # capability-carrying binary is AT_SECURE, so the loader ignores LD_LIBRARY_PATH/LD_PRELOAD for + # it and core dumps are suppressed by default. Best-effort: a box without libcap, or a + # filesystem that cannot store capabilities, just runs at default priority as before. + setcap 'cap_sys_nice=ep' /usr/bin/punktfunk-host 2>/dev/null || true # Pick up the /dev/uinput rule without a reboot (best-effort, no-op in containers). udevadm control --reload-rules 2>/dev/null || true udevadm trigger --subsystem-match=misc 2>/dev/null || true diff --git a/packaging/nix/nixos-module.nix b/packaging/nix/nixos-module.nix index d5dcf8a3..8b7c2ae6 100644 --- a/packaging/nix/nixos-module.nix +++ b/packaging/nix/nixos-module.nix @@ -356,6 +356,26 @@ in allowedUDPPorts = nativeUDP ++ optionals cfg.host.gamestream gamestreamUDP; }; + # CAP_SYS_NICE — the GPU-scheduling grant. PyroWave encodes on the GPU shader cores a game + # saturates; the elevated global-priority Vulkan queue that fixes it is gated on this + # capability (measured 2026-08-08, RTX 5070 Ti: without it EVERY priority class is refused, + # with it the encoder gets REALTIME on the first attempt; RADV behaves the same). + # + # NixOS cannot `setcap` a store path — it is read-only and shared — so this goes through + # `security.wrappers`, which builds a small setcap'd wrapper in /run/wrappers/bin. The unit's + # ExecStart points at the wrapper below; everything else about the host is unchanged. + # + # Narrow: CAP_SYS_NICE permits raising scheduling priority only — no filesystem, network or + # user-switching privilege, and the wrapper is capability-based, NOT setuid. Two side effects + # to know: the wrapped binary is AT_SECURE (the loader ignores LD_LIBRARY_PATH/LD_PRELOAD for + # it) and core dumps are suppressed by default. + security.wrappers.punktfunk-host = { + source = "${cfg.host.package}/bin/punktfunk-host"; + capabilities = "cap_sys_nice=ep"; + owner = "root"; + group = "root"; + }; + systemd.user.services.punktfunk-host = { description = "punktfunk GameStream + punktfunk/1 streaming host"; documentation = [ "https://git.unom.io/unom/punktfunk" ]; @@ -374,8 +394,12 @@ in # PUNKTFUNK_GAMESCOPE_BIN so an operator's own override of that env still wins. ++ optional cfg.host.gamescopeHdr cfg.host.gamescopePackage; serviceConfig = { + # Through the wrapper (see `security.wrappers.punktfunk-host` above), NOT the store path + # directly — the store path carries no capability and the GPU-priority lever would be + # inert. `config.security.wrapperDir` rather than a hard-coded /run/wrappers/bin so an + # operator who has moved it is still correct. ExecStart = - "${cfg.host.package}/bin/punktfunk-host serve" + optionalString cfg.host.gamestream " --gamestream"; + "${config.security.wrapperDir}/punktfunk-host serve" + optionalString cfg.host.gamestream " --gamestream"; Restart = "on-failure"; RestartSec = 2; EnvironmentFile = diff --git a/packaging/rpm/punktfunk.spec b/packaging/rpm/punktfunk.spec index d4d3eea9..77361408 100644 --- a/packaging/rpm/punktfunk.spec +++ b/packaging/rpm/punktfunk.spec @@ -477,7 +477,15 @@ install -Dm0644 scripts/punktfunk-scripting.service %{buildroot}%{_userunitdir}/ %files %license LICENSE-MIT LICENSE-APACHE THIRD-PARTY-NOTICES.txt %doc README.md packaging/README.md -%{_bindir}/punktfunk-host +# CAP_SYS_NICE — the GPU-scheduling grant, declared the RPM-native way so rpm applies it at +# install, restores it on upgrade, and VERIFIES it (a plain %post setcap does none of those). +# PyroWave encodes on the shader cores a game saturates; the elevated global-priority Vulkan queue +# that fixes it is gated on this capability. Measured 2026-08-08 on an RTX 5070 Ti: without it +# every priority class is refused, with it the encoder gets REALTIME first try (RADV the same). +# Narrow — scheduling priority only, no filesystem/network/user-switching privilege, not setuid. +# Consequences: the binary becomes AT_SECURE, so the loader ignores LD_LIBRARY_PATH/LD_PRELOAD for +# it, and core dumps are suppressed by default. +%caps(cap_sys_nice=ep) %{_bindir}/punktfunk-host %{_bindir}/punktfunk-tray %{_udevrulesdir}/60-punktfunk.rules %dir %{_libexecdir}/punktfunk diff --git a/scripts/steamdeck/install.sh b/scripts/steamdeck/install.sh index efe7c329..ce462646 100755 --- a/scripts/steamdeck/install.sh +++ b/scripts/steamdeck/install.sh @@ -344,6 +344,25 @@ if [ "$SUDO_OK" = 1 ]; then warn "(everything else works; the pad arrives as a generic Xbox 360 controller). By hand:" warn " sudo groupadd --system punktfunk; sudo usermod -aG punktfunk $USER" fi + # CAP_SYS_NICE on the host binary — the GPU-scheduling grant, and the Deck is the box that + # needs it most: a Van Gogh APU shares one small GPU between the game and PyroWave's encode + # dispatch. The driver gates the elevated global-priority Vulkan queue on this capability + # (measured 2026-08-08 on an RTX 5070 Ti: refused without it, granted REALTIME with it; RADV + # behaves the same), so without this the knob exists and does nothing. + # + # The binary lives under $HOME, not /usr — so unlike the /etc drop-ins above this survives a + # SteamOS A/B update on its own and needs no atomic-keep entry. It DOES need re-applying after + # every rebuild, because a fresh binary is a new inode; re-running this installer does that. + # + # Narrow (scheduling priority only, no filesystem/network privilege, not setuid) and + # best-effort — a failure just means the encode runs at default priority as it does today. + if [ -x "$BIN" ]; then + if sudo setcap 'cap_sys_nice=ep' "$BIN" 2>/dev/null; then + ok "granted CAP_SYS_NICE (PyroWave encode can outrank a GPU-bound game)" + else + warn "could not grant CAP_SYS_NICE to $BIN — PyroWave encode stays at default GPU priority" + fi + fi # SteamOS A/B updates rebuild /etc and DROP everything not on Valve's keep list — verified # live: an OS update stripped the udev rule + vhci autoload + UDP sysctl (gamepads silently # degrade to Xbox 360, buffers back to 208 KB). The sanctioned fix is a preserve drop-in in diff --git a/tools/loss-harness/src/main.rs b/tools/loss-harness/src/main.rs index 0caeb881..2482a37b 100644 --- a/tools/loss-harness/src/main.rs +++ b/tools/loss-harness/src/main.rs @@ -9,6 +9,7 @@ use punktfunk_core::config::{Config, FecConfig, FecScheme, ProtocolPhase, Role}; use punktfunk_core::crypto::SessionKey; use punktfunk_core::error::PunktfunkError; +use punktfunk_core::packet::{FLAG_PIC, FLAG_SOF, USER_FLAG_CHUNK_ALIGNED}; use punktfunk_core::session::Session; use punktfunk_core::transport::loopback_pair; @@ -83,6 +84,281 @@ fn run( (completed, frames) } +// --------------------------------------------------------------------------- +// PW6: partial delivery under loss — streamed AU vs whole AU +// --------------------------------------------------------------------------- +// +// The question this answers (wave-2 plan PW6, security-review finding 10): a PyroWave client +// enables `set_deliver_partial_frames` unconditionally, so a chunk-aligned AU that loses shards +// is still handed up as blocks-with-holes — one frame of localized blur instead of a freeze. But +// a STREAMED frame is excluded from that when it is UNPINNED: its size lives only on the FINAL +// block's headers (`frame_bytes` is the 0 sentinel until then), and `advance_window` refuses to +// deliver a partial it cannot truncate. So where the whole-AU path delivers blur, a streamed +// frame whose final block is entirely lost delivers NOTHING. +// +// Three legs, because a bare 2 % sweep cannot see the effect (see `partial_sweep`'s note): +// 1. `final_block_probe` — DETERMINISTIC: drop exactly the frame's last block in both shapes. +// Proves the mechanism exists (or does not) without any statistics. +// 2. `partial_sweep` — RANDOM Bernoulli loss, both shapes, same seed: the delivery rates. +// 3. the stress rows — the same sweep at higher loss, where the gap becomes measurable. + +/// The realistic PyroWave wire geometry: 1500-MTU IPv4 shards, 200 data shards per FEC block, +/// and **FEC pinned OFF** (the Phase-4 recipe — parity would mask exactly the loss under study). +fn partial_config(role: Role) -> Config { + Config { + role, + phase: ProtocolPhase::P2Punktfunk, + fec: FecConfig { + scheme: FecScheme::Gf16, + fec_percent: 0, + max_data_per_block: 200, + }, + shard_payload: 1408, + max_frame_bytes: 8 * 1024 * 1024, + encrypt: false, + key: SessionKey::Aes128Gcm([0u8; 16]), + salt: [0u8; 4], + loopback_drop_period: 0, // loss is injected here, per packet, so it can be random + } +} + +/// Reproducible xorshift64* — the harness must be re-runnable to the same numbers, and +/// `loopback_drop_period`'s deterministic 1-in-N cannot model independent per-packet loss +/// (it would systematically hit or miss the final block, which is the whole question). +struct Rng(u64); +impl Rng { + fn new(seed: u64) -> Rng { + Rng(seed | 1) + } + fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + /// True with probability `pct`/10000 (basis points, so 2 % = 200). + fn hits(&mut self, bp: u32) -> bool { + (self.next_u64() % 10_000) < bp as u64 + } + fn range(&mut self, lo: usize, hi: usize) -> usize { + lo + (self.next_u64() as usize) % (hi - lo).max(1) + } +} + +/// How each source frame ended up at the client. +#[derive(Default, Clone, Copy)] +struct Outcome { + complete: usize, + partial: usize, + nothing: usize, +} + +impl Outcome { + fn total(&self) -> usize { + self.complete + self.partial + self.nothing + } + /// Partial deliveries as a percentage of frames that did NOT arrive complete — "when the + /// frame was damaged, how often did the user still get a picture?". That ratio, not the raw + /// count, is what the two wire shapes must be compared on: they damage different numbers of + /// frames at the same packet-loss rate (streamed adds no parity but does add a final block + /// whose loss is fatal, and the shapes' block splits differ slightly). + fn rescue_pct(&self) -> f64 { + let damaged = self.partial + self.nothing; + if damaged == 0 { + return 100.0; + } + 100.0 * self.partial as f64 / damaged as f64 + } +} + +/// How many packets the frame's LAST FEC block occupies, and how many packets the whole AU +/// should seal into. With FEC pinned off there is no parity, so `packetize_each` emits exactly +/// one packet per data shard in block order — the final block is therefore the last `final_k` +/// packets of the batch. Returned together so the caller can ASSERT the packet count and fail +/// loudly if that emission shape ever changes, rather than silently probing the wrong packets. +fn final_block_span(len: usize, shard: usize, per_block: usize) -> (usize, usize) { + let shards = len.div_ceil(shard); + let blocks = shards.div_ceil(per_block); + let final_k = shards - (blocks - 1) * per_block; + (shards, final_k) +} + +/// Drive `frames` AUs through a host→client pair and classify each one. `loss_bp` is the +/// per-packet loss probability in basis points; `final_only` instead forces the frame's LAST +/// block to be dropped wholesale, and nothing else (the deterministic mechanism probe). +/// +/// `sizes` gives each frame's AU length. Real PyroWave AUs vary frame to frame under rate +/// control, and the FINAL block's size is what bounds this trap's exposure, so the sweep varies +/// the length across the whole 1..=200-shard range of final-block sizes rather than pinning one. +fn run_partial( + streamed: bool, + sizes: &[usize], + loss_bp: u32, + final_only: bool, + seed: u64, +) -> Outcome { + // Flush frames: `advance_window` only ages a frame out once something NEWER exists and the + // capture-time fuse has passed (PARTIAL_WINDOW_NS = 30 ms vs a 16.67 ms frame period), so + // the tail of the run needs successors before its verdicts land. + const FLUSH: usize = 8; + const FRAME_NS: u64 = 16_666_667; + + let (h, c) = loopback_pair(0, 0); + let mut host = Session::new(partial_config(Role::Host), Box::new(h)).unwrap(); + let mut client = Session::new(partial_config(Role::Client), Box::new(c)).unwrap(); + // The PyroWave client's real setting (`client/pump/handshake.rs` turns this on for every + // CODEC_PYROWAVE session). + client.set_deliver_partial_frames(true); + + let mut rng = Rng::new(seed); + // frame_index -> saw a complete delivery + let mut delivered: std::collections::HashMap = std::collections::HashMap::new(); + let flags = FLAG_PIC as u32 | FLAG_SOF as u32 | USER_FLAG_CHUNK_ALIGNED; + + let n = sizes.len(); + for f in 0..(n + FLUSH) { + let len = sizes[f.min(n - 1)]; + // Busy, frame-varying content — never a flat fill (a constant buffer would still + // reassemble byte-identically, but it makes every debug dump look alike). + let data: Vec = (0..len).map(|b| (b.wrapping_mul(31) ^ f) as u8).collect(); + let pts = f as u64 * FRAME_NS; + let fi = f as u32; + + // Send one sealed batch. `kill_from` is the index at/after which every packet is dropped + // outright (the deterministic final-block probe); otherwise each packet is lost + // independently at `loss_bp`. + let mut send = |host: &mut Session, wires: Vec>, kill_from: usize| { + let refs: Vec<&[u8]> = wires + .iter() + .enumerate() + .filter(|(i, _)| *i < kill_from && !(loss_bp > 0 && rng.hits(loss_bp))) + .map(|(_, w)| w.as_slice()) + .collect(); + if !refs.is_empty() { + host.send_sealed(&refs).unwrap(); + } + drop(refs); + host.reclaim_wires(wires); + }; + + if streamed { + let mut au = host.begin_streamed_frame_at(pts, flags, fi).unwrap(); + // Cut at the encoder's chunk granularity (the PW6 `AuChunker` default: 256 KiB + // rounded down to whole 1408-byte windows = 186 windows). + for chunk in data.chunks(186 * 1408) { + let wires = host.seal_streamed_chunk(&mut au, chunk, false).unwrap(); + send(&mut host, wires, usize::MAX); + } + let wires = host.seal_streamed_finish(au).unwrap(); + // The finish batch IS the final block — the only one carrying the real totals. + send(&mut host, wires, if final_only { 0 } else { usize::MAX }); + } else { + let wires = host.seal_frame_at(&data, pts, flags, fi).unwrap(); + let (shards, final_k) = final_block_span(len, 1408, 200); + assert_eq!( + wires.len(), + shards, + "FEC is off, so the whole-AU batch must be exactly one packet per data shard — \ + the final-block probe's index rule depends on it" + ); + let kill_from = if final_only { + wires.len() - final_k + } else { + usize::MAX + }; + send(&mut host, wires, kill_from); + } + + loop { + match client.poll_frame() { + Ok(got) => { + let e = delivered.entry(got.frame_index).or_insert(false); + *e |= got.complete; + } + Err(PunktfunkError::NoFrame) => break, + Err(e) => panic!("unexpected error: {e}"), + } + } + } + + let mut out = Outcome::default(); + for f in 0..n { + match delivered.get(&(f as u32)) { + Some(true) => out.complete += 1, + Some(false) => out.partial += 1, + None => out.nothing += 1, + } + } + out +} + +/// AU lengths spanning the full range of FINAL-block sizes (1..=200 shards on top of two full +/// 200-shard blocks) — 564 KB…845 KB, i.e. the 400 Mb/s-at-60fps operating point. +fn varied_sizes(count: usize, seed: u64) -> Vec { + let mut rng = Rng::new(seed); + (0..count) + .map(|_| rng.range(401 * 1408, 600 * 1408 + 1)) + .collect() +} + +fn partial_section() { + let frames: usize = std::env::var("PW6_FRAMES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(2000); + + println!("\n\npunktfunk PW6 — partial delivery under loss: STREAMED vs WHOLE AU"); + println!("(chunk-aligned AUs, deliver_partial ON, FEC pinned OFF, shard 1408, 200/block)\n"); + + // ---- Leg 1: the mechanism, deterministically ------------------------------------------- + println!("Leg 1 — DETERMINISTIC probe: the frame's LAST block is lost, nothing else."); + let sizes = varied_sizes(200, 0xC0FFEE); + for (label, streamed) in [("whole-AU", false), ("streamed", true)] { + let o = run_partial(streamed, &sizes, 0, true, 1); + println!( + " {label:>8}: complete {:>4} partial {:>4} NOTHING {:>4} (of {})", + o.complete, + o.partial, + o.nothing, + o.total() + ); + } + println!( + " → if the streamed row shows NOTHING where whole-AU shows partial, the trap is real." + ); + + // ---- Leg 2 + 3: rates under random loss ------------------------------------------------- + println!("\nLeg 2/3 — RANDOM per-packet loss, same seed and same AU sizes for both shapes."); + println!(" 'rescue' = partials / (partials + nothing): of the frames that arrived DAMAGED,"); + println!(" how many still reached the decoder as blur instead of vanishing.\n"); + println!( + "{:>7} {:>9} {:>26} {:>26}", + "loss", "shape", "complete / partial / none", "rescue of damaged" + ); + println!("{}", "-".repeat(78)); + let sizes = varied_sizes(frames, 0xBEEF); + for &bp in &[200u32, 1000, 3000, 5000] { + for (label, streamed) in [("whole-AU", false), ("streamed", true)] { + let o = run_partial(streamed, &sizes, bp, false, 0x5EED); + println!( + "{:>6.1}% {label:>9} {:>8} / {:>7} / {:>5} {:>24.2}%", + bp as f64 / 100.0, + o.complete, + o.partial, + o.nothing, + o.rescue_pct() + ); + } + } + println!( + "\nNote: at 2 % the streamed penalty is bounded by P(final block fully lost) =\n\ + E[0.02^k] over final-block sizes k — ~1e-4 — so the 2 % row is EXPECTED to tie.\n\ + The higher-loss rows are what make the gap (if any) visible; Leg 1 proves the mechanism." + ); +} + fn main() { let frames = 50; let frame_len = 100_000; // ~98 shards across 2 FEC blocks @@ -111,4 +387,6 @@ fn main() { ); } println!("\nNote: recovery drops off once per-block loss exceeds the 25% recovery budget."); + + partial_section(); }