fix(host): coalesce keyframe-request storms into one IDR per cooldown
Clients request a keyframe on every FEC-unrecoverable frame and keep asking until the IDR actually arrives and decodes — a full round-trip on a link that is already behind. The host answered every request with a full IDR and only rate-limited under intra-refresh (opt-in, off by default), so on the default path a Wi-Fi loss burst produced a 20-40x bitrate spike storm that deepened the very loss it was recovering from — the second half of the periodic double-jolt. Coalesce a request storm into at most one forced IDR per cooldown ALWAYS: serve the first immediately (a genuinely wedged decoder still recovers at once), then suppress for the window (2 s under intra-refresh's healing wave, 750 ms for a full-IDR recovery — long enough to swallow one recovery event's round-trip echo, short enough to re-issue a lost IDR promptly). Seed the cooldown at session open and stamp it on both rebuild paths so the cold-open / post-rebuild storm coalesces into the IDR the fresh encoder already emits. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -3240,8 +3240,11 @@ fn virtual_stream(ctx: SessionContext) -> Result<()> {
|
|||||||
// compositing), NOT an encoder problem. Logged every 2 s when `PUNKTFUNK_PERF`.
|
// compositing), NOT an encoder problem. Logged every 2 s when `PUNKTFUNK_PERF`.
|
||||||
let (mut diag_new, mut diag_repeat) = (0u64, 0u64);
|
let (mut diag_new, mut diag_repeat) = (0u64, 0u64);
|
||||||
let mut diag_at = std::time::Instant::now();
|
let mut diag_at = std::time::Instant::now();
|
||||||
// Last client-requested forced IDR — the intra-refresh rate limit window anchor (see below).
|
// Anchor for the forced-IDR cooldown (see the keyframe-request handling below): the timestamp of
|
||||||
let mut last_forced_idr: Option<std::time::Instant> = None;
|
// the most recent forced/opening IDR. The session's pipeline just opened on an IDR, so start the
|
||||||
|
// clock now — that coalesces the keyframe storm a client fires while its decoder wedges on the cold
|
||||||
|
// opening GOP, instead of answering it with a redundant second IDR.
|
||||||
|
let mut last_forced_idr: Option<std::time::Instant> = Some(std::time::Instant::now());
|
||||||
// Per-stage latency breakdown (PUNKTFUNK_PERF): per-call µs for the GPU-bound stages so we see
|
// Per-stage latency breakdown (PUNKTFUNK_PERF): per-call µs for the GPU-bound stages so we see
|
||||||
// exactly where the capture→encoded latency goes — cap=try_latest (ring read + colour convert),
|
// exactly where the capture→encoded latency goes — cap=try_latest (ring read + colour convert),
|
||||||
// submit=encode_picture launch, wait=lock_bitstream (the scheduling wait + ASIC encode, the one
|
// submit=encode_picture launch, wait=lock_bitstream (the scheduling wait + ASIC encode, the one
|
||||||
@@ -3355,6 +3358,7 @@ fn virtual_stream(ctx: SessionContext) -> Result<()> {
|
|||||||
inflight.clear();
|
inflight.clear();
|
||||||
last_au_at = std::time::Instant::now();
|
last_au_at = std::time::Instant::now();
|
||||||
encoder_resets = 0;
|
encoder_resets = 0;
|
||||||
|
last_forced_idr = Some(std::time::Instant::now()); // fresh encoder opens on an IDR — anchor the cooldown
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::error!(error = %format!("{e:#}"), ?new_mode,
|
tracing::error!(error = %format!("{e:#}"), ?new_mode,
|
||||||
@@ -3371,16 +3375,29 @@ fn virtual_stream(ctx: SessionContext) -> Result<()> {
|
|||||||
want_kf = true;
|
want_kf = true;
|
||||||
}
|
}
|
||||||
if want_kf {
|
if want_kf {
|
||||||
// Intra-refresh mode: clients request a keyframe on EVERY FEC-unrecoverable frame
|
// Clients request a keyframe on EVERY FEC-unrecoverable frame (`frames_dropped` polling)
|
||||||
// (`frames_dropped` polling), but the refresh wave already heals those within ~half a
|
// and keep asking until the IDR actually arrives + decodes — a full round-trip on a link
|
||||||
// second — answering each with a full IDR is the 20-40× spike cascade the wave exists
|
// that is already behind. Answering each request with a full IDR is a 20-40× bitrate spike
|
||||||
// to avoid. Serve the first request immediately (a genuinely wedged decoder recovers
|
// that DEEPENS the very loss it is recovering from: a burst of loss → a storm of IDRs →
|
||||||
// at once), then suppress further requests for one window and let the wave heal.
|
// more loss, the periodic double-jolt a Wi-Fi client sees. So coalesce a request storm into
|
||||||
const IDR_WINDOW: std::time::Duration = std::time::Duration::from_secs(2);
|
// at most ONE forced IDR per cooldown, ALWAYS — not only under intra-refresh (the old gate;
|
||||||
let suppress = enc.caps().intra_refresh
|
// a full-IDR recovery is exactly where the storm is worst). Serve the first request
|
||||||
&& last_forced_idr.is_some_and(|t| t.elapsed() < IDR_WINDOW);
|
// immediately (a genuinely wedged decoder recovers at once), then suppress for the window.
|
||||||
|
//
|
||||||
|
// Intra-refresh heals via its own gradual wave (~0.5 s) and can afford a long window; a
|
||||||
|
// full-IDR recovery relies on the keyframe itself, so its window is shorter — long enough to
|
||||||
|
// swallow the round-trip echo of one recovery event, short enough to re-issue a *lost* IDR
|
||||||
|
// promptly.
|
||||||
|
const IDR_COOLDOWN_INTRA: std::time::Duration = std::time::Duration::from_secs(2);
|
||||||
|
const IDR_COOLDOWN_FULL: std::time::Duration = std::time::Duration::from_millis(750);
|
||||||
|
let window = if enc.caps().intra_refresh {
|
||||||
|
IDR_COOLDOWN_INTRA
|
||||||
|
} else {
|
||||||
|
IDR_COOLDOWN_FULL
|
||||||
|
};
|
||||||
|
let suppress = last_forced_idr.is_some_and(|t| t.elapsed() < window);
|
||||||
if suppress {
|
if suppress {
|
||||||
tracing::debug!("keyframe request suppressed — intra-refresh wave healing");
|
tracing::debug!("keyframe request coalesced — within the IDR cooldown");
|
||||||
} else {
|
} else {
|
||||||
tracing::debug!("forcing keyframe (client decode recovery)");
|
tracing::debug!("forcing keyframe (client decode recovery)");
|
||||||
enc.request_keyframe();
|
enc.request_keyframe();
|
||||||
@@ -3523,6 +3540,7 @@ fn virtual_stream(ctx: SessionContext) -> Result<()> {
|
|||||||
interval = new_interval;
|
interval = new_interval;
|
||||||
cur_node_id = new_node_id;
|
cur_node_id = new_node_id;
|
||||||
enc.request_keyframe(); // belt-and-suspenders; a fresh encoder opens on an IDR anyway
|
enc.request_keyframe(); // belt-and-suspenders; a fresh encoder opens on an IDR anyway
|
||||||
|
last_forced_idr = Some(std::time::Instant::now()); // anchor the IDR cooldown from the rebuild
|
||||||
next = std::time::Instant::now();
|
next = std::time::Instant::now();
|
||||||
// The owed AUs died with the old encoder — drop their in-flight records and
|
// The owed AUs died with the old encoder — drop their in-flight records and
|
||||||
// restart the encode-stall clock (the rebuild loop above may have eaten seconds,
|
// restart the encode-stall clock (the rebuild loop above may have eaten seconds,
|
||||||
|
|||||||
Reference in New Issue
Block a user