fix(core/clock): re-sync survives loaded links — floor baseline, spaced rounds, bounded staleness
The mid-stream clock re-sync starved on high-bitrate LAN sessions (2026-07 PyroWave-sawtooth field report, RX 9070 XT -> 780M @ 550 Mb/s): every batch was judged against the CONNECT-TIME RTT, measured before the video data plane existed, with a 2 ms floor — mid-stream control RTTs on a loaded GbE link sit above that almost permanently, so batches were rejected for minutes while the wall clocks drifted apart and the OSD e2e figure ramped 19->150 ms before snapping back on a lucky batch. Three changes: - Rounds are spaced 7 ms apart (stamped at send time, so the spacing never lands in the RTT). An 8-round batch used to complete inside ONE ~6 ms video burst — all rounds sampled the same congestion state; spacing walks them across the frame cycle so the min-RTT round finds a quiet gap. - ResyncGuard replaces the static baseline: the guard band follows the best RTT the session has evidenced (connect RTT, then min over every completed batch — rejected ones included). - Rejection streaks are bounded: after 3 consecutive rejections the best (min-RTT) batch of the streak is applied anyway. Its queueing bias is at most ~half its RTT; unbounded wall-clock drift costs more per minute. Rejections now log at warn with the streak and floor — the starvation signature is grep-able in exactly the logs users send. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -52,8 +52,8 @@ pub(crate) struct Negotiated {
|
||||
/// Host clock minus client clock (ns); `0` = no skew handshake (old host / synced clocks).
|
||||
pub(crate) clock_offset_ns: i64,
|
||||
/// Min RTT of the connect-time skew handshake (ns); `None` = the host never answered —
|
||||
/// mid-stream re-syncs are pointless then and stay off. The re-sync acceptance guard
|
||||
/// compares each batch against this baseline ([`accept_resync`]).
|
||||
/// mid-stream re-syncs are pointless then and stay off. Seeds the re-sync admission
|
||||
/// guard's session-floor baseline ([`ResyncGuard`]).
|
||||
pub(crate) clock_rtt_ns: Option<u64>,
|
||||
/// Resolved encode bit depth: `8`, or `10` for a Main10 / HDR session.
|
||||
pub(crate) bit_depth: u8,
|
||||
|
||||
@@ -11,8 +11,8 @@ use crate::abr::BitrateController;
|
||||
use crate::config::Role;
|
||||
use crate::packet::FLAG_PROBE;
|
||||
use crate::quic::{
|
||||
accept_resync, io, wall_clock_ns, window_loss_ppm, BitrateChanged, ClipState, ClockEcho,
|
||||
ClockResync, Hello, LossReport, ProbeResult, Reconfigure, Reconfigured, RequestKeyframe,
|
||||
io, wall_clock_ns, window_loss_ppm, BitrateChanged, ClipState, ClockEcho, ClockResync, Hello,
|
||||
LossReport, ProbeResult, Reconfigure, Reconfigured, RequestKeyframe, ResyncAdmit, ResyncGuard,
|
||||
ResyncStep, SetBitrate, Start, Welcome,
|
||||
};
|
||||
use crate::session::Session;
|
||||
|
||||
@@ -52,6 +52,14 @@ impl ControlTask {
|
||||
// the read arm below; only when the host answered the connect-time handshake — an
|
||||
// old host would just eat the probes.
|
||||
let mut resync = ClockResync::new();
|
||||
let mut resync_guard = clock_rtt_ns.map(ResyncGuard::new);
|
||||
// Inter-round spacing: without it the whole 8-round batch completes inside one
|
||||
// ~6 ms video burst and every round samples the same congestion state — a batch
|
||||
// that starts mid-burst is then wholly congested and gets rejected. 7 ms staggers
|
||||
// the rounds across the ~16.7 ms frame cycle so the min-RTT round almost always
|
||||
// lands in a quiet inter-burst gap, even at PyroWave-class bitrates.
|
||||
const RESYNC_ROUND_SPACING: std::time::Duration = std::time::Duration::from_millis(7);
|
||||
let mut staged_round: Option<tokio::time::Instant> = None;
|
||||
let mut resync_tick = tokio::time::interval_at(
|
||||
tokio::time::Instant::now() + CLOCK_RESYNC_INTERVAL,
|
||||
CLOCK_RESYNC_INTERVAL,
|
||||
@@ -72,6 +80,7 @@ impl ControlTask {
|
||||
if clock_rtt_ns.is_none() {
|
||||
continue; // no connect-time handshake — host can't answer
|
||||
}
|
||||
staged_round = None; // a new batch abandons any staged round
|
||||
resync.begin(wall_clock_ns()).encode()
|
||||
}
|
||||
CtrlRequest::ClipControl(c) => c.encode(),
|
||||
@@ -83,11 +92,21 @@ impl ControlTask {
|
||||
}
|
||||
}
|
||||
_ = resync_tick.tick(), if clock_rtt_ns.is_some() => {
|
||||
staged_round = None; // a new batch abandons any staged round
|
||||
let probe = resync.begin(wall_clock_ns());
|
||||
if io::write_msg(&mut ctrl_send, &probe.encode()).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
_ = async { tokio::time::sleep_until(staged_round.unwrap()).await },
|
||||
if staged_round.is_some() => {
|
||||
staged_round = None;
|
||||
// Stamped at send time so the inter-round spacing stays out of the RTT.
|
||||
let probe = resync.next_probe(wall_clock_ns());
|
||||
if io::write_msg(&mut ctrl_send, &probe.encode()).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
msg = ctrl_recv.read_msg() => {
|
||||
let Ok(msg) = msg else { break }; // stream closed
|
||||
if let Ok(ack) = Reconfigured::decode(&msg) {
|
||||
@@ -132,16 +151,36 @@ impl ControlTask {
|
||||
*bitrate_ack.lock().unwrap() = Some(ack.bitrate_kbps);
|
||||
} else if let Ok(echo) = ClockEcho::decode(&msg) {
|
||||
match resync.on_echo(&echo, wall_clock_ns()) {
|
||||
ResyncStep::Probe(p) => {
|
||||
if io::write_msg(&mut ctrl_send, &p.encode()).await.is_err() {
|
||||
break;
|
||||
}
|
||||
ResyncStep::MoreRounds => {
|
||||
staged_round = Some(
|
||||
tokio::time::Instant::now() + RESYNC_ROUND_SPACING,
|
||||
);
|
||||
}
|
||||
ResyncStep::Done { offset_ns, rtt_ns } => {
|
||||
// Never let a congested window bias the offset (frames read
|
||||
// late exactly then) — keep the old estimate and let the next
|
||||
// periodic batch try again.
|
||||
if accept_resync(rtt_ns, clock_rtt_ns.unwrap_or(0)) {
|
||||
let Some(guard) = resync_guard.as_mut() else {
|
||||
continue; // no connect handshake — batches never start
|
||||
};
|
||||
let (apply, best_of_streak) = match guard.admit(offset_ns, rtt_ns)
|
||||
{
|
||||
ResyncAdmit::Fresh => (Some((offset_ns, rtt_ns)), false),
|
||||
ResyncAdmit::BestOfStreak { offset_ns, rtt_ns } => {
|
||||
(Some((offset_ns, rtt_ns)), true)
|
||||
}
|
||||
ResyncAdmit::Rejected { streak } => {
|
||||
// warn, not debug: repeated rejections are exactly the
|
||||
// stale-offset starvation signature the 2026-07
|
||||
// PyroWave-sawtooth report had to be diagnosed without.
|
||||
tracing::warn!(
|
||||
rtt_us = rtt_ns / 1000,
|
||||
floor_us = guard.floor_rtt_ns() / 1000,
|
||||
streak,
|
||||
"clock re-sync batch rejected — RTT above the \
|
||||
session floor (congested window)"
|
||||
);
|
||||
(None, false)
|
||||
}
|
||||
};
|
||||
if let Some((offset_ns, rtt_ns)) = apply {
|
||||
// info, not debug: ≤1/min, and it is THE forensic
|
||||
// trail for a stale-offset (stepped/slewed wall clock)
|
||||
// latency plateau — the 2026-07 two-pair investigation
|
||||
@@ -149,16 +188,11 @@ impl ControlTask {
|
||||
tracing::info!(
|
||||
offset_ns,
|
||||
rtt_us = rtt_ns / 1000,
|
||||
best_of_streak,
|
||||
"mid-stream clock re-sync applied"
|
||||
);
|
||||
clock_offset.store(offset_ns, Ordering::Relaxed);
|
||||
clock_gen.fetch_add(1, Ordering::Relaxed);
|
||||
} else {
|
||||
tracing::info!(
|
||||
rtt_us = rtt_ns / 1000,
|
||||
"clock re-sync batch discarded — RTT above the \
|
||||
connect-time baseline (congested window)"
|
||||
);
|
||||
}
|
||||
}
|
||||
ResyncStep::Idle => {}
|
||||
|
||||
Reference in New Issue
Block a user