From 744bcb468be6e4160a2f884961f9ac9a544ffc43 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Sat, 8 Aug 2026 21:43:06 +0200 Subject: [PATCH] =?UTF-8?q?feat(host/wire):=20a=20jumbo=20path=20can=20now?= =?UTF-8?q?=20be=20PROVEN=20=E2=80=94=20and=20the=20shipped=20grow=20never?= =?UTF-8?q?=20could?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wave-2 PW7a: a PyroWave session on a proven-jumbo LAN should START at the big shard, because it is the one codec that can never be re-keyed mid-stream (its client parses chunk-aligned AUs in windows of the `Welcome` value, read once over the C ABI). At an 8908-byte shard that is ~6× fewer datagrams per frame — ~49k → ~8k pps at 550 Mb/s — and proportionally less window-tail padding. THE BLOCKER FOUND FIRST: the whole jumbo leg was dead code, not just the missing half. quinn caps a peer's MTU-discovery search at `min(MtuDiscoveryConfig::upper_bound, the OTHER side's advertised max_udp_payload_size)` (`quinn_proto::connection::mtud::SearchState::new`), and `EndpointConfig::max_udp_payload_size` defaults to 1472. Nothing in the repo had ever touched `EndpointConfig`, so raising the host's PROBE ceiling — all `stream_transport_idle` did — could never make discovery settle above 1472, and the shipped mid-session grow's `settled >= sealed_datagram_bytes(target)` gate was unreachable on every path that has ever existed. Two smaller contributors, fixed here too: the watcher stopped sampling the moment `settled >= 1472`, discarding the very climb the proof needs, and a session sealed ABOVE the 1500-byte default was never checked against the path at all. The advertisement is raised on the CLIENT endpoint, under the same `jumbo_wire_mtu()` opt-in as the probe ceiling, because it is not free: quinn sizes its endpoint receive buffer `max_udp_payload_size × max_receive_segments × BATCH_SIZE`, so on a GRO-capable Linux/Android client that is ~2.9 MiB at the default and ~18 MiB at jumbo (47 KiB → 288 KiB on Apple/Windows). Consequence: jumbo now needs the opt-in on BOTH ends. Without it, every byte on the wire and every byte of buffer is exactly what it was. WHY THE GROW IS AS SAFE AS THE CLAMP, which is not obvious — the failure modes are opposite. A stale clamp only makes datagrams smaller than they had to be; a stale grow seals an oversized datagram onto a 1500-byte path, where it is silently dropped, and a PyroWave session cannot recover from that for its whole life. Mirroring the clamp's keying is therefore NOT sufficient. So the memory is demoted: the persisted verdict only decides whether it is worth WAITING for a proof, and what authorises the grow is a LIVE re-proof on the very connection being welcomed — `conn.stats().path.current_mtu` ≥ the sealed target, i.e. a datagram of exactly that size acked by this client, on this connection, seconds ago. The moved laptop cannot inherit anything: its new path's live MTU is 1472 and the grow does not happen, whatever the memory says. The remembered half is keyed strictly anyway — `(local_ip, peer_ip)`, so a verdict earned over the host's 10 GbE NIC does not apply to the same peer over Wi-Fi or a VPN — and carries the operator target it was proven under plus a 6 h TTL. It is erased by any contrary evidence: a lower settle, a session that ended before the window closed (what a client staring at black does), a changed opt-in, or a constrained-path clamp that disagrees. The proof-wait is on the bring-up critical path (`handshake.rs` sends the `Welcome` and only then kicks the display prep), so it is bounded at 300 ms, exits the instant the proof lands, and is entered ONLY for a path a previous session already proved. Its worst case is the moved laptop, and that is self-limiting: that session's watcher erases the verdict. MEASURED, NOT ARGUED: `mtu_discovery_climbs_only_as_high_as_the_peer_advertises` (`#[ignore]`d, loopback — whose own MTU is 64 KiB, so configuration is the only thing that can stop the search), on .21: leg A (server opted in, client NOT): settled at 1472 B UDP payload <- the dead-code proof leg B (both opted in): reached 8972 B in 5 ms <- the fix, and its speed Leg A is the finding restated as an experiment. Leg B says the climb costs ~5 ms once both sides advertise it, so the 300 ms proof-wait is ~60x the loopback convergence time — enough headroom for a real LAN's RTT and per-probe ack delay across the ~11 probes the search takes. Still owed: the A/B on a real jumbo LAN segment (9000-MTU NIC + switch on both ends) — pps per frame, wire/pin ratio, and a PyroWave session observed starting at 8908. Not runnable without the hardware. --- crates/punktfunk-core/src/quic/endpoint.rs | 129 ++++- crates/punktfunk-host/src/native.rs | 7 +- crates/punktfunk-host/src/native/handshake.rs | 23 +- crates/punktfunk-host/src/native/wire_mtu.rs | 524 +++++++++++++++++- 4 files changed, 649 insertions(+), 34 deletions(-) diff --git a/crates/punktfunk-core/src/quic/endpoint.rs b/crates/punktfunk-core/src/quic/endpoint.rs index 43a786ff..fa810d86 100644 --- a/crates/punktfunk-core/src/quic/endpoint.rs +++ b/crates/punktfunk-core/src/quic/endpoint.rs @@ -82,6 +82,49 @@ fn stream_transport_idle(idle: std::time::Duration) -> Arc 1500) because it is not free: quinn sizes +/// its endpoint receive buffer as `max_udp_payload_size × max_receive_segments × BATCH_SIZE`, +/// which on a GRO-capable Linux/Android client is 64 × 32 segments — ~2.9 MiB at the 1472 +/// default, ~18 MiB at jumbo. A jumbo LAN is a deliberate deployment; every other client keeps +/// today's buffer to the byte. Without the opt-in this returns the stock config, so the +/// advertisement, the wire, and the memory are all unchanged. +fn endpoint_config() -> quinn::EndpointConfig { + let mut cfg = quinn::EndpointConfig::default(); + if let Some(mtu) = crate::config::jumbo_wire_mtu() { + // Derived exactly like the probe ceiling above (IPv4 overhead — a v6 peer's sealed + // target is smaller, so this covers it), and clamped into quinn's accepted range. + let shard = crate::config::jumbo_shard_payload_for( + mtu, + std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED), + ); + let accept = crate::config::sealed_datagram_bytes(shard).clamp(1200, 65_527) as u16; + if cfg.max_udp_payload_size(accept).is_ok() { + tracing::info!( + max_udp_payload_size = accept, + wire_mtu = mtu, + "jumbo opt-in: this endpoint advertises a jumbo QUIC receive ceiling, so the \ + peer's MTU discovery can prove a jumbo path (it is capped by this value)" + ); + } + } + cfg +} + /// Server endpoint with a fresh self-signed certificate (tests/dev — production hosts /// persist an identity and use [`server_with_identity`] so clients can pin it). pub fn server(addr: std::net::SocketAddr) -> anyhow_result::Result { @@ -238,7 +281,15 @@ pub fn client_pinned_with_identity( .map_err(|e| anyhow_result::Error::msg(format!("quic client config: {e}")))?; let mut client_cfg = quinn::ClientConfig::new(Arc::new(quic_cfg)); client_cfg.transport_config(stream_transport()); // keep-alive — see stream_transport - let mut ep = quinn::Endpoint::client("0.0.0.0:0".parse().unwrap())?; + + // `Endpoint::client` hardcodes `EndpointConfig::default()`, whose 1472-byte + // `max_udp_payload_size` caps the HOST's MTU discovery (see `endpoint_config`), so the + // endpoint is built by hand to carry the jumbo opt-in. Same bind as before + // (`0.0.0.0:0`, v4 — no dual-stack flag to reproduce) and the same default runtime. + let socket = std::net::UdpSocket::bind("0.0.0.0:0")?; + let runtime = quinn::default_runtime() + .ok_or_else(|| anyhow_result::Error::msg("no async runtime found".into()))?; + let mut ep = quinn::Endpoint::new(endpoint_config(), None, socket, runtime)?; ep.set_default_client_config(client_cfg); Ok(ep) })(); @@ -348,4 +399,80 @@ mod tests { let _ = super::stream_transport_idle(std::time::Duration::MAX); let _ = super::stream_transport_idle(std::time::Duration::ZERO); } + + /// Where a connection's MTU discovery is allowed to climb to, measured rather than argued + /// (PW7a). Loopback's own MTU is 64 KiB, so the ONLY thing that can stop the search here is + /// configuration — which makes this a clean instrument for the two ceilings: + /// + /// * **leg A** — server opted in, client NOT: the search stalls at the client's default + /// `max_udp_payload_size` advertisement (1472) no matter how high the server's probe + /// ceiling is. This is why the shipped jumbo grow could never fire: `wire_mtu.rs` waits + /// for a settle at the sealed jumbo size and the peer's transport parameter forbids it. + /// * **leg B** — both opted in: the search reaches the sealed jumbo datagram, and the + /// elapsed time is what the `Welcome`'s bounded proof-wait has to cover. + /// + /// `#[ignore]`d: it sets process-wide env (each endpoint reads the opt-in at construction, + /// which is exactly how the two legs are built) and spends seconds of wall clock. + /// Run it alone: `cargo test -p punktfunk-core --features quic mtu_discovery -- --ignored + /// --nocapture --test-threads=1`. + #[tokio::test] + #[ignore = "measurement: sets process env and takes ~15 s of wall clock"] + async fn mtu_discovery_climbs_only_as_high_as_the_peer_advertises() { + async fn climb(server_jumbo: bool, client_jumbo: bool) -> (u16, u128) { + let set = |on: bool| { + if on { + std::env::set_var("PUNKTFUNK_JUMBO", "1"); + } else { + std::env::remove_var("PUNKTFUNK_JUMBO"); + } + }; + set(server_jumbo); + let server = endpoint::server("127.0.0.1:0".parse().unwrap()).unwrap(); + let addr = server.local_addr().unwrap(); + set(client_jumbo); + let client = endpoint::client_insecure().unwrap(); + set(false); + let accept = tokio::spawn(async move { + let incoming = server.accept().await.expect("incoming"); + let conn = incoming.await.expect("host side connects"); + (server, conn) + }); + let client_conn = client.connect(addr, "punktfunk").unwrap().await.unwrap(); + let (_server_ep, host_conn) = accept.await.unwrap(); + // A stream write gives the driver something to transmit, which is what starts the + // search (probes ride `poll_transmit`); after that each probe's ack drives the next. + let mut s = host_conn.open_uni().await.unwrap(); + s.write_all(b"go").await.unwrap(); + let want = crate::config::sealed_datagram_bytes(crate::config::jumbo_shard_payload_for( + 9000, + std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED), + )) as u16; + let t0 = std::time::Instant::now(); + let mut mtu = host_conn.stats().path.current_mtu; + while t0.elapsed() < std::time::Duration::from_secs(6) && mtu < want { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + mtu = host_conn.stats().path.current_mtu; + } + let elapsed = t0.elapsed().as_millis(); + drop(client_conn); + drop(client); + (mtu, elapsed) + } + + let (capped, _) = climb(true, false).await; + println!("leg A (server opted in, client not): settled at {capped} B UDP payload"); + assert_eq!( + capped, 1472, + "a peer that advertises the stock max_udp_payload_size caps the search at 1472 — \ + the whole point of raising it on the client endpoint" + ); + + let (grown, ms) = climb(true, true).await; + println!("leg B (both opted in): reached {grown} B UDP payload in {ms} ms"); + assert!( + grown >= 8972, + "both sides opted in, loopback MTU is 64 KiB — discovery should reach the sealed \ + jumbo datagram, got {grown}" + ); + } } diff --git a/crates/punktfunk-host/src/native.rs b/crates/punktfunk-host/src/native.rs index 17baebf5..f74909e3 100644 --- a/crates/punktfunk-host/src/native.rs +++ b/crates/punktfunk-host/src/native.rs @@ -1148,7 +1148,12 @@ async fn serve_session( // path verdict (WARN + learned clamp for the next session on a constrained path; clears // a stale clamp on a healthy one) — and, with the driver above, heal or grow THIS // session mid-stream. Bounded ~10 s task unless a jumbo grow leaves it as revert guard. - wire_mtu::spawn_watch(conn.clone(), welcome.shard_payload as usize, shard_reneg); + wire_mtu::spawn_watch( + conn.clone(), + welcome.shard_payload as usize, + hello.max_shard_payload, + shard_reneg, + ); // Negotiated cursor forwarding: the HOST_CAP_CURSOR bit the Welcome advertised, read back // rather than recomputed (`handshake::cursor_forward` computed it once, with the encoder // blend-capability gate — re-running it here could drift, and would re-probe). diff --git a/crates/punktfunk-host/src/native/handshake.rs b/crates/punktfunk-host/src/native/handshake.rs index 18a76d5d..f277dffd 100644 --- a/crates/punktfunk-host/src/native/handshake.rs +++ b/crates/punktfunk-host/src/native/handshake.rs @@ -148,7 +148,6 @@ pub(super) async fn negotiate( Option, Option, )> { - let peer = conn.remote_address(); let mut hello = Hello::decode(first).map_err(|e| anyhow!("Hello decode: {e:?}"))?; if hello.abi_version != punktfunk_core::WIRE_VERSION { close_rejected( @@ -495,6 +494,11 @@ pub(super) async fn negotiate( let (data_sock, direct) = bind_data_socket(data_port)?; let udp_port = data_sock.local_addr()?.port(); + // The session's video geometry (see the `shard_payload` field below). Resolved before the + // Welcome struct because a path a previous session proved jumbo is given a bounded moment + // to re-prove itself live on THIS connection — the awaited part of `negotiated_shard_payload`. + let shard_payload = wire_mtu::negotiated_shard_payload(conn, hello.max_shard_payload).await; + let mut key = [0u8; 16]; rand::thread_rng().fill_bytes(&mut key); // Fresh per-session salt alongside the fresh key. GCM nonce uniqueness only *requires* one @@ -546,14 +550,15 @@ pub(super) async fn negotiate( // hardcoded 1452 overshot the v4 ceiling (its math forgot the header/crypto ride // inside the UDP payload) and silently IP-fragmented EVERY video datagram, doubling // per-datagram loss on Wi-Fi — the "100 Mbps badly fails on the phone" root cause. - // Negotiated, so the client follows. Jumbo (≈8900) is a future negotiated bump (needs - // MAX_DATAGRAM_BYTES raised + end-to-end 9000 MTU). - // Resolution order (wire_mtu.rs): `PUNKTFUNK_WIRE_MTU` operator override, then a path - // budget learned from a prior session whose QUIC MTU discovery settled below the - // video-datagram ceiling (the "VPN on the host blackholes every video packet" field - // shape — small flows pass, the stream is an endless black screen), then this family - // default. Healthy paths take the default branch and are byte-identical to before. - shard_payload: wire_mtu::negotiated_shard_payload(peer.ip()) as u16, + // Negotiated, so the client follows. + // Resolution order (wire_mtu.rs): a JUMBO start (≈8900) on a path a previous session + // proved AND this connection has just re-proved live, then the `PUNKTFUNK_WIRE_MTU` + // operator override, then a path budget learned from a prior session whose QUIC MTU + // discovery settled below the video-datagram ceiling (the "VPN on the host blackholes + // every video packet" field shape — small flows pass, the stream is an endless black + // screen), then this family default. Healthy paths take the default branch and are + // byte-identical to before. + shard_payload: shard_payload as u16, encrypt: true, key, salt, diff --git a/crates/punktfunk-host/src/native/wire_mtu.rs b/crates/punktfunk-host/src/native/wire_mtu.rs index 408e19b7..9b963b4f 100644 --- a/crates/punktfunk-host/src/native/wire_mtu.rs +++ b/crates/punktfunk-host/src/native/wire_mtu.rs @@ -24,6 +24,14 @@ //! - **Heal** — the next handshake from that peer clamps `shard_payload` to the recorded //! budget, so a reconnect fixes the stream. A later session that reaches the ceiling erases //! the record (the learn/heal loop is self-correcting in both directions). +//! - **Grow** (PW7a) — the mirror image, for the jumbo half: a connection whose discovery +//! settles at the sealed JUMBO size has proven the path carries ~8.9 KB video datagrams, and +//! the next session on that same path *starts* there instead of at the 1500-byte default. +//! PyroWave sessions cannot be re-keyed mid-stream (the client's parse window is the +//! `Welcome` value, read once over the C ABI), so the session-start value is the ONLY way +//! they ever reach jumbo — and it is exactly where ~6× fewer datagrams per frame is worth +//! the most. See [`jumbo_session_start`] for why a remembered verdict alone is never +//! allowed to seal one byte above the default. use std::collections::HashMap; use std::net::IpAddr; @@ -63,10 +71,155 @@ fn learned() -> &'static Mutex> { LEARNED.get_or_init(|| Mutex::new(HashMap::new())) } -/// The shard payload for a new session to `peer`: `PUNKTFUNK_WIRE_MTU` override, else the -/// peer's learned path budget, else the family default (today's exact behavior). Logs whenever -/// the result differs from the default. -pub(super) fn negotiated_shard_payload(peer: IpAddr) -> usize { +/// Identity of a PATH, not of a peer — the key the jumbo verdict is filed under. +/// +/// The clamp above is keyed by peer IP alone, and that is safe *because being wrong is benign*: +/// a stale clamp only makes video datagrams smaller than they had to be. A stale GROW is the +/// opposite — one oversized datagram on a 1500-byte path is silently dropped, which is the +/// "connects fine, black screen forever" shape this whole module exists to kill. So the grow +/// keys strictly: a verdict earned over the host's 10 GbE NIC does not apply to the same peer +/// IP reached over the host's Wi-Fi or a VPN adapter, because those are different routes with +/// different MTUs. +/// +/// `local` is `Connection::local_ip()` (the address the connection was actually received on); +/// `None` where the platform can't report it, which degrades this key to the clamp's — safely, +/// because the live re-proof in [`jumbo_session_start`] is what actually protects the grow. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +struct PathKey { + local: Option, + peer: IpAddr, +} + +/// A path that a completed MTU-discovery search proved carries jumbo video datagrams. +#[derive(Clone, Copy, Debug)] +struct JumboVerdict { + /// The settled UDP-payload budget the proof measured. + udp_budget: u16, + /// The operator's jumbo target when the proof was taken. A changed `PUNKTFUNK_JUMBO` / + /// `PUNKTFUNK_WIRE_MTU` invalidates it rather than being silently reinterpreted. + target_wire_mtu: usize, + /// When it was taken ([`JUMBO_VERDICT_TTL`]). + at: std::time::Instant, +} + +/// How long a jumbo verdict may be redeemed for. Contrary evidence erases it long before this +/// (any settle below the sealed target, on any later session over the same path — the same +/// self-correction the clamp has), so the TTL is not the safety mechanism; it is a bound on how +/// stale an *unrefreshed* memory can get, for the case where the path changes while no session +/// is running. +const JUMBO_VERDICT_TTL: std::time::Duration = std::time::Duration::from_secs(6 * 3600); + +/// How long the `Welcome` may wait for THIS connection's MTU discovery to re-prove a jumbo +/// path. +/// +/// The wait is structural, not laziness: every connection restarts discovery from ~1200 bytes, +/// so the live proof the grow requires does not exist yet when the `Welcome` is built — and the +/// binary search up to sealed-jumbo needs an ACKED probe per step, each of which a peer may sit +/// on for its ack delay. Without a wait the gate would never pass and the feature would be dead. +/// +/// It is honestly on the bring-up critical path (`handshake.rs` sends the `Welcome` and only +/// THEN kicks the display prep), so it is bounded, returns the instant the proof lands, and is +/// entered ONLY for a path a previous session already proved jumbo — i.e. an opted-in operator +/// on a jumbo LAN, never anyone else. The worst case (the full wait, no proof) is the moved +/// laptop, and it is self-limiting: that session's watcher erases the verdict, so the next +/// connect doesn't wait at all. +const JUMBO_PROOF_WAIT: std::time::Duration = std::time::Duration::from_millis(300); +const JUMBO_PROOF_POLL: std::time::Duration = std::time::Duration::from_millis(10); + +/// Proven-jumbo paths. Same lifetime rules as [`learned`] — in-memory, re-earned in one session +/// after a host restart. +fn jumbo_verdicts() -> &'static Mutex> { + static JUMBO: OnceLock>> = OnceLock::new(); + JUMBO.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn path_key(conn: &quinn::Connection) -> PathKey { + PathKey { + local: conn.local_ip(), + peer: conn.remote_address().ip(), + } +} + +/// Everything the session-start jumbo decision reads. Every field but `proven_udp_budget` is +/// observed on THIS connection during THIS handshake — which is the point (see +/// [`jumbo_session_start`]). +#[derive(Clone, Copy, Debug)] +struct JumboStart { + /// The host operator's opt-in ([`jumbo_wire_mtu`]) — `None` = no jumbo, ever. + target_wire_mtu: Option, + /// `Hello::max_shard_payload`: the client's own receive ceiling (0 = legacy client, which + /// never gets a geometry it didn't ask for). + client_ceiling: u16, + /// `conn.stats().path.current_mtu` right now: the largest UDP payload quinn has had ACKED + /// on this connection. + live_udp_mtu: u16, + /// What a previous session over this same [`PathKey`] settled at, if any. + proven_udp_budget: Option, + /// The constrained-path clamp [`learned`] for this peer, if any. Contradictory evidence + /// (this peer black-screened on a small MTU recently) vetoes the grow — the two memories + /// are keyed differently and the safe one wins. + clamped_udp_budget: Option, +} + +/// The jumbo shard payload a session to `peer` could use, or `None` when there is nothing to +/// gain (no opt-in, a legacy/low client ceiling, or a target that isn't bigger than the family +/// default). Shared by the decision, the wait, and the watcher so all three agree on the number. +fn jumbo_target( + target_wire_mtu: Option, + client_ceiling: u16, + peer: IpAddr, +) -> Option { + let mtu = target_wire_mtu?; + let t = jumbo_shard_payload_for(mtu, peer).min(client_ceiling as usize); + let t = t - t % 2; // FEC requires even shards + (t > mtu1500_shard_payload_for(peer)).then_some(t) +} + +/// The session-START jumbo decision: `Some(shard_payload)` only when every gate below holds. +/// +/// **Why a remembered verdict is never enough.** A laptop that proved jumbo on the wired LAN +/// and comes back on Wi-Fi, a switch that lost its jumbo config, a client IP recycled by DHCP — +/// all of them present a path that cannot carry an 8.9 KB datagram, and a PyroWave session +/// sealed at that size cannot be re-keyed mid-stream, so it would black-screen for its whole +/// life. The memory therefore only decides whether it is worth WAITING for a proof; what +/// actually authorises the grow is `live_udp_mtu` — a datagram of exactly that size, acked by +/// this client, on this connection, seconds ago. That is why this is as safe as the clamp +/// despite the failure modes being opposite: a wrong memory cannot produce a jumbo `Welcome`, +/// only a live measurement can. +/// +/// The gates, in order: the host operator opted in; the client advertised enough receive +/// headroom; the target beats the family default (nothing to gain otherwise); no constrained-path +/// clamp contradicts it; a prior session over this exact path settled at or above the sealed +/// target; and this connection has re-proven it live. +fn jumbo_session_start(i: JumboStart, peer: IpAddr) -> Option { + let target = jumbo_target(i.target_wire_mtu, i.client_ceiling, peer)?; + let sealed = sealed_datagram_bytes(target); + if let Some(clamp) = i.clamped_udp_budget { + if (clamp as usize) < sealed { + return None; + } + } + if (i.proven_udp_budget? as usize) < sealed { + return None; + } + if (i.live_udp_mtu as usize) < sealed { + return None; + } + Some(target) +} + +/// The shard payload for a new session on `conn`: a proven-jumbo grow, else the +/// `PUNKTFUNK_WIRE_MTU` override, else the peer's learned path budget, else the family default +/// (today's exact behavior). Logs whenever the result differs from the default. +/// +/// `client_ceiling` is the client's `Hello::max_shard_payload`. Async only for the bounded +/// [`JUMBO_PROOF_WAIT`], which is entered *only* on a path a previous session already proved +/// jumbo — every other session resolves without awaiting anything. +pub(super) async fn negotiated_shard_payload( + conn: &quinn::Connection, + client_ceiling: u16, +) -> usize { + let peer = conn.remote_address().ip(); let env = match std::env::var("PUNKTFUNK_WIRE_MTU") { Ok(v) => match v.trim().parse::() { Ok(mtu) => Some(mtu), @@ -78,13 +231,80 @@ pub(super) fn negotiated_shard_payload(peer: IpAddr) -> usize { Err(_) => None, }; let learned_budget = learned().lock().unwrap().get(&peer).copied(); - resolve(env, learned_budget, peer) + let target_wire_mtu = jumbo_wire_mtu(); + let proven_udp_budget = fresh_verdict(path_key(conn), target_wire_mtu); + let mut jumbo = JumboStart { + target_wire_mtu, + client_ceiling, + live_udp_mtu: conn.stats().path.current_mtu, + proven_udp_budget, + clamped_udp_budget: learned_budget, + }; + // A proven path is worth waiting a moment for: MTU discovery starts when the handshake + // completes and needs an acked probe per binary-search step, so at `Welcome` time it may + // simply not have got there yet. Bounded, and only on paths that already proved it once. + let awaited_proof = proven_udp_budget + .and_then(|_| jumbo_target(target_wire_mtu, client_ceiling, peer)) + .map(|t| sealed_datagram_bytes(t) as u16); + if let Some(sealed) = awaited_proof { + if jumbo.live_udp_mtu < sealed { + let t0 = std::time::Instant::now(); + while t0.elapsed() < JUMBO_PROOF_WAIT { + tokio::time::sleep(JUMBO_PROOF_POLL).await; + jumbo.live_udp_mtu = conn.stats().path.current_mtu; + if jumbo.live_udp_mtu >= sealed { + break; + } + } + tracing::debug!( + peer = %peer, + waited_ms = t0.elapsed().as_millis() as u64, + live_udp_mtu = jumbo.live_udp_mtu, + needed = sealed, + "wire MTU: waited for this connection to re-prove its jumbo path" + ); + } + } + resolve(env, learned_budget, jumbo, peer) } -/// Pure resolution (env override > learned budget > family default) — the tested core of -/// [`negotiated_shard_payload`]. -fn resolve(env_wire_mtu: Option, learned_udp_budget: Option, peer: IpAddr) -> usize { +/// The peer's jumbo verdict if it is still redeemable: same operator target, inside the TTL. +/// A verdict that fails either test is dropped on the spot rather than left to rot. +fn fresh_verdict(key: PathKey, target_wire_mtu: Option) -> Option { + let target = target_wire_mtu?; + let mut map = jumbo_verdicts().lock().unwrap(); + let v = *map.get(&key)?; + if v.target_wire_mtu != target || v.at.elapsed() > JUMBO_VERDICT_TTL { + map.remove(&key); + return None; + } + Some(v.udp_budget) +} + +/// Pure resolution (proven jumbo > env override > learned budget > family default) — the tested +/// core of [`negotiated_shard_payload`]. +fn resolve( + env_wire_mtu: Option, + learned_udp_budget: Option, + jumbo: JumboStart, + peer: IpAddr, +) -> usize { let default = mtu1500_shard_payload_for(peer); + // First, because the two are mutually exclusive by construction: `jumbo_wire_mtu()` only + // fires above 1500, and the env branch below CLAMPS to the family default, so a + // `PUNKTFUNK_WIRE_MTU=9000` operator would otherwise get 1408 and never a jumbo start. + if let Some(p) = jumbo_session_start(jumbo, peer) { + tracing::info!( + peer = %peer, + shard_payload = p, + default, + live_udp_mtu = jumbo.live_udp_mtu, + proven_udp_budget = jumbo.proven_udp_budget, + "wire MTU: session starts at the JUMBO shard — this path proved it in a previous \ + session AND re-proved it live on this connection (~6× fewer datagrams per frame)" + ); + return p; + } if let Some(mtu) = env_wire_mtu { let p = shard_payload_for_wire_mtu(mtu, peer); if p != default { @@ -119,34 +339,73 @@ fn resolve(env_wire_mtu: Option, learned_udp_budget: Option, peer: I /// into a verdict — and, with a [`ShardReneg`] driver, act on it MID-SESSION /// (design/shard-payload-reneg.md Phase 2): a below-ceiling verdict shrinks the live wire at /// the ~3–10 s mark (session 1 heals instead of staying black), and a settled-at-jumbo -/// verdict grows it, ack-gated, when the operator opted in. Spawned once per negotiated -/// session; without a grow the task ends after the final sample (bounded ~10 s lifetime, -/// holding only a cheap `Connection` handle) — after a grow it stays as the revert guard -/// until the connection closes. +/// verdict grows it, ack-gated, when the operator opted in. The same settled-at-jumbo reading +/// also writes this path's next-session verdict (PW7a) — `client_ceiling` is the client's +/// `Hello::max_shard_payload`, which decides what "jumbo" is worth proving for this peer. +/// Spawned once per negotiated session; without a grow the task ends after the final sample +/// (bounded ~10 s lifetime, holding only a cheap `Connection` handle) — after a grow, or on a +/// session that STARTED jumbo, it stays as the revert guard until the connection closes. pub(super) fn spawn_watch( conn: quinn::Connection, session_shard_payload: usize, + client_ceiling: u16, reneg: Option, ) { tokio::spawn(async move { let peer = conn.remote_address().ip(); let ceiling = video_datagram_udp_ceiling() as u16; + // The sealed size a JUMBO proof has to reach on this path (PW7a) — `None` unless the + // operator opted in AND this client advertised the headroom. Read once: the verdict + // records the target it was proven under, and the two must be the same number. + let target_wire_mtu = jumbo_wire_mtu(); + let jumbo_proof = + jumbo_target(target_wire_mtu, client_ceiling, peer).map(sealed_datagram_bytes); // Discovery finishes in a handful of RTTs on a LAN (well under the first sample) but // needs a loss timeout per failed probe on a constrained path — the second sample // covers that with margin. Max, because discovery only ever raises `current_mtu` // (the post-grow revert guard below re-reads it live, where blackhole detection CAN - // lower it again). + // lower it again). Stop early only once nothing more is expected: with a jumbo opt-in + // the search keeps climbing past the 1500-byte ceiling, and stopping there would throw + // away the very measurement the proof needs. + let goal = jumbo_proof + .unwrap_or(ceiling as usize) + .max(ceiling as usize) as u16; let mut settled = 0u16; for wait_s in [3u64, 7] { tokio::time::sleep(std::time::Duration::from_secs(wait_s)).await; settled = settled.max(conn.stats().path.current_mtu); - if settled >= ceiling { + if settled >= goal { break; } } // The wire this session is CURRENTLY sealed at — moves on a mid-session shrink/grow. let mut current = session_shard_payload; let mut reneg = reneg; + // PW7a bookkeeping, before anything else can return: this is where a jumbo path earns + // its next-session verdict — and, far more importantly, where it LOSES it. Recording + // needs a live connection that reached the sealed target; anything else (a lower + // settle, a connection that died before the window closed, i.e. exactly what a client + // staring at a black screen does) erases, so the next session falls back to the + // 1500-byte default and has to prove itself again from scratch. + if let Some(need) = jumbo_proof { + let key = path_key(&conn); + if settled as usize >= need && conn.close_reason().is_none() { + jumbo_verdicts().lock().unwrap().insert( + key, + JumboVerdict { + udp_budget: settled, + target_wire_mtu: target_wire_mtu.unwrap_or_default(), + at: std::time::Instant::now(), + }, + ); + tracing::info!(peer = %peer, discovered_udp_mtu = settled, needed = need, + "wire MTU: this path carries JUMBO video datagrams — the next session over \ + it starts at the big shard (it still has to re-prove the path live)"); + } else if jumbo_verdicts().lock().unwrap().remove(&key).is_some() { + tracing::info!(peer = %peer, discovered_udp_mtu = settled, needed = need, + "wire MTU: jumbo verdict cleared — this path no longer proves it"); + } + } if settled >= ceiling { // The path carries full-size video datagrams — erase any stale learned clamp so // the next session returns to the default wire. @@ -154,6 +413,34 @@ pub(super) fn spawn_watch( tracing::info!(peer = %peer, "wire MTU: path re-measured at full size — learned clamp cleared"); } + // …but "full size" is the 1500-byte ceiling, and this session may have STARTED + // above it (a PW7a jumbo start whose path changed since the proof, or a client + // that roamed onto a 1500-MTU link). Then every video datagram is dying right now. + // The verdict is already erased above; heal the live wire if this session can be + // re-keyed at all — a PyroWave client cannot (its parse window is the `Welcome` + // value), so for those the WARN plus a corrected next session is all there is. + if sealed_datagram_bytes(current) > settled as usize { + tracing::warn!( + peer = %peer, + discovered_udp_mtu = settled, + shard_payload = current, + "wire MTU: this session started at a JUMBO shard but the path does not \ + carry it — video datagrams are oversized for a hop, which streams as a \ + black screen with zero reported loss. The jumbo verdict for this path is \ + cleared: the next connect starts at the standard 1500-byte wire." + ); + if let Some(r) = reneg.as_ref() { + let back = shard_payload_for_udp_budget(settled as usize, peer); + if back < current + && r.change_tx.send(back as u16).is_ok() + && r.apply_tx.send(back).is_ok() + { + tracing::info!(peer = %peer, shard_payload = back, was = current, + "wire MTU: video re-keyed mid-session back to the standard wire"); + current = back; + } + } + } } else { // A closed connection stops discovering, so a session that ended before the final // sample proves nothing (a healthy high-RTT path could still be mid-search): learn @@ -203,12 +490,41 @@ pub(super) fn spawn_watch( } } } + // PW7a revert guard for a session that STARTED jumbo and has no re-key channel (the + // PyroWave case, and the only reason the session-start grow exists). Nothing can save + // this session if the path stops fitting mid-stream — but the NEXT one must not repeat + // it, so keep sampling and drop the verdict the moment quinn's blackhole detection or + // a re-search says the path shrank. Cheap: one `Connection` handle, one sample per 5 s. + // Only for a session that is currently FITTING — one that already failed the check + // above has been warned about and had its verdict erased there. + if current > mtu1500_shard_payload_for(peer) + && reneg.is_none() + && sealed_datagram_bytes(current) <= settled as usize + { + loop { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + if conn.close_reason().is_some() { + return; + } + let mtu_now = conn.stats().path.current_mtu; + if (mtu_now as usize) < sealed_datagram_bytes(current) { + jumbo_verdicts().lock().unwrap().remove(&path_key(&conn)); + tracing::warn!(peer = %peer, discovered_udp_mtu = mtu_now, + shard_payload = current, + "wire MTU: the jumbo path this session started on stopped fitting — this \ + session cannot be re-keyed (chunk-aligned client parse window), so it \ + will not recover, but the verdict is cleared and the next connect \ + starts at the standard wire"); + return; + } + } + } // Phase 2 up-leg: jumbo grow — operator opt-in (PUNKTFUNK_JUMBO / PUNKTFUNK_WIRE_MTU // > 1500, which also raised the endpoint's probe ceiling so `settled` can even reach // here), client-advertised headroom, and a settled-at-jumbo proof. The grow is // ACK-GATED: not one sealed datagram above the old size leaves before the client's // ack, even though its buffers are statically sized — the rule must not erode. - let (Some(mtu), Some(r)) = (jumbo_wire_mtu(), reneg.as_mut()) else { + let (Some(mtu), Some(r)) = (target_wire_mtu, reneg.as_mut()) else { return; }; let target = jumbo_shard_payload_for(mtu, peer).min(r.client_ceiling as usize); @@ -275,34 +591,196 @@ mod tests { const V4: IpAddr = IpAddr::V4(Ipv4Addr::new(192, 168, 1, 2)); const V6: IpAddr = IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1)); + /// No jumbo anywhere — what every session that isn't on an opted-in jumbo LAN passes. + const NO_JUMBO: JumboStart = JumboStart { + target_wire_mtu: None, + client_ceiling: 0, + live_udp_mtu: 0, + proven_udp_budget: None, + clamped_udp_budget: None, + }; + /// A 9000-MTU LAN, a modern client, a path proven last session and re-proven live now. + fn proven_jumbo() -> JumboStart { + JumboStart { + target_wire_mtu: Some(9000), + client_ceiling: punktfunk_core::config::max_shard_payload() as u16, + live_udp_mtu: 8972, + proven_udp_budget: Some(8972), + clamped_udp_budget: None, + } + } #[test] fn default_when_nothing_known() { - assert_eq!(resolve(None, None, V4), mtu1500_shard_payload_for(V4)); - assert_eq!(resolve(None, None, V6), mtu1500_shard_payload_for(V6)); + assert_eq!( + resolve(None, None, NO_JUMBO, V4), + mtu1500_shard_payload_for(V4) + ); + assert_eq!( + resolve(None, None, NO_JUMBO, V6), + mtu1500_shard_payload_for(V6) + ); } #[test] fn env_override_beats_learned() { // 1280 wire − 28 IP/UDP − 64 header/crypto = 1188. - assert_eq!(resolve(Some(1280), Some(1472), V4), 1188); + assert_eq!(resolve(Some(1280), Some(1472), NO_JUMBO, V4), 1188); } #[test] fn learned_budget_clamps() { // A WARP-shaped path: 1280-byte UDP budget → 1280 − 64 = 1216. - assert_eq!(resolve(None, Some(1280), V4), 1216); + assert_eq!(resolve(None, Some(1280), NO_JUMBO, V4), 1216); } #[test] fn learned_at_or_above_ceiling_is_the_default_wire() { - assert_eq!(resolve(None, Some(1472), V4), mtu1500_shard_payload_for(V4)); - assert_eq!(resolve(None, Some(2000), V4), mtu1500_shard_payload_for(V4)); + assert_eq!( + resolve(None, Some(1472), NO_JUMBO, V4), + mtu1500_shard_payload_for(V4) + ); + assert_eq!( + resolve(None, Some(2000), NO_JUMBO, V4), + mtu1500_shard_payload_for(V4) + ); } #[test] fn env_full_mtu_is_the_default_wire_both_families() { - assert_eq!(resolve(Some(1500), None, V4), mtu1500_shard_payload_for(V4)); - assert_eq!(resolve(Some(1500), None, V6), mtu1500_shard_payload_for(V6)); + assert_eq!( + resolve(Some(1500), None, NO_JUMBO, V4), + mtu1500_shard_payload_for(V4) + ); + assert_eq!( + resolve(Some(1500), None, NO_JUMBO, V6), + mtu1500_shard_payload_for(V6) + ); + } + + /// The happy path, both families: 9000 − 28 (IPv4) − 64 = 8908, and 9000 − 48 − 64 = 8888. + #[test] + fn proven_and_reproven_path_starts_jumbo() { + assert_eq!(jumbo_session_start(proven_jumbo(), V4), Some(8908)); + let mut v6 = proven_jumbo(); + v6.live_udp_mtu = 8952; + v6.proven_udp_budget = Some(8952); + assert_eq!(jumbo_session_start(v6, V6), Some(8888)); + // …and it is what `resolve` returns, ahead of the env branch that would clamp a + // >1500 `PUNKTFUNK_WIRE_MTU` back down to the family default. + assert_eq!(resolve(Some(9000), None, proven_jumbo(), V4), 8908); + } + + /// THE guard: the laptop that proved jumbo on the wired LAN and came back on a 1500-MTU + /// link. The memory still says jumbo; the live connection says otherwise; the live one + /// wins, every time. This is what makes the grow as safe as the clamp. + #[test] + fn a_remembered_verdict_never_grows_without_a_live_reproof() { + let mut moved = proven_jumbo(); + moved.live_udp_mtu = 1472; // a clean 1500-MTU path, freshly measured + assert_eq!(jumbo_session_start(moved, V4), None); + assert_eq!( + resolve(None, None, moved, V4), + mtu1500_shard_payload_for(V4) + ); + // Not even one byte of headroom short of the sealed target is enough. + let mut nearly = proven_jumbo(); + nearly.live_udp_mtu = 8971; + assert_eq!(jumbo_session_start(nearly, V4), None); + } + + /// …and the mirror: a live-proven path with no prior verdict still starts at the default. + /// Both halves are required, so a single fluke on either side cannot seal a jumbo wire. + #[test] + fn a_live_proof_alone_does_not_grow() { + let mut first_ever = proven_jumbo(); + first_ever.proven_udp_budget = None; + assert_eq!(jumbo_session_start(first_ever, V4), None); + let mut weak_memory = proven_jumbo(); + weak_memory.proven_udp_budget = Some(1472); + assert_eq!(jumbo_session_start(weak_memory, V4), None); + } + + /// The two memories are keyed differently (clamp: peer; verdict: route), so they can + /// disagree. When they do, the one that keeps datagrams small wins. + #[test] + fn a_constrained_path_clamp_vetoes_the_grow() { + let mut contradicted = proven_jumbo(); + contradicted.clamped_udp_budget = Some(1280); + assert_eq!(jumbo_session_start(contradicted, V4), None); + // A clamp that is itself at or above the sealed target isn't contrary evidence. + let mut roomy = proven_jumbo(); + roomy.clamped_udp_budget = Some(8972); + assert_eq!(jumbo_session_start(roomy, V4), Some(8908)); + } + + #[test] + fn without_the_operator_opt_in_nothing_grows() { + let mut no_optin = proven_jumbo(); + no_optin.target_wire_mtu = None; + assert_eq!(jumbo_session_start(no_optin, V4), None); + } + + /// A legacy client (no `Hello::max_shard_payload`) is never handed a geometry it did not + /// advertise, and a client whose ceiling lands under the family default is left alone + /// rather than being "grown" to something smaller. + #[test] + fn the_client_ceiling_is_binding() { + let mut legacy = proven_jumbo(); + legacy.client_ceiling = 0; + assert_eq!(jumbo_session_start(legacy, V4), None); + let mut small = proven_jumbo(); + small.client_ceiling = 1408; + assert_eq!(jumbo_session_start(small, V4), None); + // A ceiling between the default and the path target caps the grow — and the proof + // then only has to cover the SMALLER sealed size. + let mut capped = proven_jumbo(); + capped.client_ceiling = 4000; + assert_eq!(jumbo_session_start(capped, V4), Some(4000)); + } + + /// Every shard payload the grow can produce is even (Leopard FEC splits shards in halves) + /// and fits the receive ceiling every client sizes its buffers from. + #[test] + fn grown_shards_stay_even_and_inside_the_receive_ceiling() { + for mtu in [2000usize, 4000, 4001, 9000, 9216, 64000] { + for peer in [V4, V6] { + let Some(t) = jumbo_target(Some(mtu), u16::MAX, peer) else { + continue; + }; + assert_eq!(t % 2, 0, "odd shard for mtu {mtu}"); + assert!(t <= punktfunk_core::config::max_shard_payload()); + assert!(t > mtu1500_shard_payload_for(peer)); + assert!( + sealed_datagram_bytes(t) <= punktfunk_core::packet::MAX_DATAGRAM_BYTES, + "sealed datagram overflows the receive ceiling at mtu {mtu}" + ); + } + } + // Below the family default there is nothing to grow to. + assert_eq!(jumbo_target(Some(1500), u16::MAX, V4), None); + assert_eq!(jumbo_target(None, u16::MAX, V4), None); + } + + /// A path is a (local interface, peer) pair, not a peer: the same client reached over the + /// host's other NIC is a different route with a different MTU. + #[test] + fn the_verdict_key_separates_routes_to_the_same_peer() { + let over_10g = PathKey { + local: Some(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))), + peer: V4, + }; + let over_wifi = PathKey { + local: Some(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1))), + peer: V4, + }; + assert_ne!(over_10g, over_wifi); + assert_ne!( + over_10g, + PathKey { + local: None, + peer: V4 + } + ); } }