diff --git a/crates/punktfunk-core/src/quic/endpoint.rs b/crates/punktfunk-core/src/quic/endpoint.rs index 43a786ff..fa810d86 100644 --- a/crates/punktfunk-core/src/quic/endpoint.rs +++ b/crates/punktfunk-core/src/quic/endpoint.rs @@ -82,6 +82,49 @@ fn stream_transport_idle(idle: std::time::Duration) -> Arc 1500) because it is not free: quinn sizes +/// its endpoint receive buffer as `max_udp_payload_size × max_receive_segments × BATCH_SIZE`, +/// which on a GRO-capable Linux/Android client is 64 × 32 segments — ~2.9 MiB at the 1472 +/// default, ~18 MiB at jumbo. A jumbo LAN is a deliberate deployment; every other client keeps +/// today's buffer to the byte. Without the opt-in this returns the stock config, so the +/// advertisement, the wire, and the memory are all unchanged. +fn endpoint_config() -> quinn::EndpointConfig { + let mut cfg = quinn::EndpointConfig::default(); + if let Some(mtu) = crate::config::jumbo_wire_mtu() { + // Derived exactly like the probe ceiling above (IPv4 overhead — a v6 peer's sealed + // target is smaller, so this covers it), and clamped into quinn's accepted range. + let shard = crate::config::jumbo_shard_payload_for( + mtu, + std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED), + ); + let accept = crate::config::sealed_datagram_bytes(shard).clamp(1200, 65_527) as u16; + if cfg.max_udp_payload_size(accept).is_ok() { + tracing::info!( + max_udp_payload_size = accept, + wire_mtu = mtu, + "jumbo opt-in: this endpoint advertises a jumbo QUIC receive ceiling, so the \ + peer's MTU discovery can prove a jumbo path (it is capped by this value)" + ); + } + } + cfg +} + /// Server endpoint with a fresh self-signed certificate (tests/dev — production hosts /// persist an identity and use [`server_with_identity`] so clients can pin it). pub fn server(addr: std::net::SocketAddr) -> anyhow_result::Result { @@ -238,7 +281,15 @@ pub fn client_pinned_with_identity( .map_err(|e| anyhow_result::Error::msg(format!("quic client config: {e}")))?; let mut client_cfg = quinn::ClientConfig::new(Arc::new(quic_cfg)); client_cfg.transport_config(stream_transport()); // keep-alive — see stream_transport - let mut ep = quinn::Endpoint::client("0.0.0.0:0".parse().unwrap())?; + + // `Endpoint::client` hardcodes `EndpointConfig::default()`, whose 1472-byte + // `max_udp_payload_size` caps the HOST's MTU discovery (see `endpoint_config`), so the + // endpoint is built by hand to carry the jumbo opt-in. Same bind as before + // (`0.0.0.0:0`, v4 — no dual-stack flag to reproduce) and the same default runtime. + let socket = std::net::UdpSocket::bind("0.0.0.0:0")?; + let runtime = quinn::default_runtime() + .ok_or_else(|| anyhow_result::Error::msg("no async runtime found".into()))?; + let mut ep = quinn::Endpoint::new(endpoint_config(), None, socket, runtime)?; ep.set_default_client_config(client_cfg); Ok(ep) })(); @@ -348,4 +399,80 @@ mod tests { let _ = super::stream_transport_idle(std::time::Duration::MAX); let _ = super::stream_transport_idle(std::time::Duration::ZERO); } + + /// Where a connection's MTU discovery is allowed to climb to, measured rather than argued + /// (PW7a). Loopback's own MTU is 64 KiB, so the ONLY thing that can stop the search here is + /// configuration — which makes this a clean instrument for the two ceilings: + /// + /// * **leg A** — server opted in, client NOT: the search stalls at the client's default + /// `max_udp_payload_size` advertisement (1472) no matter how high the server's probe + /// ceiling is. This is why the shipped jumbo grow could never fire: `wire_mtu.rs` waits + /// for a settle at the sealed jumbo size and the peer's transport parameter forbids it. + /// * **leg B** — both opted in: the search reaches the sealed jumbo datagram, and the + /// elapsed time is what the `Welcome`'s bounded proof-wait has to cover. + /// + /// `#[ignore]`d: it sets process-wide env (each endpoint reads the opt-in at construction, + /// which is exactly how the two legs are built) and spends seconds of wall clock. + /// Run it alone: `cargo test -p punktfunk-core --features quic mtu_discovery -- --ignored + /// --nocapture --test-threads=1`. + #[tokio::test] + #[ignore = "measurement: sets process env and takes ~15 s of wall clock"] + async fn mtu_discovery_climbs_only_as_high_as_the_peer_advertises() { + async fn climb(server_jumbo: bool, client_jumbo: bool) -> (u16, u128) { + let set = |on: bool| { + if on { + std::env::set_var("PUNKTFUNK_JUMBO", "1"); + } else { + std::env::remove_var("PUNKTFUNK_JUMBO"); + } + }; + set(server_jumbo); + let server = endpoint::server("127.0.0.1:0".parse().unwrap()).unwrap(); + let addr = server.local_addr().unwrap(); + set(client_jumbo); + let client = endpoint::client_insecure().unwrap(); + set(false); + let accept = tokio::spawn(async move { + let incoming = server.accept().await.expect("incoming"); + let conn = incoming.await.expect("host side connects"); + (server, conn) + }); + let client_conn = client.connect(addr, "punktfunk").unwrap().await.unwrap(); + let (_server_ep, host_conn) = accept.await.unwrap(); + // A stream write gives the driver something to transmit, which is what starts the + // search (probes ride `poll_transmit`); after that each probe's ack drives the next. + let mut s = host_conn.open_uni().await.unwrap(); + s.write_all(b"go").await.unwrap(); + let want = crate::config::sealed_datagram_bytes(crate::config::jumbo_shard_payload_for( + 9000, + std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED), + )) as u16; + let t0 = std::time::Instant::now(); + let mut mtu = host_conn.stats().path.current_mtu; + while t0.elapsed() < std::time::Duration::from_secs(6) && mtu < want { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + mtu = host_conn.stats().path.current_mtu; + } + let elapsed = t0.elapsed().as_millis(); + drop(client_conn); + drop(client); + (mtu, elapsed) + } + + let (capped, _) = climb(true, false).await; + println!("leg A (server opted in, client not): settled at {capped} B UDP payload"); + assert_eq!( + capped, 1472, + "a peer that advertises the stock max_udp_payload_size caps the search at 1472 — \ + the whole point of raising it on the client endpoint" + ); + + let (grown, ms) = climb(true, true).await; + println!("leg B (both opted in): reached {grown} B UDP payload in {ms} ms"); + assert!( + grown >= 8972, + "both sides opted in, loopback MTU is 64 KiB — discovery should reach the sealed \ + jumbo datagram, got {grown}" + ); + } } diff --git a/crates/punktfunk-host/src/native.rs b/crates/punktfunk-host/src/native.rs index 122de3f3..be01a1f9 100644 --- a/crates/punktfunk-host/src/native.rs +++ b/crates/punktfunk-host/src/native.rs @@ -1148,7 +1148,12 @@ async fn serve_session( // path verdict (WARN + learned clamp for the next session on a constrained path; clears // a stale clamp on a healthy one) — and, with the driver above, heal or grow THIS // session mid-stream. Bounded ~10 s task unless a jumbo grow leaves it as revert guard. - wire_mtu::spawn_watch(conn.clone(), welcome.shard_payload as usize, shard_reneg); + wire_mtu::spawn_watch( + conn.clone(), + welcome.shard_payload as usize, + hello.max_shard_payload, + shard_reneg, + ); // Negotiated cursor forwarding: the HOST_CAP_CURSOR bit the Welcome advertised, read back // rather than recomputed (`handshake::cursor_forward` computed it once, with the encoder // blend-capability gate — re-running it here could drift, and would re-probe). diff --git a/crates/punktfunk-host/src/native/handshake.rs b/crates/punktfunk-host/src/native/handshake.rs index 2eae0786..60afb95b 100644 --- a/crates/punktfunk-host/src/native/handshake.rs +++ b/crates/punktfunk-host/src/native/handshake.rs @@ -148,7 +148,6 @@ pub(super) async fn negotiate( Option, Option, )> { - let peer = conn.remote_address(); let mut hello = Hello::decode(first).map_err(|e| anyhow!("Hello decode: {e:?}"))?; if hello.abi_version != punktfunk_core::WIRE_VERSION { close_rejected( @@ -497,6 +496,11 @@ pub(super) async fn negotiate( let (data_sock, direct) = bind_data_socket(data_port)?; let udp_port = data_sock.local_addr()?.port(); + // The session's video geometry (see the `shard_payload` field below). Resolved before the + // Welcome struct because a path a previous session proved jumbo is given a bounded moment + // to re-prove itself live on THIS connection — the awaited part of `negotiated_shard_payload`. + let shard_payload = wire_mtu::negotiated_shard_payload(conn, hello.max_shard_payload).await; + let mut key = [0u8; 16]; rand::thread_rng().fill_bytes(&mut key); // Fresh per-session salt alongside the fresh key. GCM nonce uniqueness only *requires* one @@ -548,14 +552,15 @@ pub(super) async fn negotiate( // hardcoded 1452 overshot the v4 ceiling (its math forgot the header/crypto ride // inside the UDP payload) and silently IP-fragmented EVERY video datagram, doubling // per-datagram loss on Wi-Fi — the "100 Mbps badly fails on the phone" root cause. - // Negotiated, so the client follows. Jumbo (≈8900) is a future negotiated bump (needs - // MAX_DATAGRAM_BYTES raised + end-to-end 9000 MTU). - // Resolution order (wire_mtu.rs): `PUNKTFUNK_WIRE_MTU` operator override, then a path - // budget learned from a prior session whose QUIC MTU discovery settled below the - // video-datagram ceiling (the "VPN on the host blackholes every video packet" field - // shape — small flows pass, the stream is an endless black screen), then this family - // default. Healthy paths take the default branch and are byte-identical to before. - shard_payload: wire_mtu::negotiated_shard_payload(peer.ip()) as u16, + // Negotiated, so the client follows. + // Resolution order (wire_mtu.rs): a JUMBO start (≈8900) on a path a previous session + // proved AND this connection has just re-proved live, then the `PUNKTFUNK_WIRE_MTU` + // operator override, then a path budget learned from a prior session whose QUIC MTU + // discovery settled below the video-datagram ceiling (the "VPN on the host blackholes + // every video packet" field shape — small flows pass, the stream is an endless black + // screen), then this family default. Healthy paths take the default branch and are + // byte-identical to before. + shard_payload: shard_payload as u16, encrypt: true, key, salt, diff --git a/crates/punktfunk-host/src/native/wire_mtu.rs b/crates/punktfunk-host/src/native/wire_mtu.rs index 408e19b7..9b963b4f 100644 --- a/crates/punktfunk-host/src/native/wire_mtu.rs +++ b/crates/punktfunk-host/src/native/wire_mtu.rs @@ -24,6 +24,14 @@ //! - **Heal** — the next handshake from that peer clamps `shard_payload` to the recorded //! budget, so a reconnect fixes the stream. A later session that reaches the ceiling erases //! the record (the learn/heal loop is self-correcting in both directions). +//! - **Grow** (PW7a) — the mirror image, for the jumbo half: a connection whose discovery +//! settles at the sealed JUMBO size has proven the path carries ~8.9 KB video datagrams, and +//! the next session on that same path *starts* there instead of at the 1500-byte default. +//! PyroWave sessions cannot be re-keyed mid-stream (the client's parse window is the +//! `Welcome` value, read once over the C ABI), so the session-start value is the ONLY way +//! they ever reach jumbo — and it is exactly where ~6× fewer datagrams per frame is worth +//! the most. See [`jumbo_session_start`] for why a remembered verdict alone is never +//! allowed to seal one byte above the default. use std::collections::HashMap; use std::net::IpAddr; @@ -63,10 +71,155 @@ fn learned() -> &'static Mutex> { LEARNED.get_or_init(|| Mutex::new(HashMap::new())) } -/// The shard payload for a new session to `peer`: `PUNKTFUNK_WIRE_MTU` override, else the -/// peer's learned path budget, else the family default (today's exact behavior). Logs whenever -/// the result differs from the default. -pub(super) fn negotiated_shard_payload(peer: IpAddr) -> usize { +/// Identity of a PATH, not of a peer — the key the jumbo verdict is filed under. +/// +/// The clamp above is keyed by peer IP alone, and that is safe *because being wrong is benign*: +/// a stale clamp only makes video datagrams smaller than they had to be. A stale GROW is the +/// opposite — one oversized datagram on a 1500-byte path is silently dropped, which is the +/// "connects fine, black screen forever" shape this whole module exists to kill. So the grow +/// keys strictly: a verdict earned over the host's 10 GbE NIC does not apply to the same peer +/// IP reached over the host's Wi-Fi or a VPN adapter, because those are different routes with +/// different MTUs. +/// +/// `local` is `Connection::local_ip()` (the address the connection was actually received on); +/// `None` where the platform can't report it, which degrades this key to the clamp's — safely, +/// because the live re-proof in [`jumbo_session_start`] is what actually protects the grow. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +struct PathKey { + local: Option, + peer: IpAddr, +} + +/// A path that a completed MTU-discovery search proved carries jumbo video datagrams. +#[derive(Clone, Copy, Debug)] +struct JumboVerdict { + /// The settled UDP-payload budget the proof measured. + udp_budget: u16, + /// The operator's jumbo target when the proof was taken. A changed `PUNKTFUNK_JUMBO` / + /// `PUNKTFUNK_WIRE_MTU` invalidates it rather than being silently reinterpreted. + target_wire_mtu: usize, + /// When it was taken ([`JUMBO_VERDICT_TTL`]). + at: std::time::Instant, +} + +/// How long a jumbo verdict may be redeemed for. Contrary evidence erases it long before this +/// (any settle below the sealed target, on any later session over the same path — the same +/// self-correction the clamp has), so the TTL is not the safety mechanism; it is a bound on how +/// stale an *unrefreshed* memory can get, for the case where the path changes while no session +/// is running. +const JUMBO_VERDICT_TTL: std::time::Duration = std::time::Duration::from_secs(6 * 3600); + +/// How long the `Welcome` may wait for THIS connection's MTU discovery to re-prove a jumbo +/// path. +/// +/// The wait is structural, not laziness: every connection restarts discovery from ~1200 bytes, +/// so the live proof the grow requires does not exist yet when the `Welcome` is built — and the +/// binary search up to sealed-jumbo needs an ACKED probe per step, each of which a peer may sit +/// on for its ack delay. Without a wait the gate would never pass and the feature would be dead. +/// +/// It is honestly on the bring-up critical path (`handshake.rs` sends the `Welcome` and only +/// THEN kicks the display prep), so it is bounded, returns the instant the proof lands, and is +/// entered ONLY for a path a previous session already proved jumbo — i.e. an opted-in operator +/// on a jumbo LAN, never anyone else. The worst case (the full wait, no proof) is the moved +/// laptop, and it is self-limiting: that session's watcher erases the verdict, so the next +/// connect doesn't wait at all. +const JUMBO_PROOF_WAIT: std::time::Duration = std::time::Duration::from_millis(300); +const JUMBO_PROOF_POLL: std::time::Duration = std::time::Duration::from_millis(10); + +/// Proven-jumbo paths. Same lifetime rules as [`learned`] — in-memory, re-earned in one session +/// after a host restart. +fn jumbo_verdicts() -> &'static Mutex> { + static JUMBO: OnceLock>> = OnceLock::new(); + JUMBO.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn path_key(conn: &quinn::Connection) -> PathKey { + PathKey { + local: conn.local_ip(), + peer: conn.remote_address().ip(), + } +} + +/// Everything the session-start jumbo decision reads. Every field but `proven_udp_budget` is +/// observed on THIS connection during THIS handshake — which is the point (see +/// [`jumbo_session_start`]). +#[derive(Clone, Copy, Debug)] +struct JumboStart { + /// The host operator's opt-in ([`jumbo_wire_mtu`]) — `None` = no jumbo, ever. + target_wire_mtu: Option, + /// `Hello::max_shard_payload`: the client's own receive ceiling (0 = legacy client, which + /// never gets a geometry it didn't ask for). + client_ceiling: u16, + /// `conn.stats().path.current_mtu` right now: the largest UDP payload quinn has had ACKED + /// on this connection. + live_udp_mtu: u16, + /// What a previous session over this same [`PathKey`] settled at, if any. + proven_udp_budget: Option, + /// The constrained-path clamp [`learned`] for this peer, if any. Contradictory evidence + /// (this peer black-screened on a small MTU recently) vetoes the grow — the two memories + /// are keyed differently and the safe one wins. + clamped_udp_budget: Option, +} + +/// The jumbo shard payload a session to `peer` could use, or `None` when there is nothing to +/// gain (no opt-in, a legacy/low client ceiling, or a target that isn't bigger than the family +/// default). Shared by the decision, the wait, and the watcher so all three agree on the number. +fn jumbo_target( + target_wire_mtu: Option, + client_ceiling: u16, + peer: IpAddr, +) -> Option { + let mtu = target_wire_mtu?; + let t = jumbo_shard_payload_for(mtu, peer).min(client_ceiling as usize); + let t = t - t % 2; // FEC requires even shards + (t > mtu1500_shard_payload_for(peer)).then_some(t) +} + +/// The session-START jumbo decision: `Some(shard_payload)` only when every gate below holds. +/// +/// **Why a remembered verdict is never enough.** A laptop that proved jumbo on the wired LAN +/// and comes back on Wi-Fi, a switch that lost its jumbo config, a client IP recycled by DHCP — +/// all of them present a path that cannot carry an 8.9 KB datagram, and a PyroWave session +/// sealed at that size cannot be re-keyed mid-stream, so it would black-screen for its whole +/// life. The memory therefore only decides whether it is worth WAITING for a proof; what +/// actually authorises the grow is `live_udp_mtu` — a datagram of exactly that size, acked by +/// this client, on this connection, seconds ago. That is why this is as safe as the clamp +/// despite the failure modes being opposite: a wrong memory cannot produce a jumbo `Welcome`, +/// only a live measurement can. +/// +/// The gates, in order: the host operator opted in; the client advertised enough receive +/// headroom; the target beats the family default (nothing to gain otherwise); no constrained-path +/// clamp contradicts it; a prior session over this exact path settled at or above the sealed +/// target; and this connection has re-proven it live. +fn jumbo_session_start(i: JumboStart, peer: IpAddr) -> Option { + let target = jumbo_target(i.target_wire_mtu, i.client_ceiling, peer)?; + let sealed = sealed_datagram_bytes(target); + if let Some(clamp) = i.clamped_udp_budget { + if (clamp as usize) < sealed { + return None; + } + } + if (i.proven_udp_budget? as usize) < sealed { + return None; + } + if (i.live_udp_mtu as usize) < sealed { + return None; + } + Some(target) +} + +/// The shard payload for a new session on `conn`: a proven-jumbo grow, else the +/// `PUNKTFUNK_WIRE_MTU` override, else the peer's learned path budget, else the family default +/// (today's exact behavior). Logs whenever the result differs from the default. +/// +/// `client_ceiling` is the client's `Hello::max_shard_payload`. Async only for the bounded +/// [`JUMBO_PROOF_WAIT`], which is entered *only* on a path a previous session already proved +/// jumbo — every other session resolves without awaiting anything. +pub(super) async fn negotiated_shard_payload( + conn: &quinn::Connection, + client_ceiling: u16, +) -> usize { + let peer = conn.remote_address().ip(); let env = match std::env::var("PUNKTFUNK_WIRE_MTU") { Ok(v) => match v.trim().parse::() { Ok(mtu) => Some(mtu), @@ -78,13 +231,80 @@ pub(super) fn negotiated_shard_payload(peer: IpAddr) -> usize { Err(_) => None, }; let learned_budget = learned().lock().unwrap().get(&peer).copied(); - resolve(env, learned_budget, peer) + let target_wire_mtu = jumbo_wire_mtu(); + let proven_udp_budget = fresh_verdict(path_key(conn), target_wire_mtu); + let mut jumbo = JumboStart { + target_wire_mtu, + client_ceiling, + live_udp_mtu: conn.stats().path.current_mtu, + proven_udp_budget, + clamped_udp_budget: learned_budget, + }; + // A proven path is worth waiting a moment for: MTU discovery starts when the handshake + // completes and needs an acked probe per binary-search step, so at `Welcome` time it may + // simply not have got there yet. Bounded, and only on paths that already proved it once. + let awaited_proof = proven_udp_budget + .and_then(|_| jumbo_target(target_wire_mtu, client_ceiling, peer)) + .map(|t| sealed_datagram_bytes(t) as u16); + if let Some(sealed) = awaited_proof { + if jumbo.live_udp_mtu < sealed { + let t0 = std::time::Instant::now(); + while t0.elapsed() < JUMBO_PROOF_WAIT { + tokio::time::sleep(JUMBO_PROOF_POLL).await; + jumbo.live_udp_mtu = conn.stats().path.current_mtu; + if jumbo.live_udp_mtu >= sealed { + break; + } + } + tracing::debug!( + peer = %peer, + waited_ms = t0.elapsed().as_millis() as u64, + live_udp_mtu = jumbo.live_udp_mtu, + needed = sealed, + "wire MTU: waited for this connection to re-prove its jumbo path" + ); + } + } + resolve(env, learned_budget, jumbo, peer) } -/// Pure resolution (env override > learned budget > family default) — the tested core of -/// [`negotiated_shard_payload`]. -fn resolve(env_wire_mtu: Option, learned_udp_budget: Option, peer: IpAddr) -> usize { +/// The peer's jumbo verdict if it is still redeemable: same operator target, inside the TTL. +/// A verdict that fails either test is dropped on the spot rather than left to rot. +fn fresh_verdict(key: PathKey, target_wire_mtu: Option) -> Option { + let target = target_wire_mtu?; + let mut map = jumbo_verdicts().lock().unwrap(); + let v = *map.get(&key)?; + if v.target_wire_mtu != target || v.at.elapsed() > JUMBO_VERDICT_TTL { + map.remove(&key); + return None; + } + Some(v.udp_budget) +} + +/// Pure resolution (proven jumbo > env override > learned budget > family default) — the tested +/// core of [`negotiated_shard_payload`]. +fn resolve( + env_wire_mtu: Option, + learned_udp_budget: Option, + jumbo: JumboStart, + peer: IpAddr, +) -> usize { let default = mtu1500_shard_payload_for(peer); + // First, because the two are mutually exclusive by construction: `jumbo_wire_mtu()` only + // fires above 1500, and the env branch below CLAMPS to the family default, so a + // `PUNKTFUNK_WIRE_MTU=9000` operator would otherwise get 1408 and never a jumbo start. + if let Some(p) = jumbo_session_start(jumbo, peer) { + tracing::info!( + peer = %peer, + shard_payload = p, + default, + live_udp_mtu = jumbo.live_udp_mtu, + proven_udp_budget = jumbo.proven_udp_budget, + "wire MTU: session starts at the JUMBO shard — this path proved it in a previous \ + session AND re-proved it live on this connection (~6× fewer datagrams per frame)" + ); + return p; + } if let Some(mtu) = env_wire_mtu { let p = shard_payload_for_wire_mtu(mtu, peer); if p != default { @@ -119,34 +339,73 @@ fn resolve(env_wire_mtu: Option, learned_udp_budget: Option, peer: I /// into a verdict — and, with a [`ShardReneg`] driver, act on it MID-SESSION /// (design/shard-payload-reneg.md Phase 2): a below-ceiling verdict shrinks the live wire at /// the ~3–10 s mark (session 1 heals instead of staying black), and a settled-at-jumbo -/// verdict grows it, ack-gated, when the operator opted in. Spawned once per negotiated -/// session; without a grow the task ends after the final sample (bounded ~10 s lifetime, -/// holding only a cheap `Connection` handle) — after a grow it stays as the revert guard -/// until the connection closes. +/// verdict grows it, ack-gated, when the operator opted in. The same settled-at-jumbo reading +/// also writes this path's next-session verdict (PW7a) — `client_ceiling` is the client's +/// `Hello::max_shard_payload`, which decides what "jumbo" is worth proving for this peer. +/// Spawned once per negotiated session; without a grow the task ends after the final sample +/// (bounded ~10 s lifetime, holding only a cheap `Connection` handle) — after a grow, or on a +/// session that STARTED jumbo, it stays as the revert guard until the connection closes. pub(super) fn spawn_watch( conn: quinn::Connection, session_shard_payload: usize, + client_ceiling: u16, reneg: Option, ) { tokio::spawn(async move { let peer = conn.remote_address().ip(); let ceiling = video_datagram_udp_ceiling() as u16; + // The sealed size a JUMBO proof has to reach on this path (PW7a) — `None` unless the + // operator opted in AND this client advertised the headroom. Read once: the verdict + // records the target it was proven under, and the two must be the same number. + let target_wire_mtu = jumbo_wire_mtu(); + let jumbo_proof = + jumbo_target(target_wire_mtu, client_ceiling, peer).map(sealed_datagram_bytes); // Discovery finishes in a handful of RTTs on a LAN (well under the first sample) but // needs a loss timeout per failed probe on a constrained path — the second sample // covers that with margin. Max, because discovery only ever raises `current_mtu` // (the post-grow revert guard below re-reads it live, where blackhole detection CAN - // lower it again). + // lower it again). Stop early only once nothing more is expected: with a jumbo opt-in + // the search keeps climbing past the 1500-byte ceiling, and stopping there would throw + // away the very measurement the proof needs. + let goal = jumbo_proof + .unwrap_or(ceiling as usize) + .max(ceiling as usize) as u16; let mut settled = 0u16; for wait_s in [3u64, 7] { tokio::time::sleep(std::time::Duration::from_secs(wait_s)).await; settled = settled.max(conn.stats().path.current_mtu); - if settled >= ceiling { + if settled >= goal { break; } } // The wire this session is CURRENTLY sealed at — moves on a mid-session shrink/grow. let mut current = session_shard_payload; let mut reneg = reneg; + // PW7a bookkeeping, before anything else can return: this is where a jumbo path earns + // its next-session verdict — and, far more importantly, where it LOSES it. Recording + // needs a live connection that reached the sealed target; anything else (a lower + // settle, a connection that died before the window closed, i.e. exactly what a client + // staring at a black screen does) erases, so the next session falls back to the + // 1500-byte default and has to prove itself again from scratch. + if let Some(need) = jumbo_proof { + let key = path_key(&conn); + if settled as usize >= need && conn.close_reason().is_none() { + jumbo_verdicts().lock().unwrap().insert( + key, + JumboVerdict { + udp_budget: settled, + target_wire_mtu: target_wire_mtu.unwrap_or_default(), + at: std::time::Instant::now(), + }, + ); + tracing::info!(peer = %peer, discovered_udp_mtu = settled, needed = need, + "wire MTU: this path carries JUMBO video datagrams — the next session over \ + it starts at the big shard (it still has to re-prove the path live)"); + } else if jumbo_verdicts().lock().unwrap().remove(&key).is_some() { + tracing::info!(peer = %peer, discovered_udp_mtu = settled, needed = need, + "wire MTU: jumbo verdict cleared — this path no longer proves it"); + } + } if settled >= ceiling { // The path carries full-size video datagrams — erase any stale learned clamp so // the next session returns to the default wire. @@ -154,6 +413,34 @@ pub(super) fn spawn_watch( tracing::info!(peer = %peer, "wire MTU: path re-measured at full size — learned clamp cleared"); } + // …but "full size" is the 1500-byte ceiling, and this session may have STARTED + // above it (a PW7a jumbo start whose path changed since the proof, or a client + // that roamed onto a 1500-MTU link). Then every video datagram is dying right now. + // The verdict is already erased above; heal the live wire if this session can be + // re-keyed at all — a PyroWave client cannot (its parse window is the `Welcome` + // value), so for those the WARN plus a corrected next session is all there is. + if sealed_datagram_bytes(current) > settled as usize { + tracing::warn!( + peer = %peer, + discovered_udp_mtu = settled, + shard_payload = current, + "wire MTU: this session started at a JUMBO shard but the path does not \ + carry it — video datagrams are oversized for a hop, which streams as a \ + black screen with zero reported loss. The jumbo verdict for this path is \ + cleared: the next connect starts at the standard 1500-byte wire." + ); + if let Some(r) = reneg.as_ref() { + let back = shard_payload_for_udp_budget(settled as usize, peer); + if back < current + && r.change_tx.send(back as u16).is_ok() + && r.apply_tx.send(back).is_ok() + { + tracing::info!(peer = %peer, shard_payload = back, was = current, + "wire MTU: video re-keyed mid-session back to the standard wire"); + current = back; + } + } + } } else { // A closed connection stops discovering, so a session that ended before the final // sample proves nothing (a healthy high-RTT path could still be mid-search): learn @@ -203,12 +490,41 @@ pub(super) fn spawn_watch( } } } + // PW7a revert guard for a session that STARTED jumbo and has no re-key channel (the + // PyroWave case, and the only reason the session-start grow exists). Nothing can save + // this session if the path stops fitting mid-stream — but the NEXT one must not repeat + // it, so keep sampling and drop the verdict the moment quinn's blackhole detection or + // a re-search says the path shrank. Cheap: one `Connection` handle, one sample per 5 s. + // Only for a session that is currently FITTING — one that already failed the check + // above has been warned about and had its verdict erased there. + if current > mtu1500_shard_payload_for(peer) + && reneg.is_none() + && sealed_datagram_bytes(current) <= settled as usize + { + loop { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + if conn.close_reason().is_some() { + return; + } + let mtu_now = conn.stats().path.current_mtu; + if (mtu_now as usize) < sealed_datagram_bytes(current) { + jumbo_verdicts().lock().unwrap().remove(&path_key(&conn)); + tracing::warn!(peer = %peer, discovered_udp_mtu = mtu_now, + shard_payload = current, + "wire MTU: the jumbo path this session started on stopped fitting — this \ + session cannot be re-keyed (chunk-aligned client parse window), so it \ + will not recover, but the verdict is cleared and the next connect \ + starts at the standard wire"); + return; + } + } + } // Phase 2 up-leg: jumbo grow — operator opt-in (PUNKTFUNK_JUMBO / PUNKTFUNK_WIRE_MTU // > 1500, which also raised the endpoint's probe ceiling so `settled` can even reach // here), client-advertised headroom, and a settled-at-jumbo proof. The grow is // ACK-GATED: not one sealed datagram above the old size leaves before the client's // ack, even though its buffers are statically sized — the rule must not erode. - let (Some(mtu), Some(r)) = (jumbo_wire_mtu(), reneg.as_mut()) else { + let (Some(mtu), Some(r)) = (target_wire_mtu, reneg.as_mut()) else { return; }; let target = jumbo_shard_payload_for(mtu, peer).min(r.client_ceiling as usize); @@ -275,34 +591,196 @@ mod tests { const V4: IpAddr = IpAddr::V4(Ipv4Addr::new(192, 168, 1, 2)); const V6: IpAddr = IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1)); + /// No jumbo anywhere — what every session that isn't on an opted-in jumbo LAN passes. + const NO_JUMBO: JumboStart = JumboStart { + target_wire_mtu: None, + client_ceiling: 0, + live_udp_mtu: 0, + proven_udp_budget: None, + clamped_udp_budget: None, + }; + /// A 9000-MTU LAN, a modern client, a path proven last session and re-proven live now. + fn proven_jumbo() -> JumboStart { + JumboStart { + target_wire_mtu: Some(9000), + client_ceiling: punktfunk_core::config::max_shard_payload() as u16, + live_udp_mtu: 8972, + proven_udp_budget: Some(8972), + clamped_udp_budget: None, + } + } #[test] fn default_when_nothing_known() { - assert_eq!(resolve(None, None, V4), mtu1500_shard_payload_for(V4)); - assert_eq!(resolve(None, None, V6), mtu1500_shard_payload_for(V6)); + assert_eq!( + resolve(None, None, NO_JUMBO, V4), + mtu1500_shard_payload_for(V4) + ); + assert_eq!( + resolve(None, None, NO_JUMBO, V6), + mtu1500_shard_payload_for(V6) + ); } #[test] fn env_override_beats_learned() { // 1280 wire − 28 IP/UDP − 64 header/crypto = 1188. - assert_eq!(resolve(Some(1280), Some(1472), V4), 1188); + assert_eq!(resolve(Some(1280), Some(1472), NO_JUMBO, V4), 1188); } #[test] fn learned_budget_clamps() { // A WARP-shaped path: 1280-byte UDP budget → 1280 − 64 = 1216. - assert_eq!(resolve(None, Some(1280), V4), 1216); + assert_eq!(resolve(None, Some(1280), NO_JUMBO, V4), 1216); } #[test] fn learned_at_or_above_ceiling_is_the_default_wire() { - assert_eq!(resolve(None, Some(1472), V4), mtu1500_shard_payload_for(V4)); - assert_eq!(resolve(None, Some(2000), V4), mtu1500_shard_payload_for(V4)); + assert_eq!( + resolve(None, Some(1472), NO_JUMBO, V4), + mtu1500_shard_payload_for(V4) + ); + assert_eq!( + resolve(None, Some(2000), NO_JUMBO, V4), + mtu1500_shard_payload_for(V4) + ); } #[test] fn env_full_mtu_is_the_default_wire_both_families() { - assert_eq!(resolve(Some(1500), None, V4), mtu1500_shard_payload_for(V4)); - assert_eq!(resolve(Some(1500), None, V6), mtu1500_shard_payload_for(V6)); + assert_eq!( + resolve(Some(1500), None, NO_JUMBO, V4), + mtu1500_shard_payload_for(V4) + ); + assert_eq!( + resolve(Some(1500), None, NO_JUMBO, V6), + mtu1500_shard_payload_for(V6) + ); + } + + /// The happy path, both families: 9000 − 28 (IPv4) − 64 = 8908, and 9000 − 48 − 64 = 8888. + #[test] + fn proven_and_reproven_path_starts_jumbo() { + assert_eq!(jumbo_session_start(proven_jumbo(), V4), Some(8908)); + let mut v6 = proven_jumbo(); + v6.live_udp_mtu = 8952; + v6.proven_udp_budget = Some(8952); + assert_eq!(jumbo_session_start(v6, V6), Some(8888)); + // …and it is what `resolve` returns, ahead of the env branch that would clamp a + // >1500 `PUNKTFUNK_WIRE_MTU` back down to the family default. + assert_eq!(resolve(Some(9000), None, proven_jumbo(), V4), 8908); + } + + /// THE guard: the laptop that proved jumbo on the wired LAN and came back on a 1500-MTU + /// link. The memory still says jumbo; the live connection says otherwise; the live one + /// wins, every time. This is what makes the grow as safe as the clamp. + #[test] + fn a_remembered_verdict_never_grows_without_a_live_reproof() { + let mut moved = proven_jumbo(); + moved.live_udp_mtu = 1472; // a clean 1500-MTU path, freshly measured + assert_eq!(jumbo_session_start(moved, V4), None); + assert_eq!( + resolve(None, None, moved, V4), + mtu1500_shard_payload_for(V4) + ); + // Not even one byte of headroom short of the sealed target is enough. + let mut nearly = proven_jumbo(); + nearly.live_udp_mtu = 8971; + assert_eq!(jumbo_session_start(nearly, V4), None); + } + + /// …and the mirror: a live-proven path with no prior verdict still starts at the default. + /// Both halves are required, so a single fluke on either side cannot seal a jumbo wire. + #[test] + fn a_live_proof_alone_does_not_grow() { + let mut first_ever = proven_jumbo(); + first_ever.proven_udp_budget = None; + assert_eq!(jumbo_session_start(first_ever, V4), None); + let mut weak_memory = proven_jumbo(); + weak_memory.proven_udp_budget = Some(1472); + assert_eq!(jumbo_session_start(weak_memory, V4), None); + } + + /// The two memories are keyed differently (clamp: peer; verdict: route), so they can + /// disagree. When they do, the one that keeps datagrams small wins. + #[test] + fn a_constrained_path_clamp_vetoes_the_grow() { + let mut contradicted = proven_jumbo(); + contradicted.clamped_udp_budget = Some(1280); + assert_eq!(jumbo_session_start(contradicted, V4), None); + // A clamp that is itself at or above the sealed target isn't contrary evidence. + let mut roomy = proven_jumbo(); + roomy.clamped_udp_budget = Some(8972); + assert_eq!(jumbo_session_start(roomy, V4), Some(8908)); + } + + #[test] + fn without_the_operator_opt_in_nothing_grows() { + let mut no_optin = proven_jumbo(); + no_optin.target_wire_mtu = None; + assert_eq!(jumbo_session_start(no_optin, V4), None); + } + + /// A legacy client (no `Hello::max_shard_payload`) is never handed a geometry it did not + /// advertise, and a client whose ceiling lands under the family default is left alone + /// rather than being "grown" to something smaller. + #[test] + fn the_client_ceiling_is_binding() { + let mut legacy = proven_jumbo(); + legacy.client_ceiling = 0; + assert_eq!(jumbo_session_start(legacy, V4), None); + let mut small = proven_jumbo(); + small.client_ceiling = 1408; + assert_eq!(jumbo_session_start(small, V4), None); + // A ceiling between the default and the path target caps the grow — and the proof + // then only has to cover the SMALLER sealed size. + let mut capped = proven_jumbo(); + capped.client_ceiling = 4000; + assert_eq!(jumbo_session_start(capped, V4), Some(4000)); + } + + /// Every shard payload the grow can produce is even (Leopard FEC splits shards in halves) + /// and fits the receive ceiling every client sizes its buffers from. + #[test] + fn grown_shards_stay_even_and_inside_the_receive_ceiling() { + for mtu in [2000usize, 4000, 4001, 9000, 9216, 64000] { + for peer in [V4, V6] { + let Some(t) = jumbo_target(Some(mtu), u16::MAX, peer) else { + continue; + }; + assert_eq!(t % 2, 0, "odd shard for mtu {mtu}"); + assert!(t <= punktfunk_core::config::max_shard_payload()); + assert!(t > mtu1500_shard_payload_for(peer)); + assert!( + sealed_datagram_bytes(t) <= punktfunk_core::packet::MAX_DATAGRAM_BYTES, + "sealed datagram overflows the receive ceiling at mtu {mtu}" + ); + } + } + // Below the family default there is nothing to grow to. + assert_eq!(jumbo_target(Some(1500), u16::MAX, V4), None); + assert_eq!(jumbo_target(None, u16::MAX, V4), None); + } + + /// A path is a (local interface, peer) pair, not a peer: the same client reached over the + /// host's other NIC is a different route with a different MTU. + #[test] + fn the_verdict_key_separates_routes_to_the_same_peer() { + let over_10g = PathKey { + local: Some(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))), + peer: V4, + }; + let over_wifi = PathKey { + local: Some(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1))), + peer: V4, + }; + assert_ne!(over_10g, over_wifi); + assert_ne!( + over_10g, + PathKey { + local: None, + peer: V4 + } + ); } }