From a8744b98d125e692655996099cecf52025c57bba Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Wed, 29 Jul 2026 13:35:04 +0200 Subject: [PATCH] feat(devtools): reproduce the capture-model cursor path without a real client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit punktfunk-probe grows --cursor-capture: advertise CLIENT_CAP_CURSOR, flip the channel to the capture model (CursorRenderMode client_draws= false), and wiggle RELATIVE pointer motion for the whole dump — decode the .h265 and the host-composited pointer must be in the pixels. Plus --codec pyrowave (advertised only on request so the dump format of existing recipes never changes). tools/cursor-probe stands up the capture side alone: virtual output with the out-of-band cursor, production PipeWire capturer, production injector, absolute then relative motion — and reports whether SPA_META_Cursor ever yields an overlay. It is how Mutter's pointer-in-stream metadata gate was isolated on-glass. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 13 ++ Cargo.toml | 1 + clients/probe/src/main.rs | 93 +++++++++++++- tools/cursor-probe/Cargo.toml | 26 ++++ tools/cursor-probe/src/main.rs | 217 +++++++++++++++++++++++++++++++++ 5 files changed, 344 insertions(+), 6 deletions(-) create mode 100644 tools/cursor-probe/Cargo.toml create mode 100644 tools/cursor-probe/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 756c6440..3aa8debc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -945,6 +945,19 @@ dependencies = [ "libloading", ] +[[package]] +name = "cursor-probe" +version = "0.21.0" +dependencies = [ + "anyhow", + "pf-capture", + "pf-inject", + "pf-vdisplay", + "punktfunk-core", + "tracing", + "tracing-subscriber", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" diff --git a/Cargo.toml b/Cargo.toml index 2f1e40f4..61e3c8af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ members = [ "clients/session", "clients/windows", "clients/android/native", + "tools/cursor-probe", "tools/display-disturb", "tools/latency-probe", "tools/loss-harness", diff --git a/clients/probe/src/main.rs b/clients/probe/src/main.rs index d8760bfe..976f57b5 100644 --- a/clients/probe/src/main.rs +++ b/clients/probe/src/main.rs @@ -53,8 +53,9 @@ use punktfunk_core::config::Role; use punktfunk_core::input::{InputEvent, InputKind}; use punktfunk_core::packet::FLAG_PROBE; use punktfunk_core::quic::{ - endpoint, io, window_loss_ppm, BitrateChanged, Hello, LossReport, ProbeRequest, ProbeResult, - Reconfigure, Reconfigured, RequestKeyframe, SetBitrate, Start, Welcome, + endpoint, io, window_loss_ppm, BitrateChanged, CursorRenderMode, Hello, LossReport, + ProbeRequest, ProbeResult, Reconfigure, Reconfigured, RequestKeyframe, SetBitrate, Start, + Welcome, }; use punktfunk_core::transport::UdpTransport; use punktfunk_core::{CompositorPref, Mode, PunktfunkError, Session}; @@ -115,6 +116,12 @@ struct Args { /// `--speed-test KBPS:MS` — after the stream starts, ask the host for a `MS`-millisecond /// bandwidth probe burst at `KBPS`, then report measured throughput + loss. speed_test: Option<(u32, u32)>, + /// `--cursor-capture` — negotiate the cursor channel (`CLIENT_CAP_CURSOR`) and immediately + /// flip it to the capture model (`CursorRenderMode { client_draws: false }`), then wiggle the + /// pointer with RELATIVE motion for the whole stream: the headless reproduction of a + /// pointer-lock client expecting the HOST to composite the cursor into the video. Decode the + /// dump and look for the pointer — a cursorless dump is the bug this flag was built to catch. + cursor_capture: bool, /// `--discover [SECS]` — browse the LAN for native (`_punktfunk._udp`) hosts for `SECS` /// seconds (default 4), print what's found, and exit. No connection is made. discover: Option, @@ -278,6 +285,7 @@ fn parse_args() -> Args { "h264" | "avc" => punktfunk_core::quic::CODEC_H264, "hevc" | "h265" => punktfunk_core::quic::CODEC_HEVC, "av1" => punktfunk_core::quic::CODEC_AV1, + "pyrowave" => punktfunk_core::quic::CODEC_PYROWAVE, _ => 0, // auto — no preference }, launch: get("--launch").map(str::to_string), @@ -292,6 +300,7 @@ fn parse_args() -> Args { .any(|a| a == "--discover") .then(|| get("--discover").and_then(|s| s.parse().ok()).unwrap_or(4)), clock_resync: argv.iter().any(|a| a == "--clock-resync"), + cursor_capture: argv.iter().any(|a| a == "--cursor-capture"), } } @@ -308,6 +317,7 @@ fn codec_ext(codec: u8) -> &'static str { match codec { punktfunk_core::quic::CODEC_H264 => "h264", punktfunk_core::quic::CODEC_AV1 => "av1", + punktfunk_core::quic::CODEC_PYROWAVE => "pyrowave", _ => "h265", } } @@ -517,16 +527,29 @@ async fn session(args: Args) -> Result<()> { // `Welcome::codec`; the dump extension follows that. video_codecs: punktfunk_core::quic::CODEC_H264 | punktfunk_core::quic::CODEC_HEVC - | punktfunk_core::quic::CODEC_AV1, + | punktfunk_core::quic::CODEC_AV1 + // PyroWave only on request: a PyroWave-default host would otherwise pick it + // for every probe run, changing the dump format under long-standing recipes. + | if args.preferred_codec == punktfunk_core::quic::CODEC_PYROWAVE { + punktfunk_core::quic::CODEC_PYROWAVE + } else { + 0 + }, // `--codec` soft preference (0 = auto). The host honors it when it can emit it. preferred_codec: args.preferred_codec, // PUNKTFUNK_CLIENT_PEAK_NITS= advertises a synthetic display volume — the host // writes it into the virtual display's EDID (CTA HDR block), so the EDID-forwarding // path can be validated headlessly (check the host's monitor caps / ADD log line). display_hdr: punktfunk_core::client::display_hdr_env_override(), - // No CLIENT_CAP_CURSOR: this headless tool renders nothing — advertising it would - // just strip the pointer from the dumped bitstream. - client_caps: 0, + // No CLIENT_CAP_CURSOR by default: this headless tool renders nothing — advertising + // it would just strip the pointer from the dumped bitstream. `--cursor-capture` + // advertises it deliberately and then flips the channel to the capture model, so the + // HOST composites and the dump is where the pointer must appear. + client_caps: if args.cursor_capture { + punktfunk_core::quic::CLIENT_CAP_CURSOR + } else { + 0 + }, } .encode(), ) @@ -822,6 +845,64 @@ async fn session(args: Args) -> Result<()> { "SPEED TEST complete", ); }); + } else if args.cursor_capture { + // Capture-model cursor repro: flip the negotiated cursor channel to "host composites" + // and drive RELATIVE pointer motion, exactly like a pointer-lock client. Owns BOTH + // control halves: the host may send CursorShape (0x50) messages while the channel is + // still in its initial client-draws state, and dropping our recv half would fail those + // writes host-side. + let mut cs = send; + let mut cr = recv; + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + match io::write_msg( + &mut cs, + &CursorRenderMode { + client_draws: false, + } + .encode(), + ) + .await + { + Ok(()) => tracing::info!( + "cursor-capture: CursorRenderMode {{ client_draws: false }} sent — the host \ + must now composite the pointer into the video" + ), + Err(e) => { + tracing::error!(error = %format!("{e:#}"), "cursor-capture: render-mode write failed"); + return; + } + } + // Drain (and just log) whatever the host still sends on the control stream. + while let Ok(b) = cr.read_msg().await { + tracing::debug!( + len = b.len(), + ty = b.get(4).copied().unwrap_or(0), + "cursor-capture: control message drained" + ); + } + }); + let wiggle_conn = conn.clone(); + tokio::spawn(async move { + // Relative circles, forever: keeps the host pointer moving (and, on metadata-cursor + // compositors, keeps cursor updates flowing) for the whole dump. + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + tracing::info!("cursor-capture: relative pointer wiggle running"); + let mut t = 0.0f64; + loop { + let e = InputEvent { + kind: InputKind::MouseMove, + _pad: [0; 3], + code: 0, + x: (10.0 * t.cos()) as i32, + y: (10.0 * t.sin()) as i32, + flags: 0, + }; + let _ = wiggle_conn.send_datagram(e.encode().to_vec().into()); + t += 0.2; + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + }); } else { // Normal stream mode: relay the data loop's windowed loss estimate to the host as periodic // LossReports, so it can size FEC to the link (adaptive FEC), and — like the real clients — diff --git a/tools/cursor-probe/Cargo.toml b/tools/cursor-probe/Cargo.toml new file mode 100644 index 00000000..1fab0ea8 --- /dev/null +++ b/tools/cursor-probe/Cargo.toml @@ -0,0 +1,26 @@ +# On-glass probe for the Linux cursor-as-metadata capture path: creates a virtual output with the +# out-of-band cursor requested (exactly like a cursor-channel session), attaches the production +# PipeWire capturer, wiggles the pointer over it via the production injector, and reports whether +# `SPA_META_Cursor` ever yields an overlay (position + bitmap). Answers "does THIS compositor +# deliver cursor metadata on a virtual output" without needing a client. +[package] +name = "cursor-probe" +description = "On-glass probe: does this compositor's virtual output deliver SPA_META_Cursor?" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[target.'cfg(target_os = "linux")'.dependencies] +pf-vdisplay = { path = "../../crates/pf-vdisplay" } +pf-capture = { path = "../../crates/pf-capture" } +pf-inject = { path = "../../crates/pf-inject" } +punktfunk-core = { path = "../../crates/punktfunk-core", features = ["quic"] } +anyhow = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +[lints] +workspace = true diff --git a/tools/cursor-probe/src/main.rs b/tools/cursor-probe/src/main.rs new file mode 100644 index 00000000..0950d0ed --- /dev/null +++ b/tools/cursor-probe/src/main.rs @@ -0,0 +1,217 @@ +//! Cursor-as-metadata on-glass probe (see Cargo.toml). Run inside the target user session: +//! +//! ```sh +//! DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u)/bus cursor-probe [--seconds 20] [--embedded] [--gpu] +//! ``` +//! +//! `--embedded` A/Bs the pre-channel path (compositor embeds the pointer, no metadata expected); +//! `--gpu` takes the zero-copy dmabuf negotiation a real session uses instead of the CPU mmap path. + +#[cfg(target_os = "linux")] +fn main() -> anyhow::Result<()> { + linux::run() +} + +#[cfg(not(target_os = "linux"))] +fn main() { + eprintln!("cursor-probe is Linux-only (compositor virtual outputs)"); + std::process::exit(1); +} + +#[cfg(target_os = "linux")] +mod linux { + use anyhow::{Context, Result}; + use punktfunk_core::input::{InputEvent, InputKind}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + use std::time::{Duration, Instant}; + + const W: u32 = 1280; + const H: u32 = 800; + + pub fn run() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + let args: Vec = std::env::args().collect(); + let arg = |name: &str| { + args.iter() + .skip_while(|a| a.as_str() != name) + .nth(1) + .cloned() + }; + let secs: u64 = arg("--seconds").and_then(|s| s.parse().ok()).unwrap_or(20); + let embedded = args.iter().any(|a| a == "--embedded"); + let gpu = args.iter().any(|a| a == "--gpu"); + + let compositor = pf_vdisplay::detect()?; + println!("cursor-probe: compositor = {compositor:?}"); + let mut vd = pf_vdisplay::open(compositor)?; + // The cursor-channel session's request: out-of-band cursor (Mutter/KWin metadata mode). + vd.set_hw_cursor(!embedded); + println!( + "cursor-probe: virtual output with {} cursor", + if embedded { + "EMBEDDED (compositor paints it)" + } else { + "METADATA (out-of-band)" + } + ); + let vout = vd + .create(pf_vdisplay::Mode { + width: W, + height: H, + refresh_hz: 60, + }) + .context("create the virtual output")?; + println!("cursor-probe: node_id = {}", vout.node_id); + + // Encode-backend facts don't matter to the metadata question; all-off = CPU-friendly. + let policy = pf_capture::ZeroCopyPolicy { + backend_is_vaapi: false, + backend_is_gpu: gpu, + pyrowave_session: false, + native_nv12_session: false, + pyrowave_modifiers: Vec::new(), + hdr_cuda_ok: false, + }; + let mut cap = pf_capture::open_virtual_output( + vout.remote_fd, + vout.node_id, + vout.preferred_mode, + vout.keepalive, + gpu, + false, + false, + policy, + vout.expect_exact_dims, + ) + .context("attach the PipeWire capturer")?; + cap.set_active(true); + // A cursor-channel session in capture mode: the HOST composites, so the capturer must + // keep surfacing the overlay (this is a no-op on the Linux portal capturer, but state it + // the way the encode loop does). + cap.set_cursor_forward(false); + + // Wiggle the pointer over the virtual output through the production injector (libei on + // GNOME/KWin). The region ladder maps the absolute events onto the output by size match — + // W×H is deliberately distinctive. + let backend = pf_inject::default_backend(); + println!("cursor-probe: input backend = {backend:?} (4 s device-resume wait)"); + let stop = Arc::new(AtomicBool::new(false)); + let stop_inj = stop.clone(); + // `dyn InputInjector` isn't Send — build it on the thread that drives it. + let injector = std::thread::spawn(move || { + let mut inj = match pf_inject::open(backend) { + Ok(i) => i, + Err(e) => { + tracing::error!(error = %format!("{e:#}"), "cursor-probe: injector open failed"); + return; + } + }; + std::thread::sleep(Duration::from_secs(4)); + let flags = (W << 16) | (H & 0xffff); + let (cx, cy, r) = ((W / 2) as f64, (H / 2) as f64, 200.0f64); + let mut t = 0.0f64; + let started = Instant::now(); + let mut announced_rel = false; + while !stop_inj.load(Ordering::Relaxed) { + // Phase 1 (8 s): ABSOLUTE wiggle — parks the pointer on the virtual output and + // proves the metadata path. Phase 2: RELATIVE circles — what a capture-mode + // (pointer-lock) client sends; watch whether the live meta position keeps moving. + let e = if started.elapsed() < Duration::from_secs(8) { + InputEvent { + kind: InputKind::MouseMoveAbs, + _pad: [0; 3], + code: 0, + x: (cx + r * t.cos()) as i32, + y: (cy + r * t.sin()) as i32, + flags, + } + } else { + if !announced_rel { + announced_rel = true; + println!("cursor-probe: switching to RELATIVE motion (capture-mode model)"); + } + InputEvent { + kind: InputKind::MouseMove, + _pad: [0; 3], + code: 0, + x: (12.0 * t.cos()) as i32, + y: (12.0 * t.sin()) as i32, + flags: 0, + } + }; + if let Err(err) = inj.inject(&e) { + tracing::warn!(error = %format!("{err:#}"), "cursor-probe: inject failed"); + } + t += 0.15; + std::thread::sleep(Duration::from_millis(25)); + } + }); + + let deadline = Instant::now() + Duration::from_secs(secs); + let (mut frames, mut with_overlay) = (0u64, 0u64); + let mut last_report = Instant::now(); + let mut live_seen = false; + while Instant::now() < deadline { + // Damage-driven source: timeouts (gaps) are normal, hence the missing error arm. + if let Ok(f) = cap.next_frame_within(Duration::from_millis(500)) { + frames += 1; + if let Some(c) = &f.cursor { + with_overlay += 1; + if with_overlay == 1 { + println!( + "cursor-probe: FIRST frame-attached overlay: pos=({}, {}) \ + {}x{} serial={} visible={}", + c.x, c.y, c.w, c.h, c.serial, c.visible + ); + } + } + } + if last_report.elapsed() >= Duration::from_secs(2) { + last_report = Instant::now(); + let live = cap.cursor(); + if live.is_some() { + live_seen = true; + } + match live { + Some(c) => println!( + "cursor-probe: frames={frames} with_overlay={with_overlay} live: \ + pos=({}, {}) {}x{} serial={} visible={}", + c.x, c.y, c.w, c.h, c.serial, c.visible + ), + None => println!( + "cursor-probe: frames={frames} with_overlay={with_overlay} live: NONE \ + (no SPA_META_Cursor bitmap yet)" + ), + } + } + } + stop.store(true, Ordering::Relaxed); + let _ = injector.join(); + + println!("---"); + if embedded { + println!( + "cursor-probe: EMBEDDED run — no metadata expected; check the stream visually. \ + frames={frames}" + ); + } else if live_seen || with_overlay > 0 { + println!( + "cursor-probe: VERDICT: metadata cursor DELIVERED ({with_overlay}/{frames} \ + frames carried it; live overlay seen: {live_seen})" + ); + } else { + println!( + "cursor-probe: VERDICT: NO cursor metadata after {secs}s of pointer motion over \ + the virtual output ({frames} frames) — the compositor is not delivering \ + SPA_META_Cursor on this stream" + ); + } + Ok(()) + } +}