From c8491af893aa473d72867af276cbef6eba70285c Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Tue, 9 Jun 2026 07:51:49 +0000 Subject: [PATCH] =?UTF-8?q?feat(m2):=20real=20desktop=20capture=20in=20the?= =?UTF-8?q?=20video=20stream=20(portal=20=E2=86=92=20Moonlight)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire M0's portal desktop capture into the GameStream video plane: with LUMEN_VIDEO_SOURCE=portal the stream captures the headless wlroots desktop (PipeWire RGB) instead of the synthetic pattern, opens NVENC from the first captured frame's format/size, and streams it. Verified live: a stock Moonlight client shows the real 5120×1440 desktop at ~42 fps (release build). - capture.rs: FastSyntheticCapturer (cheap fill pattern, real-time at 5K) so both sources share the Capturer trait - stream.rs: source select (portal | synthetic), encoder opened from the first frame, wall-clock 90 kHz RTP timestamps (correct under a variable capture rate) Note: the CPU-copy RGB→rgb0 path caps ~42 fps at 5K (single-threaded); dmabuf zero-copy is the deferred optimization (plan §9). Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/lumen-host/src/capture.rs | 43 +++++++++++++ crates/lumen-host/src/gamestream/stream.rs | 75 +++++++++++----------- 2 files changed, 82 insertions(+), 36 deletions(-) diff --git a/crates/lumen-host/src/capture.rs b/crates/lumen-host/src/capture.rs index dfa466c..6bc26a6 100644 --- a/crates/lumen-host/src/capture.rs +++ b/crates/lumen-host/src/capture.rs @@ -121,6 +121,49 @@ impl Capturer for SyntheticCapturer { } } +/// A cheap moving test pattern (BGRx) for the streaming path: a pulsing field + a white band +/// sweeping down, generated with whole-buffer `fill`s so it stays real-time even at 5K. +pub struct FastSyntheticCapturer { + width: u32, + height: u32, + frame_idx: u64, + buf: Vec, +} + +impl FastSyntheticCapturer { + pub fn new(width: u32, height: u32) -> Self { + assert!(width > 0 && height > 0); + FastSyntheticCapturer { + width, + height, + frame_idx: 0, + buf: vec![0u8; width as usize * height as usize * 4], + } + } +} + +impl Capturer for FastSyntheticCapturer { + fn next_frame(&mut self) -> Result { + let (w, h) = (self.width as usize, self.height as usize); + let row = w * 4; + let shade = (self.frame_idx % 256) as u8; + self.buf.fill(shade); + let band_h = (h / 20).max(1); + let band_y = (self.frame_idx as usize * 6) % h; + for y in band_y..(band_y + band_h).min(h) { + self.buf[y * row..(y + 1) * row].fill(0xff); + } + self.frame_idx += 1; + Ok(CapturedFrame { + width: self.width, + height: self.height, + pts_ns: 0, + format: PixelFormat::Bgrx, + cpu_bytes: self.buf.clone(), + }) + } +} + /// Open a live capturer for a client-sized monitor via the xdg ScreenCast portal /// (`ashpd`) → PipeWire (`pipewire`). Implemented in the `linux` submodule. #[cfg(target_os = "linux")] diff --git a/crates/lumen-host/src/gamestream/stream.rs b/crates/lumen-host/src/gamestream/stream.rs index 8f8e3f9..95ed81e 100644 --- a/crates/lumen-host/src/gamestream/stream.rs +++ b/crates/lumen-host/src/gamestream/stream.rs @@ -1,11 +1,11 @@ //! The video data plane: on RTSP PLAY, learn the client's UDP endpoint (it pings the video -//! port), then run capture → NVENC encode → [`VideoPacketizer`] → UDP send. P1.3 uses a fast -//! synthetic test pattern (proves the wire format with no compositor); swapping in real -//! portal desktop capture is a one-line source change. Runs on its own native thread. +//! port), then run capture → NVENC encode → [`VideoPacketizer`] → UDP send. The source is +//! either real portal desktop capture (`LUMEN_VIDEO_SOURCE=portal`, the M0 PipeWire path) or +//! a synthetic test pattern (default). Runs on its own native thread. use super::video::{FrameType, VideoPacketizer}; use super::VIDEO_PORT; -use crate::capture::{CapturedFrame, PixelFormat}; +use crate::capture::{self, Capturer, FastSyntheticCapturer}; use crate::encode::{self, Codec}; use anyhow::{Context, Result}; use std::net::UdpSocket; @@ -55,48 +55,46 @@ fn run(cfg: StreamConfig, running: &AtomicBool) -> Result<()> { .context("connect client video endpoint")?; tracing::info!(%client, "video: client endpoint learned"); - let (w, h, fps) = (cfg.width, cfg.height, cfg.fps); + let use_portal = std::env::var("LUMEN_VIDEO_SOURCE").is_ok_and(|v| v == "portal"); + let mut capturer: Box = if use_portal { + tracing::info!("video source: portal desktop capture"); + capture::open_portal_monitor().context("open portal capturer")? + } else { + tracing::info!("video source: synthetic test pattern"); + Box::new(FastSyntheticCapturer::new(cfg.width, cfg.height)) + }; + + // The first frame establishes the authoritative size/format for the encoder. + let mut frame = capturer.next_frame().context("capture first frame")?; + if frame.width != cfg.width || frame.height != cfg.height { + tracing::warn!( + captured = ?(frame.width, frame.height), + negotiated = ?(cfg.width, cfg.height), + "captured size != negotiated size — Moonlight expects the negotiated size; resize the output" + ); + } let mut enc = encode::open_video( cfg.codec, - PixelFormat::Bgrx, - w, - h, - fps, + frame.format, + frame.width, + frame.height, + cfg.fps, cfg.bitrate_kbps as u64 * 1000, ) .context("open NVENC for stream")?; let mut pk = VideoPacketizer::new(cfg.packet_size); - let bpp = 4usize; - let row = w as usize * bpp; - let mut buf = vec![0u8; row * h as usize]; - let frame_interval = Duration::from_secs_f64(1.0 / fps as f64); + let frame_interval = Duration::from_secs_f64(1.0 / cfg.fps as f64); let mut frame_idx: u32 = 0; let mut sent_pkts: u64 = 0; + let stream_start = Instant::now(); while running.load(Ordering::SeqCst) { let tick = Instant::now(); - - // Fast synthetic test pattern: a pulsing grey field + a white band sweeping down. - let shade = (frame_idx % 256) as u8; - buf.fill(shade); - let band_h = (h as usize / 20).max(1); - let band_y = (frame_idx as usize * 6) % h as usize; - for y in band_y..(band_y + band_h).min(h as usize) { - buf[y * row..(y + 1) * row].fill(0xff); - } - - let frame = CapturedFrame { - width: w, - height: h, - pts_ns: frame_idx as u64 * 1_000_000_000 / fps as u64, - format: PixelFormat::Bgrx, - cpu_bytes: std::mem::take(&mut buf), - }; enc.submit(&frame).context("encoder submit")?; - buf = frame.cpu_bytes; // reclaim the buffer (no per-frame realloc) - let ts = (frame_idx as u64 * 90_000 / fps as u64) as u32; + // 90 kHz RTP timestamp from wall-clock, so a variable capture rate stays correct. + let ts = (stream_start.elapsed().as_secs_f64() * 90_000.0) as u32; let mut client_gone = false; while let Some(au) = enc.poll().context("encoder poll")? { let ft = if au.keyframe { @@ -121,13 +119,18 @@ fn run(cfg: StreamConfig, running: &AtomicBool) -> Result<()> { } frame_idx += 1; - if frame_idx % (fps.max(1) * 2) == 0 { + if frame_idx % (cfg.fps.max(1) * 2) == 0 { tracing::info!(frame_idx, sent_pkts, "video: streaming"); } - let elapsed = tick.elapsed(); - if elapsed < frame_interval { - std::thread::sleep(frame_interval - elapsed); + // Synthetic produces instantly, so pace it; the portal's next_frame() blocks at the + // capture rate and paces itself. + if !use_portal { + let elapsed = tick.elapsed(); + if elapsed < frame_interval { + std::thread::sleep(frame_interval - elapsed); + } } + frame = capturer.next_frame().context("capture frame")?; } Ok(()) }