feat(m2): real desktop capture in the video stream (portal → Moonlight)

Wire M0's portal desktop capture into the GameStream video plane: with
LUMEN_VIDEO_SOURCE=portal the stream captures the headless wlroots desktop
(PipeWire RGB) instead of the synthetic pattern, opens NVENC from the first
captured frame's format/size, and streams it. Verified live: a stock Moonlight
client shows the real 5120×1440 desktop at ~42 fps (release build).

- capture.rs: FastSyntheticCapturer (cheap fill pattern, real-time at 5K) so both
  sources share the Capturer trait
- stream.rs: source select (portal | synthetic), encoder opened from the first
  frame, wall-clock 90 kHz RTP timestamps (correct under a variable capture rate)

Note: the CPU-copy RGB→rgb0 path caps ~42 fps at 5K (single-threaded); dmabuf
zero-copy is the deferred optimization (plan §9).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-09 07:51:49 +00:00
parent de60650ed3
commit c8491af893
2 changed files with 82 additions and 36 deletions
+43
View File
@@ -121,6 +121,49 @@ impl Capturer for SyntheticCapturer {
} }
} }
/// A cheap moving test pattern (BGRx) for the streaming path: a pulsing field + a white band
/// sweeping down, generated with whole-buffer `fill`s so it stays real-time even at 5K.
pub struct FastSyntheticCapturer {
width: u32,
height: u32,
frame_idx: u64,
buf: Vec<u8>,
}
impl FastSyntheticCapturer {
pub fn new(width: u32, height: u32) -> Self {
assert!(width > 0 && height > 0);
FastSyntheticCapturer {
width,
height,
frame_idx: 0,
buf: vec![0u8; width as usize * height as usize * 4],
}
}
}
impl Capturer for FastSyntheticCapturer {
fn next_frame(&mut self) -> Result<CapturedFrame> {
let (w, h) = (self.width as usize, self.height as usize);
let row = w * 4;
let shade = (self.frame_idx % 256) as u8;
self.buf.fill(shade);
let band_h = (h / 20).max(1);
let band_y = (self.frame_idx as usize * 6) % h;
for y in band_y..(band_y + band_h).min(h) {
self.buf[y * row..(y + 1) * row].fill(0xff);
}
self.frame_idx += 1;
Ok(CapturedFrame {
width: self.width,
height: self.height,
pts_ns: 0,
format: PixelFormat::Bgrx,
cpu_bytes: self.buf.clone(),
})
}
}
/// Open a live capturer for a client-sized monitor via the xdg ScreenCast portal /// Open a live capturer for a client-sized monitor via the xdg ScreenCast portal
/// (`ashpd`) → PipeWire (`pipewire`). Implemented in the `linux` submodule. /// (`ashpd`) → PipeWire (`pipewire`). Implemented in the `linux` submodule.
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
+39 -36
View File
@@ -1,11 +1,11 @@
//! The video data plane: on RTSP PLAY, learn the client's UDP endpoint (it pings the video //! The video data plane: on RTSP PLAY, learn the client's UDP endpoint (it pings the video
//! port), then run capture → NVENC encode → [`VideoPacketizer`] → UDP send. P1.3 uses a fast //! port), then run capture → NVENC encode → [`VideoPacketizer`] → UDP send. The source is
//! synthetic test pattern (proves the wire format with no compositor); swapping in real //! either real portal desktop capture (`LUMEN_VIDEO_SOURCE=portal`, the M0 PipeWire path) or
//! portal desktop capture is a one-line source change. Runs on its own native thread. //! a synthetic test pattern (default). Runs on its own native thread.
use super::video::{FrameType, VideoPacketizer}; use super::video::{FrameType, VideoPacketizer};
use super::VIDEO_PORT; use super::VIDEO_PORT;
use crate::capture::{CapturedFrame, PixelFormat}; use crate::capture::{self, Capturer, FastSyntheticCapturer};
use crate::encode::{self, Codec}; use crate::encode::{self, Codec};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use std::net::UdpSocket; use std::net::UdpSocket;
@@ -55,48 +55,46 @@ fn run(cfg: StreamConfig, running: &AtomicBool) -> Result<()> {
.context("connect client video endpoint")?; .context("connect client video endpoint")?;
tracing::info!(%client, "video: client endpoint learned"); tracing::info!(%client, "video: client endpoint learned");
let (w, h, fps) = (cfg.width, cfg.height, cfg.fps); let use_portal = std::env::var("LUMEN_VIDEO_SOURCE").is_ok_and(|v| v == "portal");
let mut capturer: Box<dyn Capturer> = if use_portal {
tracing::info!("video source: portal desktop capture");
capture::open_portal_monitor().context("open portal capturer")?
} else {
tracing::info!("video source: synthetic test pattern");
Box::new(FastSyntheticCapturer::new(cfg.width, cfg.height))
};
// The first frame establishes the authoritative size/format for the encoder.
let mut frame = capturer.next_frame().context("capture first frame")?;
if frame.width != cfg.width || frame.height != cfg.height {
tracing::warn!(
captured = ?(frame.width, frame.height),
negotiated = ?(cfg.width, cfg.height),
"captured size != negotiated size — Moonlight expects the negotiated size; resize the output"
);
}
let mut enc = encode::open_video( let mut enc = encode::open_video(
cfg.codec, cfg.codec,
PixelFormat::Bgrx, frame.format,
w, frame.width,
h, frame.height,
fps, cfg.fps,
cfg.bitrate_kbps as u64 * 1000, cfg.bitrate_kbps as u64 * 1000,
) )
.context("open NVENC for stream")?; .context("open NVENC for stream")?;
let mut pk = VideoPacketizer::new(cfg.packet_size); let mut pk = VideoPacketizer::new(cfg.packet_size);
let bpp = 4usize; let frame_interval = Duration::from_secs_f64(1.0 / cfg.fps as f64);
let row = w as usize * bpp;
let mut buf = vec![0u8; row * h as usize];
let frame_interval = Duration::from_secs_f64(1.0 / fps as f64);
let mut frame_idx: u32 = 0; let mut frame_idx: u32 = 0;
let mut sent_pkts: u64 = 0; let mut sent_pkts: u64 = 0;
let stream_start = Instant::now();
while running.load(Ordering::SeqCst) { while running.load(Ordering::SeqCst) {
let tick = Instant::now(); let tick = Instant::now();
// Fast synthetic test pattern: a pulsing grey field + a white band sweeping down.
let shade = (frame_idx % 256) as u8;
buf.fill(shade);
let band_h = (h as usize / 20).max(1);
let band_y = (frame_idx as usize * 6) % h as usize;
for y in band_y..(band_y + band_h).min(h as usize) {
buf[y * row..(y + 1) * row].fill(0xff);
}
let frame = CapturedFrame {
width: w,
height: h,
pts_ns: frame_idx as u64 * 1_000_000_000 / fps as u64,
format: PixelFormat::Bgrx,
cpu_bytes: std::mem::take(&mut buf),
};
enc.submit(&frame).context("encoder submit")?; enc.submit(&frame).context("encoder submit")?;
buf = frame.cpu_bytes; // reclaim the buffer (no per-frame realloc)
let ts = (frame_idx as u64 * 90_000 / fps as u64) as u32; // 90 kHz RTP timestamp from wall-clock, so a variable capture rate stays correct.
let ts = (stream_start.elapsed().as_secs_f64() * 90_000.0) as u32;
let mut client_gone = false; let mut client_gone = false;
while let Some(au) = enc.poll().context("encoder poll")? { while let Some(au) = enc.poll().context("encoder poll")? {
let ft = if au.keyframe { let ft = if au.keyframe {
@@ -121,13 +119,18 @@ fn run(cfg: StreamConfig, running: &AtomicBool) -> Result<()> {
} }
frame_idx += 1; frame_idx += 1;
if frame_idx % (fps.max(1) * 2) == 0 { if frame_idx % (cfg.fps.max(1) * 2) == 0 {
tracing::info!(frame_idx, sent_pkts, "video: streaming"); tracing::info!(frame_idx, sent_pkts, "video: streaming");
} }
let elapsed = tick.elapsed(); // Synthetic produces instantly, so pace it; the portal's next_frame() blocks at the
if elapsed < frame_interval { // capture rate and paces itself.
std::thread::sleep(frame_interval - elapsed); if !use_portal {
let elapsed = tick.elapsed();
if elapsed < frame_interval {
std::thread::sleep(frame_interval - elapsed);
}
} }
frame = capturer.next_frame().context("capture frame")?;
} }
Ok(()) Ok(())
} }