feat: M0 capture→encode pipeline + M2 GameStream host (pairing, RTSP, video)
M0 (lumen-host) — verified on NVIDIA RTX 5070 Ti / Ubuntu 25.10: headless wlroots → xdg ScreenCast portal → PipeWire → NVENC HEVC → playable file, with each access unit round-tripped through a lumen_core host↔client Session (FEC + packetize + reassemble), 0 mismatches. - capture.rs: SyntheticCapturer + portal capture (ashpd 0.13 + pipewire 0.9), format-aware - encode/linux.rs: NVENC via ffmpeg-next 7 (BGRx/RGB → rgb0, no host-side swscale) - m0.rs: capture→encode→file + lumen-core loopback verification M2 P1 (lumen-host gamestream/) — a stock Moonlight client pairs + launches, verified live: - mDNS _nvstream._tcp + nvhttp /serverinfo (HTTP 47989, mutual-TLS HTTPS 47984) - 4-phase pairing: PIN→AES-128-ECB / SHA-256 / RSA-PKCS1v15 / X.509, custom rustls ClientCertVerifier for the mutual-TLS pairchallenge - /applist, /launch (rikey/rikeyid/mode), hand-rolled RTSP (OPTIONS/DESCRIBE/SETUP×3/ ANNOUNCE/PLAY, one-request-per-TCP-connection per moonlight-common-c's read-to-EOF) - video.rs: GameStream RTP + NV_VIDEO_PACKET wire packetizer, data-shards-only (0% FEC, clean-LAN), unit-tested (single/multi-block) Docs: docs/m2-plan.md (phased plan) + docs/research/ (ground-truth protocol spec). Bootstrap/setup updated for the verified path (libnvidia-gl, render/video groups, GPU EGL, pipewire 0.9). Workspace clippy-clean, tests green. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,15 +1,48 @@
|
||||
//! Frame capture (plan §7). On Linux: a PipeWire ScreenCast portal stream delivering
|
||||
//! dmabuf frames with no copy to the CPU. The encoder imports the dmabuf directly.
|
||||
//! Frame capture (plan §7). On Linux: a PipeWire ScreenCast portal stream. M0 uses the
|
||||
//! CPU-copy fallback (the portal delivers a CPU buffer; the encoder uploads it to the GPU
|
||||
//! internally). Zero-copy dmabuf→NVENC import is deferred (plan §9 risk).
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
/// A captured frame. For zero-copy the real type wraps a dmabuf fd + modifier; the CPU
|
||||
/// buffer is only a fallback path (plan §9 risk: per-GPU dmabuf import quirks).
|
||||
/// Packed pixel layout of a [`CapturedFrame`]. The ScreenCast portal negotiates the
|
||||
/// format; on wlroots it is commonly packed `RGB` (3 bytes/pixel). The encoder maps these
|
||||
/// to an NVENC-accepted input format (`rgb0`/`bgr0`/`rgba`/`bgra`), expanding 3→4 bytes
|
||||
/// where needed — no host-side colour conversion.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum PixelFormat {
|
||||
/// `[B,G,R,x]`, 4 bpp.
|
||||
Bgrx,
|
||||
/// `[R,G,B,x]`, 4 bpp.
|
||||
Rgbx,
|
||||
/// `[B,G,R,A]`, 4 bpp.
|
||||
Bgra,
|
||||
/// `[R,G,B,A]`, 4 bpp.
|
||||
Rgba,
|
||||
/// `[R,G,B]`, 3 bpp.
|
||||
Rgb,
|
||||
/// `[B,G,R]`, 3 bpp.
|
||||
Bgr,
|
||||
}
|
||||
|
||||
impl PixelFormat {
|
||||
pub fn bytes_per_pixel(self) -> usize {
|
||||
match self {
|
||||
PixelFormat::Rgb | PixelFormat::Bgr => 3,
|
||||
_ => 4,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A captured frame. For zero-copy the real type would wrap a dmabuf fd + modifier; the
|
||||
/// CPU buffer is the M0 fallback path (plan §9 risk: per-GPU dmabuf import quirks).
|
||||
pub struct CapturedFrame {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub pts_ns: u64,
|
||||
/// Fallback CPU pixels (empty when a dmabuf is used).
|
||||
/// Pixel layout of `cpu_bytes`.
|
||||
pub format: PixelFormat,
|
||||
/// Tightly-packed pixels in `format`, `width * height * format.bytes_per_pixel()`
|
||||
/// bytes (no row padding).
|
||||
pub cpu_bytes: Vec<u8>,
|
||||
}
|
||||
|
||||
@@ -19,14 +52,86 @@ pub trait Capturer: Send {
|
||||
fn next_frame(&mut self) -> Result<CapturedFrame>;
|
||||
}
|
||||
|
||||
/// Open a capturer for a PipeWire node id (from the ScreenCast portal).
|
||||
pub fn open_pipewire(_node_id: u32) -> Result<Box<dyn Capturer>> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
anyhow::bail!("pipewire capture not yet implemented (M0)")
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
anyhow::bail!("capture requires Linux + PipeWire")
|
||||
/// A deterministic moving test pattern (BGRx). Lets M0 exercise the encode → file →
|
||||
/// `lumen_core` path with no live capture session, and produces obviously non-static
|
||||
/// content (a sweeping bar + animated gradient) so the encoded output is verifiable.
|
||||
pub struct SyntheticCapturer {
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
frame_idx: u64,
|
||||
buf: Vec<u8>,
|
||||
}
|
||||
|
||||
impl SyntheticCapturer {
|
||||
const BPP: usize = 4; // emits BGRx
|
||||
|
||||
pub fn new(width: u32, height: u32, fps: u32) -> Self {
|
||||
assert!(width > 0 && height > 0 && fps > 0);
|
||||
let buf = vec![0u8; width as usize * height as usize * Self::BPP];
|
||||
SyntheticCapturer {
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
frame_idx: 0,
|
||||
buf,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Capturer for SyntheticCapturer {
|
||||
fn next_frame(&mut self) -> Result<CapturedFrame> {
|
||||
let w = self.width as usize;
|
||||
let h = self.height as usize;
|
||||
let bpp = Self::BPP;
|
||||
let t = self.frame_idx;
|
||||
// A vertical bar sweeps left→right once every ~2s; the background is a gradient
|
||||
// whose phase advances each frame, so every pixel changes frame-to-frame.
|
||||
let bar_x = ((t * w as u64) / (self.fps as u64 * 2)) % w as u64;
|
||||
let phase = (t % 256) as usize;
|
||||
for y in 0..h {
|
||||
let row = y * w * bpp;
|
||||
for x in 0..w {
|
||||
let i = row + x * bpp;
|
||||
let on_bar = (x as u64).abs_diff(bar_x) < 8;
|
||||
// BGRx byte order: [B, G, R, x]
|
||||
self.buf[i] = if on_bar {
|
||||
255
|
||||
} else {
|
||||
((x + phase) & 0xff) as u8
|
||||
};
|
||||
self.buf[i + 1] = if on_bar {
|
||||
255
|
||||
} else {
|
||||
((y + phase) & 0xff) as u8
|
||||
};
|
||||
self.buf[i + 2] = if on_bar { 255 } else { ((x + y) & 0xff) as u8 };
|
||||
self.buf[i + 3] = 0;
|
||||
}
|
||||
}
|
||||
let pts_ns = self.frame_idx * 1_000_000_000 / self.fps as u64;
|
||||
self.frame_idx += 1;
|
||||
Ok(CapturedFrame {
|
||||
width: self.width,
|
||||
height: self.height,
|
||||
pts_ns,
|
||||
format: PixelFormat::Bgrx,
|
||||
cpu_bytes: self.buf.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Open a live capturer for a client-sized monitor via the xdg ScreenCast portal
|
||||
/// (`ashpd`) → PipeWire (`pipewire`). Implemented in the `linux` submodule.
|
||||
#[cfg(target_os = "linux")]
|
||||
pub fn open_portal_monitor() -> Result<Box<dyn Capturer>> {
|
||||
linux::PortalCapturer::open().map(|c| Box::new(c) as Box<dyn Capturer>)
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub fn open_portal_monitor() -> Result<Box<dyn Capturer>> {
|
||||
anyhow::bail!("portal capture requires Linux (xdg-desktop-portal + PipeWire)")
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
|
||||
Reference in New Issue
Block a user