feat: M2 P1.6 — audio (Opus + AES-CBC) and steady-rate video pacing
A stock Moonlight client now gets video + full input + AUDIO from the from-scratch GameStream host (verified live end-to-end on a macOS client). Audio (audio.rs, audio/linux.rs, gamestream/audio.rs): - Capture the default PipeWire sink's monitor (system output) as interleaved f32 stereo @ 48kHz via stream.capture.sink, on its own thread. - Opus-encode 5ms/240-sample stereo frames (RESTRICTED_LOWDELAY, CBR) and send as GameStream RTP audio: 12-byte BE RTP_PACKET (packetType 97, seq+1/pkt, timestamp += packetDuration, ssrc 0) on UDP 48000, after learning the client endpoint from its port-learning ping. - Encrypt the Opus payload with AES-128-CBC (PKCS7), key = launch rikey, IV = BE32(rikeyid + seq) in [0..4]. Like the control stream, modern Moonlight always decrypts audio regardless of the negotiated flags — plaintext makes it log "Failed to decrypt audio packet" and play silence (diagnosed from the client log). RTP header stays in the clear. Scheme cross-checked against Sunshine stream.cpp/crypto.cpp + moonlight AudioStream.c. - Pace each frame to its 5ms slot (PipeWire delivers ~1024-frame buffers) to avoid bursts the client's jitter buffer hears as glitches. LUMEN_AUDIO_GAIN applies optional linear gain for quiet sources. - DESCRIBE SDP advertises the stereo Opus config (a=fmtp:97 surround-params). Video (stream.rs): pace at a steady ≤60fps, re-encoding the last captured frame when the compositor produces none. wlroots only emits on damage, so a static or slow-updating desktop previously starved the client into a "network too slow" abort; an unchanged frame costs a near-empty P-frame. Adds a non-blocking Capturer::try_latest (portal drains to the freshest queued frame). Misc: serialize pipewire init across the video + audio capture threads (pwinit.rs, std::sync::Once) to avoid a concurrent pw_init race. Deps: opus, cbc; libopus-dev in bootstrap-ubuntu.sh. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -84,13 +84,23 @@ fn run(cfg: StreamConfig, running: &AtomicBool) -> Result<()> {
|
||||
.context("open NVENC for stream")?;
|
||||
let mut pk = VideoPacketizer::new(cfg.packet_size);
|
||||
|
||||
let frame_interval = Duration::from_secs_f64(1.0 / cfg.fps as f64);
|
||||
let mut frame_idx: u32 = 0;
|
||||
// Pace at a steady rate (capped at 60fps), re-encoding the last captured frame when the
|
||||
// compositor produced no new one. wlroots only emits frames on damage, so a static or
|
||||
// slow-updating desktop would otherwise starve the client into a "network too slow" abort.
|
||||
// Re-encoding an unchanged frame is cheap — NVENC emits a near-empty P-frame.
|
||||
let target_fps = cfg.fps.clamp(1, 60);
|
||||
let frame_interval = Duration::from_secs_f64(1.0 / target_fps as f64);
|
||||
let mut sent_pkts: u64 = 0;
|
||||
let mut fps_count: u32 = 0;
|
||||
let mut fps_t = Instant::now();
|
||||
let stream_start = Instant::now();
|
||||
|
||||
while running.load(Ordering::SeqCst) {
|
||||
let tick = Instant::now();
|
||||
// Advance to the freshest captured frame if one arrived; otherwise reuse the last.
|
||||
if let Some(f) = capturer.try_latest().context("capture frame")? {
|
||||
frame = f;
|
||||
}
|
||||
enc.submit(&frame).context("encoder submit")?;
|
||||
|
||||
// 90 kHz RTP timestamp from wall-clock, so a variable capture rate stays correct.
|
||||
@@ -118,19 +128,16 @@ fn run(cfg: StreamConfig, running: &AtomicBool) -> Result<()> {
|
||||
break;
|
||||
}
|
||||
|
||||
frame_idx += 1;
|
||||
if frame_idx % (cfg.fps.max(1) * 2) == 0 {
|
||||
tracing::info!(frame_idx, sent_pkts, "video: streaming");
|
||||
fps_count += 1;
|
||||
if fps_t.elapsed() >= Duration::from_secs(1) {
|
||||
tracing::info!(fps = fps_count, sent_pkts, "video: streaming");
|
||||
fps_count = 0;
|
||||
fps_t = Instant::now();
|
||||
}
|
||||
// Synthetic produces instantly, so pace it; the portal's next_frame() blocks at the
|
||||
// capture rate and paces itself.
|
||||
if !use_portal {
|
||||
let elapsed = tick.elapsed();
|
||||
if elapsed < frame_interval {
|
||||
std::thread::sleep(frame_interval - elapsed);
|
||||
}
|
||||
let elapsed = tick.elapsed();
|
||||
if elapsed < frame_interval {
|
||||
std::thread::sleep(frame_interval - elapsed);
|
||||
}
|
||||
frame = capturer.next_frame().context("capture frame")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user