From ffc0b07b461ee48fb85389e91ed95f1bf5687f12 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Wed, 1 Jul 2026 23:13:39 +0000 Subject: [PATCH] feat(protocol,host): negotiate video codec + add a GPU-less software (openh264) encode path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of codec negotiation, and the Linux software H.264 encode path it unblocks. **Codec negotiation (core `quic`):** - `Hello.video_codecs` (bitfield: CODEC_H264/HEVC/AV1) — the client advertises what it can decode; appended as a trailing byte (older client → 0 = HEVC-only, back-compat). - `Welcome.codec` — the single codec the host resolved and will emit; trailing byte (older host → HEVC). - `resolve_codec(client, host_capable)` picks the shared codec (precedence HEVC > AV1 > H.264) or `None` → the host refuses honestly rather than sending an undecodable stream. - Roundtrip + back-compat tests; cbindgen exports the CODEC_* constants. **Software encoder (host):** - The openh264 `OpenH264Encoder` (was Windows-only) is now built on Linux too — it's platform-agnostic (consumes CPU RGB `CapturedFrame`s, statically-bundled openh264). `openh264` moved to the shared linux+windows Cargo target. - `PUNKTFUNK_ENCODER=software` selects it: `open_video` gains a `software` branch (H.264 only), and `session_plan::resolve_encoder` / `capture::gpu_encode` resolve `EncoderBackend::Software` → `output_format().gpu = false`, so the portal capturer delivers CPU RGB. Explicit-only (auto never picks it — a box with a dead driver still has /dev/nvidiactl and would mis-resolve NVENC). **Host codec resolution (`punktfunk1`):** - The native path no longer hardcodes HEVC: it resolves the codec from the client's advertised set ∩ the host's capability (`Codec::host_wire_caps`: software→H.264, else HEVC), threads it through `SessionPlan.codec`, and opens the encoder + validates reconfigures at that codec. A software host + HEVC-only client is refused with a clear error. - 4:4:4 is gated on HEVC (it's HEVC-only). **Probe:** advertises H264|HEVC|AV1 and logs the resolved codec. Validated on the GPU-less dev box: negotiation is live end-to-end (probe advertises 0x07 → host resolves H.264 → Welcome reports it → plan = Software/H264), and the openh264 unit test (CPU RGB → AnnexB IDR) now runs on Linux. Full capture→encode still needs a GPU on this box — every compositor screencast path (KWin GL, gamescope VK_EXT_physical_device_drm, wlroots EGL) requires one; software render (llvmpipe/pixman) can't be captured — so this box exercises negotiation + encoder, not live capture. The software path unblocks GPU-less-*encode* boxes that still have a display GPU. Phase 2 (clients advertising real codecs + decoding per Welcome.codec) is a follow-up. Co-Authored-By: Claude Opus 4.8 --- clients/probe/src/main.rs | 18 ++ crates/punktfunk-core/src/client.rs | 4 + crates/punktfunk-core/src/quic.rs | 175 +++++++++++++++++- crates/punktfunk-host/Cargo.toml | 7 +- crates/punktfunk-host/src/capture.rs | 8 +- crates/punktfunk-host/src/encode.rs | 54 +++++- .../src/encode/{windows => }/sw.rs | 0 crates/punktfunk-host/src/punktfunk1.rs | 54 +++++- crates/punktfunk-host/src/session_plan.rs | 23 ++- include/punktfunk_core.h | 18 ++ 10 files changed, 331 insertions(+), 30 deletions(-) rename crates/punktfunk-host/src/encode/{windows => }/sw.rs (100%) diff --git a/clients/probe/src/main.rs b/clients/probe/src/main.rs index cf1f7c7..5faaa4f 100644 --- a/clients/probe/src/main.rs +++ b/clients/probe/src/main.rs @@ -231,6 +231,16 @@ fn now_ns() -> u64 { .unwrap_or(0) } +/// Human name for the negotiated `Welcome::codec` (also the natural `--out` file extension). The +/// bitstream is dumped verbatim, so an H.264 software-host session should be saved as `.h264`. +fn codec_ext(codec: u8) -> &'static str { + match codec { + punktfunk_core::quic::CODEC_H264 => "h264", + punktfunk_core::quic::CODEC_AV1 => "av1", + _ => "h265", + } +} + fn main() { tracing_subscriber::fmt() .with_env_filter( @@ -411,6 +421,13 @@ async fn session(args: Args) -> Result<()> { // `--audio-channels` (default stereo); the probe multistream-decodes + validates the // host's frames to exercise the surround encode path headlessly. audio_channels: args.audio_channels, + // The probe just dumps the bitstream (no decode), so it advertises every codec — HEVC + // (the host default) AND H.264 (so it can drive a GPU-less software host, + // `PUNKTFUNK_ENCODER=software`) AND AV1. The host picks one and reports it in + // `Welcome::codec`; the dump extension follows that. + video_codecs: punktfunk_core::quic::CODEC_H264 + | punktfunk_core::quic::CODEC_HEVC + | punktfunk_core::quic::CODEC_AV1, } .encode(), ) @@ -429,6 +446,7 @@ async fn session(args: Args) -> Result<()> { hdr = welcome.color.is_hdr(), chroma_444 = welcome.chroma_format == punktfunk_core::quic::CHROMA_IDC_444, chroma_format_idc = welcome.chroma_format, + codec = codec_ext(welcome.codec), "session offer" ); diff --git a/crates/punktfunk-core/src/client.rs b/crates/punktfunk-core/src/client.rs index 1daf88a..02d28e5 100644 --- a/crates/punktfunk-core/src/client.rs +++ b/crates/punktfunk-core/src/client.rs @@ -789,6 +789,10 @@ async fn worker_main(args: WorkerArgs) { video_caps, // Requested surround channel count; the host echoes the resolved value in Welcome. audio_channels, + // Phase 1: the embeddable clients decode HEVC (their decoders are still HEVC-wired), + // so advertise HEVC-only until Phase 2 threads real per-client codec caps through the + // connect ABI and switches decoders on `Welcome::codec`. + video_codecs: crate::quic::CODEC_HEVC, } .encode(), ) diff --git a/crates/punktfunk-core/src/quic.rs b/crates/punktfunk-core/src/quic.rs index a6a4bca..06bb246 100644 --- a/crates/punktfunk-core/src/quic.rs +++ b/crates/punktfunk-core/src/quic.rs @@ -86,6 +86,14 @@ pub struct Hello { /// deterministic offset. Omitted by older clients / when `2` (decodes to `2`, i.e. stereo) so /// the stereo wire form stays byte-identical to the pre-surround build. pub audio_channels: u8, + /// Which video codecs the client can decode — a bitfield of [`CODEC_H264`] / [`CODEC_HEVC`] / + /// [`CODEC_AV1`]. The host picks one it can also produce (see [`resolve_codec`]) and reports it in + /// [`Welcome::codec`]; a client that only reaches a GPU-less **software** host must set + /// [`CODEC_H264`] (openh264 emits H.264). Appended after `audio_channels` as a single trailing + /// byte (forcing the video_caps/audio_channels placeholders when present). Omitted by older + /// clients (decodes to `0`, which [`resolve_codec`] treats as HEVC-only — every pre-negotiation + /// build decoded HEVC). + pub video_codecs: u8, } /// [`Hello::video_caps`] bit: the client can decode a 10-bit (Main10) HEVC stream. @@ -100,6 +108,37 @@ pub const VIDEO_CAP_HDR: u8 = 0x02; /// chroma decision, bit depth is a depth decision; the two may combine where the hardware allows). pub const VIDEO_CAP_444: u8 = 0x04; +/// [`Hello::video_codecs`] bit: the client can decode H.264 / AVC. The GPU-less **software** +/// encode path (openh264) emits H.264, so a client that wants to stream from a software host MUST +/// advertise this. +pub const CODEC_H264: u8 = 0x01; +/// [`Hello::video_codecs`] bit: the client can decode H.265 / HEVC — the default every existing +/// build produces and decodes (a peer that omits [`Hello::video_codecs`] is treated as HEVC-only). +pub const CODEC_HEVC: u8 = 0x02; +/// [`Hello::video_codecs`] bit: the client can decode AV1. +pub const CODEC_AV1: u8 = 0x04; + +/// Resolve which single codec the host will emit, from the client's advertised [`Hello::video_codecs`] +/// bitfield (`0` = an older client, treated as HEVC-only) intersected with what the host's chosen +/// encoder can produce (`host_capable`, also a bitfield). Precedence when several are shared: +/// **HEVC > AV1 > H.264** (HEVC is the established, best-tested path; H.264 is the compatibility / +/// software floor). Returns the single-bit codec value, or `None` when the two share nothing — the +/// caller then refuses the session with a clear error rather than emitting a stream the client can't +/// decode. +pub fn resolve_codec(client_codecs: u8, host_capable: u8) -> Option { + // An older client (no codec byte) decodes HEVC — the only codec every pre-negotiation build sent. + let client = if client_codecs == 0 { + CODEC_HEVC + } else { + client_codecs + }; + let shared = client & host_capable; + // Precedence: HEVC > AV1 > H.264. + [CODEC_HEVC, CODEC_AV1, CODEC_H264] + .into_iter() + .find(|&c| shared & c != 0) +} + /// HEVC `chroma_format_idc` for 4:2:0 — what every pre-4:4:4 build produced and the back-compat /// default when a peer omits [`Welcome::chroma_format`]. pub const CHROMA_IDC_420: u8 = 1; @@ -235,6 +274,12 @@ pub struct Welcome { /// request — so an older host that omits the byte (→ `2`) always yields working stereo. Appended /// after `chroma_format` as a single trailing byte. pub audio_channels: u8, + /// The single video codec the host resolved and **will** emit — [`CODEC_H264`], [`CODEC_HEVC`] + /// (default), or [`CODEC_AV1`] — from [`resolve_codec`] over the client's [`Hello::video_codecs`] + /// and the host encoder's capability. The client builds its decoder from THIS (never assuming + /// HEVC). Appended after `audio_channels` as a single trailing byte; an older host that omits it + /// decodes to [`CODEC_HEVC`] (every pre-negotiation host sent HEVC). + pub codec: u8, } /// `client → host`: data plane is bound, begin streaming. @@ -671,7 +716,8 @@ impl Hello { // present (video_caps non-zero / audio_channels not stereo) the name/launch length bytes // AND the video_caps byte must still be emitted (0 / 0) so the later byte lands at a // deterministic offset — the same discipline `launch` already imposes on `name`. - let need_placeholders = self.video_caps != 0 || self.audio_channels != 2; + let need_placeholders = + self.video_caps != 0 || self.audio_channels != 2 || self.video_codecs != 0; match (&self.name, &self.launch) { (None, None) if !need_placeholders => {} (name, _) => { @@ -686,15 +732,19 @@ impl Hello { b.push(l.len() as u8); b.extend_from_slice(l.as_bytes()); } - // video_caps: single trailing byte. Emitted when non-zero OR when audio_channels follows - // (so audio_channels lands at a deterministic offset right after it). - if self.video_caps != 0 || self.audio_channels != 2 { + // video_caps: single trailing byte. Emitted when non-zero OR when a later field follows (so + // that field lands at a deterministic offset right after it). + if self.video_caps != 0 || self.audio_channels != 2 || self.video_codecs != 0 { b.push(self.video_caps); } - // audio_channels: single trailing byte. Last field; omitted when stereo (default). - if self.audio_channels != 2 { + // audio_channels: single trailing byte. Emitted when non-stereo OR when video_codecs follows. + if self.audio_channels != 2 || self.video_codecs != 0 { b.push(self.audio_channels); } + // video_codecs: single trailing byte. Last field; omitted when `0` (older client → HEVC-only). + if self.video_codecs != 0 { + b.push(self.video_codecs); + } b } @@ -766,6 +816,15 @@ impl Hello { let video_caps_off = launch_off + 1 + launch_len; crate::audio::normalize_channels(b.get(video_caps_off + 1).copied().unwrap_or(2)) }, + // Optional trailing video-codecs byte, one past audio_channels. Absent on an older client + // → `0` (which `resolve_codec` treats as HEVC-only). + video_codecs: { + let name_len = b.get(26).copied().unwrap_or(0) as usize; + let launch_off = 27 + name_len; + let launch_len = b.get(launch_off).copied().unwrap_or(0) as usize; + let video_caps_off = launch_off + 1 + launch_len; + b.get(video_caps_off + 2).copied().unwrap_or(0) + }, }) } } @@ -803,6 +862,8 @@ impl Welcome { b.push(self.chroma_format); // Audio channel count at offset 65 — older clients stop before this → stereo (2). b.push(self.audio_channels); + // Resolved video codec at offset 66 — older clients stop before this → HEVC. + b.push(self.codec); b } @@ -811,8 +872,8 @@ impl Welcome { // scheme[22] pct[23] max_data[24..26] shard[26..28] encrypt[28] key[29..45] // salt[45..49] frames[49..53] compositor[53] gamepad[54] bitrate_kbps[55..59] // bit_depth[59] color.primaries[60] color.transfer[61] color.matrix[62] color.range[63] - // chroma_format[64] audio_channels[65] (everything from compositor on is an optional - // trailing byte; an older host stops earlier). + // chroma_format[64] audio_channels[65] codec[66] (everything from compositor on is an + // optional trailing byte; an older host stops earlier). if b.len() < 53 || &b[0..4] != MAGIC { return Err(PunktfunkError::InvalidArg("bad Welcome")); } @@ -878,6 +939,13 @@ impl Welcome { // Optional trailing audio-channel byte — absent on an older host → stereo. Any // non-{6,8} value normalizes to stereo so a corrupt byte never builds a bad decoder. audio_channels: crate::audio::normalize_channels(b.get(65).copied().unwrap_or(2)), + // Optional trailing codec byte — absent on an older host (or an unknown value) → HEVC, + // the codec every pre-negotiation host emitted. + codec: match b.get(66).copied() { + Some(CODEC_H264) => CODEC_H264, + Some(CODEC_AV1) => CODEC_AV1, + _ => CODEC_HEVC, + }, }) } @@ -1950,10 +2018,93 @@ mod tests { color: ColorInfo::HDR10_BT2020_PQ, chroma_format: CHROMA_IDC_444, audio_channels: 2, + codec: CODEC_H264, // exercise a non-default codec through the roundtrip }; assert_eq!(Welcome::decode(&w.encode()).unwrap(), w); } + #[test] + fn codec_negotiation_and_back_compat() { + // resolve_codec precedence (HEVC > AV1 > H.264) and the no-shared-codec refusal. + assert_eq!( + resolve_codec(CODEC_H264 | CODEC_HEVC, CODEC_HEVC | CODEC_AV1), + Some(CODEC_HEVC) + ); + assert_eq!( + resolve_codec(CODEC_H264 | CODEC_AV1, CODEC_AV1 | CODEC_H264), + Some(CODEC_AV1) + ); + assert_eq!(resolve_codec(CODEC_H264, CODEC_H264), Some(CODEC_H264)); + // A software host (H.264 only) + an HEVC-only client share nothing → refuse. + assert_eq!(resolve_codec(CODEC_HEVC, CODEC_H264), None); + // An older client (0 = no codec byte) is treated as HEVC-only. + assert_eq!(resolve_codec(0, CODEC_HEVC | CODEC_H264), Some(CODEC_HEVC)); + assert_eq!(resolve_codec(0, CODEC_H264), None); + + // A Hello advertising codecs roundtrips, and the wire form of a codec-only Hello decodes on + // a build that ignores the trailing byte (back-compat: extra bytes are skipped). + let h = Hello { + abi_version: 2, + mode: Mode { + width: 1280, + height: 720, + refresh_hz: 60, + }, + compositor: CompositorPref::Auto, + gamepad: GamepadPref::Auto, + bitrate_kbps: 0, + name: None, + launch: None, + video_caps: 0, + audio_channels: 2, // stereo — forces the video_caps/audio_channels placeholders + video_codecs: CODEC_H264 | CODEC_HEVC, + }; + let enc = h.encode(); + assert_eq!( + Hello::decode(&enc).unwrap().video_codecs, + CODEC_H264 | CODEC_HEVC + ); + // A pre-codec Hello (no trailing codec byte) decodes to 0 → HEVC-only via resolve_codec. + let legacy = &enc[..enc.len() - 1]; // drop the codec byte (it was the last field) + assert_eq!(Hello::decode(legacy).unwrap().video_codecs, 0); + + // A pre-codec Welcome (no codec byte) decodes to HEVC. + let mut w = Welcome::decode( + &Welcome { + abi_version: 2, + udp_port: 1, + mode: h.mode, + fec: FecConfig { + scheme: FecScheme::Gf16, + fec_percent: 0, + max_data_per_block: 1024, + }, + shard_payload: 1024, + encrypt: false, + key: [0; 16], + salt: [0; 4], + frames: 0, + compositor: CompositorPref::Auto, + gamepad: GamepadPref::Auto, + bitrate_kbps: 0, + bit_depth: 8, + color: ColorInfo::SDR_BT709, + chroma_format: CHROMA_IDC_420, + audio_channels: 2, + codec: CODEC_H264, + } + .encode(), + ) + .unwrap(); + assert_eq!(w.codec, CODEC_H264); + w.codec = CODEC_HEVC; + let wenc = w.encode(); + assert_eq!( + Welcome::decode(&wenc[..wenc.len() - 1]).unwrap().codec, + CODEC_HEVC + ); + } + #[test] fn hdr_meta_datagram_roundtrip_and_truncation() { let m = HdrMeta { @@ -1993,6 +2144,7 @@ mod tests { launch: Some("steam:570".into()), video_caps: VIDEO_CAP_10BIT, audio_channels: 2, + video_codecs: CODEC_H264 | CODEC_HEVC, // exercise the codec bitfield roundtrip }; assert_eq!(Hello::decode(&h.encode()).unwrap(), h); let s = Start { @@ -2073,6 +2225,7 @@ mod tests { launch: None, video_caps: 0, audio_channels: 2, + video_codecs: 0, }; let enc = h.encode(); assert_eq!(enc.len(), 26); @@ -2114,9 +2267,10 @@ mod tests { color: ColorInfo::HDR10_BT2020_PQ, chroma_format: CHROMA_IDC_444, audio_channels: 6, // 5.1 — exercises the non-default trailing byte + codec: CODEC_HEVC, }; let wenc = w.encode(); - assert_eq!(wenc.len(), 66); // 60 base + 4 colour + 1 chroma + 1 audio-channels byte + assert_eq!(wenc.len(), 67); // 60 base + 4 colour + 1 chroma + 1 audio-channels + 1 codec byte let legacy_w = Welcome::decode(&wenc[..53]).unwrap(); assert_eq!(legacy_w.compositor, CompositorPref::Auto); assert_eq!(legacy_w.gamepad, GamepadPref::Auto); @@ -2177,6 +2331,7 @@ mod tests { launch: None, video_caps: 0, audio_channels: 2, + video_codecs: 0, }; let enc = base.encode(); assert_eq!( @@ -2225,6 +2380,7 @@ mod tests { launch: None, video_caps: 0, audio_channels: 2, + video_codecs: 0, }; // launch alone (no name): a zero-length name placeholder keeps the offset deterministic. let with_launch = Hello { @@ -2432,6 +2588,7 @@ mod tests { launch: None, video_caps: 0, audio_channels: 2, + video_codecs: 0, } .encode(); assert!(PairRequest::decode(&h).is_err(), "abi {abi} parsed as pair"); diff --git a/crates/punktfunk-host/Cargo.toml b/crates/punktfunk-host/Cargo.toml index d24d941..dee214f 100644 --- a/crates/punktfunk-host/Cargo.toml +++ b/crates/punktfunk-host/Cargo.toml @@ -71,6 +71,10 @@ tempfile = "3" # MSVC too (needs CMake + NASM, both on the box). Both platforms that have an audio-capture backend. [target.'cfg(any(target_os = "linux", target_os = "windows"))'.dependencies] opus = "0.3" +# Software H.264 encoder — the GPU-less encode path on both Linux and Windows (and a fallback when no +# hardware encoder is available). The default `source` feature statically compiles OpenH264 (BSD-2) — +# no system lib, builds on MSVC; nasm on PATH adds the SIMD fast path. +openh264 = "0.9" [target.'cfg(target_os = "linux")'.dependencies] # `screencast` gates the ScreenCast portal module; `remote_desktop` adds the RemoteDesktop @@ -198,9 +202,6 @@ winreg = "0.56" # Parse each Xbox/Game-Pass game's MicrosoftGame.config (GDK manifest XML) for the Xbox store # provider — a small read-only DOM is all we need (Identity/Executable/ShellVisuals/StoreId). roxmltree = "0.21" -# Software H.264 encoder (GPU-less path + NVENC fallback). The default `source` feature statically -# compiles OpenH264 (BSD-2) — no system lib, builds on MSVC; nasm on PATH adds the SIMD fast path. -openh264 = "0.9" # WASAPI loopback audio capture (default render endpoint -> 48 kHz stereo f32 for the Opus path). wasapi = "0.23" # Virtual Xbox 360 gamepad: the in-tree XUSB companion UMDF driver (packaging/windows/xusb-driver), diff --git a/crates/punktfunk-host/src/capture.rs b/crates/punktfunk-host/src/capture.rs index 2e70b79..346108b 100644 --- a/crates/punktfunk-host/src/capture.rs +++ b/crates/punktfunk-host/src/capture.rs @@ -95,7 +95,13 @@ pub(crate) fn gpu_encode() -> bool { } #[cfg(not(target_os = "windows"))] pub(crate) fn gpu_encode() -> bool { - true + // The GPU-less software encoder (openh264) needs CPU-staged RGB frames; every other Linux + // backend (NVENC/CUDA, VAAPI) is GPU-resident. Mirrors `session_plan::resolve_encoder`, for the + // GameStream/spike entry points that use `OutputFormat::resolve` instead of a full `SessionPlan`. + !matches!( + crate::config::config().encoder_pref.as_str(), + "software" | "sw" | "openh264" + ) } /// A captured frame. [`format`](Self::format)/dimensions describe the pixels regardless of diff --git a/crates/punktfunk-host/src/encode.rs b/crates/punktfunk-host/src/encode.rs index 238036c..913099b 100644 --- a/crates/punktfunk-host/src/encode.rs +++ b/crates/punktfunk-host/src/encode.rs @@ -57,6 +57,37 @@ impl ChromaFormat { } impl Codec { + /// Map a negotiated `quic` codec bit ([`punktfunk_core::quic::CODEC_H264`] etc.) to the encoder + /// [`Codec`]. Unknown / `0` → HEVC (the pre-negotiation default). Inverse of [`Codec::to_wire`]. + pub fn from_wire(bit: u8) -> Codec { + match bit { + punktfunk_core::quic::CODEC_H264 => Codec::H264, + punktfunk_core::quic::CODEC_AV1 => Codec::Av1, + _ => Codec::H265, + } + } + + /// The single `quic` codec bit for this codec (echoed in [`punktfunk_core::quic::Welcome::codec`]). + pub fn to_wire(self) -> u8 { + match self { + Codec::H264 => punktfunk_core::quic::CODEC_H264, + Codec::H265 => punktfunk_core::quic::CODEC_HEVC, + Codec::Av1 => punktfunk_core::quic::CODEC_AV1, + } + } + + /// The `quic` codec bitfield the host can currently **emit** on the punktfunk/1 native path, + /// given the resolved encode backend. The GPU-less software encoder (openh264) produces H.264 + /// only; every GPU backend emits HEVC today (per-GPU H.264/AV1 negotiation on the native path is + /// future work — GameStream already negotiates codecs with Moonlight separately). Fed to + /// [`punktfunk_core::quic::resolve_codec`] against the client's advertised codecs. + pub fn host_wire_caps() -> u8 { + match crate::config::config().encoder_pref.as_str() { + "software" | "sw" | "openh264" => punktfunk_core::quic::CODEC_H264, + _ => punktfunk_core::quic::CODEC_HEVC, + } + } + /// The FFmpeg NVENC encoder name (selected by name, not codec id — the latter would /// pick the software encoder). pub fn nvenc_name(self) -> &'static str { @@ -283,6 +314,21 @@ pub fn open_video( chroma, ), "vaapi" | "amd" | "intel" => open_vaapi(), + // GPU-less software H.264 (openh264) — for a headless / GPU-lost box. Explicit-only: + // `auto` never picks it (a box with `/dev/nvidiactl` present but a dead driver would + // otherwise wrongly resolve to NVENC). Needs H.264 (openh264 emits only that) and a CPU + // RGB frame, which the capturer delivers because the software backend resolves `gpu=false`. + "software" | "sw" | "openh264" => { + if codec != Codec::H264 { + anyhow::bail!( + "the software encoder emits H.264 only; the session negotiated {codec:?} \ + (a client must advertise CODEC_H264 to reach a software host)" + ); + } + let _ = (cuda, bit_depth); // software path is CPU + 8-bit only + sw::OpenH264Encoder::open(format, width, height, fps, bitrate_bps) + .map(|e| Box::new(e) as Box) + } "auto" | "" => { // A CUDA frame can ONLY be consumed by NVENC, and a box with the NVIDIA device // nodes always prefers it. Everything else (AMD/Intel) takes the VAAPI path. @@ -303,7 +349,7 @@ pub fn open_video( } } other => anyhow::bail!( - "unknown PUNKTFUNK_ENCODER={other:?} — use auto (default), nvenc, or vaapi" + "unknown PUNKTFUNK_ENCODER={other:?} — use auto (default), nvenc, vaapi, or software" ), } } @@ -708,8 +754,10 @@ mod linux; #[cfg(all(target_os = "windows", feature = "nvenc"))] #[path = "encode/windows/nvenc.rs"] mod nvenc; -#[cfg(target_os = "windows")] -#[path = "encode/windows/sw.rs"] +// Software (openh264) H.264 encoder — the GPU-less path on BOTH Windows and Linux (a headless / +// GPU-less test box, or a fallback when no hardware encoder is available). Platform-agnostic: it +// consumes CPU RGB `CapturedFrame`s and the statically-bundled openh264 build. +#[cfg(any(target_os = "windows", target_os = "linux"))] mod sw; #[cfg(target_os = "linux")] #[path = "encode/linux/vaapi.rs"] diff --git a/crates/punktfunk-host/src/encode/windows/sw.rs b/crates/punktfunk-host/src/encode/sw.rs similarity index 100% rename from crates/punktfunk-host/src/encode/windows/sw.rs rename to crates/punktfunk-host/src/encode/sw.rs diff --git a/crates/punktfunk-host/src/punktfunk1.rs b/crates/punktfunk-host/src/punktfunk1.rs index 8e82d6e..e0a255d 100644 --- a/crates/punktfunk-host/src/punktfunk1.rs +++ b/crates/punktfunk-host/src/punktfunk1.rs @@ -658,12 +658,30 @@ async fn serve_session( ); // The pairing gate (require_pairing → paired? else park for delegated approval) ran above, // before this future, so a client reaching here is paired (or the host is `--open`). - crate::encode::validate_dimensions( - crate::encode::Codec::H265, - hello.mode.width, - hello.mode.height, - ) - .context("client-requested mode")?; + + // Codec negotiation: pick the one codec this host will emit (its backend capability ∩ the + // client's advertised codecs). A GPU-less software host emits H.264, so an HEVC-only client + // shares nothing with it → refuse honestly rather than send a stream it can't decode. + let host_codecs = crate::encode::Codec::host_wire_caps(); + let codec_bit = punktfunk_core::quic::resolve_codec(hello.video_codecs, host_codecs) + .ok_or_else(|| { + anyhow!( + "no shared video codec: client advertised 0x{:02x}, host can emit 0x{:02x} \ + (a software-encode host produces H.264 — the client must advertise CODEC_H264)", + hello.video_codecs, + host_codecs + ) + })?; + let codec = crate::encode::Codec::from_wire(codec_bit); + tracing::info!( + ?codec, + client_codecs = format_args!("0x{:02x}", hello.video_codecs), + host_codecs = format_args!("0x{host_codecs:02x}"), + "video codec negotiated" + ); + + crate::encode::validate_dimensions(codec, hello.mode.width, hello.mode.height) + .context("client-requested mode")?; // Resolve the client's compositor preference to a concrete backend *now*, so the Welcome // can report what we'll actually drive. Only the Virtual source has a compositor; the @@ -749,7 +767,11 @@ async fn serve_session( // the cheap gates already pass. The result is cached process-wide (a negative latches until // restart — acceptable: a GPU either supports HEVC 4:4:4 or it doesn't, and a transient open // failure here is rare since the session's own encoder isn't open yet). - let gpu_supports_444 = if host_wants_444 && client_supports_444 && capture_supports_444 { + let gpu_supports_444 = if codec == crate::encode::Codec::H265 + && host_wants_444 + && client_supports_444 + && capture_supports_444 + { tokio::task::spawn_blocking(|| { crate::encode::can_encode_444(crate::encode::Codec::H265) }) @@ -826,6 +848,9 @@ async fn serve_session( // The resolved audio channel count the audio thread will capture + Opus-(multi)stream // encode (2/6/8). The client builds its decoder from this echoed value. audio_channels, + // The negotiated codec the encoder will emit (H.264 for a software host, else HEVC). The + // client builds its decoder from this instead of assuming HEVC. + codec: codec_bit, }; io::write_msg(&mut send, &welcome.encode()).await?; @@ -838,6 +863,9 @@ async fn serve_session( .await .map_err(|_| anyhow!("handshake timed out after {HANDSHAKE_TIMEOUT:?}"))??; let (mut ctrl_send, mut ctrl_recv) = (send, recv); + // Negotiated codec (HEVC / H.264), derived from the Welcome. `Copy`, so the control task's + // `async move` captures a copy and it stays usable for the data-plane SessionContext below. + let codec = crate::encode::Codec::from_wire(welcome.codec); let client_udp = std::net::SocketAddr::new(peer.ip(), start.client_udp_port); tracing::info!( %client_udp, @@ -874,7 +902,7 @@ async fn serve_session( if let Ok(req) = Reconfigure::decode(&msg) { let ok = req.mode.refresh_hz > 0 && crate::encode::validate_dimensions( - crate::encode::Codec::H265, + codec, req.mode.width, req.mode.height, ) @@ -1169,6 +1197,7 @@ async fn serve_session( bitrate_kbps, bit_depth, chroma, + codec, probe_rx, probe_result_tx, fec_target: fec_target_dp, @@ -2727,6 +2756,9 @@ struct SessionContext { bit_depth: u8, /// Negotiated chroma subsampling (4:2:0, or 4:4:4 when the client + host + GPU all support it). chroma: crate::encode::ChromaFormat, + /// Negotiated video codec the encoder emits (HEVC by default; H.264 for a software host). Also + /// used to rebuild the encoder at the same codec across a mid-stream mode reconfigure. + codec: crate::encode::Codec, /// Speed-test burst requests (see [`service_probes`]). probe_rx: std::sync::mpsc::Receiver, /// Speed-test results back to the control task. @@ -2758,7 +2790,7 @@ fn virtual_stream(ctx: SessionContext) -> Result<()> { // path now reads this typed `SessionPlan` instead of re-deriving from config at each dispatch site // (the latent "capture and encode disagree on the backend" hazard, plan §2.4). `bit_depth` is the // only per-session input — capture/topology/encoder are otherwise pure functions of `HostConfig`. - let plan = crate::session_plan::SessionPlan::resolve(ctx.bit_depth, ctx.chroma); + let plan = crate::session_plan::SessionPlan::resolve(ctx.bit_depth, ctx.chroma, ctx.codec); tracing::info!(?plan, "resolved session plan"); // Single-process path: unpack the context into the locals the loop below uses (names unchanged, so the // body is byte-for-byte the same; the receivers are now owned but `try_recv()` is identical). @@ -2774,6 +2806,8 @@ fn virtual_stream(ctx: SessionContext) -> Result<()> { bit_depth, // The resolved chroma is already captured in `plan` (above); ignore the duplicate here. chroma: _, + // Likewise the codec — `plan.codec` (resolved from `ctx.codec`) is the source of truth below. + codec: _, probe_rx, probe_result_tx, fec_target, @@ -3448,7 +3482,7 @@ fn build_pipeline( // `bit_depth` is the handshake-negotiated value (8, or 10 = HEVC Main10 when the client // advertised VIDEO_CAP_10BIT and the host opted in). Threaded down from the Welcome. let enc = crate::encode::open_video( - crate::encode::Codec::H265, + plan.codec, frame.format, frame.width, frame.height, diff --git a/crates/punktfunk-host/src/session_plan.rs b/crates/punktfunk-host/src/session_plan.rs index 55d5c98..9dc93c1 100644 --- a/crates/punktfunk-host/src/session_plan.rs +++ b/crates/punktfunk-host/src/session_plan.rs @@ -94,12 +94,19 @@ pub struct SessionPlan { /// Handshake-negotiated chroma subsampling (4:2:0, or full-chroma 4:4:4 when the client + host + /// GPU all support it). Resolved before the Welcome; `Yuv420` on every backend that declined it. pub chroma: crate::encode::ChromaFormat, + /// Handshake-negotiated video codec the encoder emits — HEVC by default, H.264 for a GPU-less + /// software host (`resolve_codec` over the client's advertised codecs ∩ the host's capability). + pub codec: crate::encode::Codec, } impl SessionPlan { - /// Resolve the whole plan once from [`config`](crate::config) + the negotiated `bit_depth` and - /// `chroma`. - pub fn resolve(bit_depth: u8, chroma: crate::encode::ChromaFormat) -> Self { + /// Resolve the whole plan once from [`config`](crate::config) + the negotiated `bit_depth`, + /// `chroma`, and `codec`. + pub fn resolve( + bit_depth: u8, + chroma: crate::encode::ChromaFormat, + codec: crate::encode::Codec, + ) -> Self { SessionPlan { capture: CaptureBackend::resolve(), topology: resolve_topology(), @@ -107,6 +114,7 @@ impl SessionPlan { bit_depth, hdr: bit_depth >= 10, chroma, + codec, } } @@ -154,5 +162,12 @@ fn resolve_encoder() -> EncoderBackend { #[cfg(not(target_os = "windows"))] fn resolve_encoder() -> EncoderBackend { - EncoderBackend::PlatformAuto + // `PUNKTFUNK_ENCODER=software` forces the GPU-less openh264 path — which must take CPU-staged + // capture (`EncoderBackend::Software.is_gpu() == false` → `output_format().gpu = false`), so the + // portal capturer delivers CPU RGB. Everything else stays `PlatformAuto` (NVENC/VAAPI resolved + // inside `encode::open_video`). + match crate::config::config().encoder_pref.as_str() { + "software" | "sw" | "openh264" => EncoderBackend::Software, + _ => EncoderBackend::PlatformAuto, + } } diff --git a/include/punktfunk_core.h b/include/punktfunk_core.h index 3d76739..08eaa4c 100644 --- a/include/punktfunk_core.h +++ b/include/punktfunk_core.h @@ -244,6 +244,24 @@ #define VIDEO_CAP_444 4 #endif +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`Hello::video_codecs`] bit: the client can decode H.264 / AVC. The GPU-less **software** +// encode path (openh264) emits H.264, so a client that wants to stream from a software host MUST +// advertise this. +#define CODEC_H264 1 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`Hello::video_codecs`] bit: the client can decode H.265 / HEVC — the default every existing +// build produces and decodes (a peer that omits [`Hello::video_codecs`] is treated as HEVC-only). +#define CODEC_HEVC 2 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`Hello::video_codecs`] bit: the client can decode AV1. +#define CODEC_AV1 4 +#endif + #if defined(PUNKTFUNK_FEATURE_QUIC) // HEVC `chroma_format_idc` for 4:2:0 — what every pre-4:4:4 build produced and the back-compat // default when a peer omits [`Welcome::chroma_format`].