unom/punktfunk
1
0
Fork 0
Code Issues Pull Requests Actions 23 Packages Projects Releases 9 Wiki Activity

feat(protocol,clients): codec preference negotiation + Linux client decodes per Welcome (Phase 2a)

Browse Source 
Adds a client-selectable **preferred codec** and wires the core + ABI + probe + Linux client to
negotiate and decode it. (Windows/Apple/Android follow in 2b.)

**Core:**
- `Hello.preferred_codec` (a single CODEC_* bit, 0 = auto) — a soft hint appended after
  `video_codecs`. `resolve_codec(client, host, preferred)` now honors the preference when the host
  can also emit it, else falls back to precedence (HEVC > AV1 > H.264). Roundtrip + preference tests.
- `NativeClient::connect` takes `video_codecs` + `preferred_codec`; `NativeClient.codec` exposes the
  resolved `Welcome.codec`.
- ABI: `punktfunk_connect_ex7` (adds the two codec params; `ex6` delegates to it advertising
  HEVC-only) + `punktfunk_connection_codec` getter + `PUNKTFUNK_CODEC_{H264,HEVC,AV1}` constants
  (drift-guarded against the wire values). Header regenerated.

**Host:** passes `hello.preferred_codec` into `resolve_codec`.

**probe:** `--codec h264|hevc|av1|auto` sets the preference (still advertises it can decode all
three); the dump extension already follows the resolved codec.

**Linux client:** advertises the codecs FFmpeg can actually decode (`decodable_codecs()`), threads
the user's `codec` setting as the preference, and builds the decoder — both the software and VAAPI
paths, plus the mid-session VAAPI→software demotion — from the negotiated `Welcome.codec` instead of
hardcoding HEVC. New "Video codec" dropdown in Preferences (Automatic/HEVC/H.264/AV1).

Live-validated on the dev box: probe `--codec hevc` against a software (H.264-only) host resolves to
H.264 (graceful soft-preference fallback), no failure. clippy + core (57) + host (133) tests green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Enrico Bühler
2026-07-02 00:13:26 +00:00
parent ffc0b07b46
commit 12843fe253
36 changed files with 529 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
clients/linux/src/app.rs
+6 -3
View File
@@ -529,9 +529,11 @@ fn speed_test(app: Rc<App>, req: ConnectRequest) {
},
CompositorPref::Auto,
GamepadPref::Auto,
0, // bitrate_kbps (host default)
0, // video_caps: the Linux client has no 10-bit/HDR present path yet
2, // audio_channels: speed-test probe, stereo
0, // bitrate_kbps (host default)
0, // video_caps: the Linux client has no 10-bit/HDR present path yet
2, // audio_channels: speed-test probe, stereo
crate::video::decodable_codecs(), // codecs (unused by the probe, but honest)
0, // preferred_codec: no preference for a speed-test probe
None, // launch: speed-test probe connect, no game
pin,
Some(identity),
@@ -689,6 +691,7 @@ fn start_session_with(app: Rc<App>, req: ConnectRequest, pin: Option<[u8; 32]>,
bitrate_kbps: s.bitrate_kbps,
mic_enabled: s.mic_enabled,
audio_channels: s.audio_channels,
preferred_codec: s.preferred_codec(),
pin,
identity: app.identity.clone(),
connect_timeout: opts.connect_timeout,
clients/linux/src/session.rs
+14 -2
View File
@@ -22,6 +22,9 @@ pub struct SessionParams {
pub bitrate_kbps: u32,
/// Requested audio channel count (2/6/8); the host echoes the resolved value.
pub audio_channels: u8,
/// The user's preferred video codec (a `quic::CODEC_*` bit, `0` = auto). Soft — the host honors
/// it when it can emit it, else falls back; the resolved codec drives the decoder.
pub preferred_codec: u8,
/// Stream the default microphone to the host's virtual mic source.
pub mic_enabled: bool,
/// Pinned host fingerprint; `None` = trust on first use (caller persists the observed one).
@@ -141,7 +144,9 @@ fn pump(
params.bitrate_kbps,
0, // video_caps: the Linux client has no 10-bit/HDR present path yet
params.audio_channels,
None, // launch: the Linux client has no library picker yet
crate::video::decodable_codecs(), // codecs FFmpeg can decode (HEVC/H.264/AV1)
params.preferred_codec, // the user's soft codec preference (0 = auto)
None, // launch: the Linux client has no library picker yet
params.pin,
Some(params.identity),
params.connect_timeout,
@@ -170,7 +175,14 @@ fn pump(
fingerprint: connector.host_fingerprint,
});
let mut decoder = match Decoder::new() {
// Build the decoder for the codec the host resolved (never assume HEVC).
let codec_id = crate::video::ffmpeg_codec_id(connector.codec);
tracing::info!(
?codec_id,
welcome_codec = connector.codec,
"negotiated video codec"
);
let mut decoder = match Decoder::new(codec_id) {
Ok(d) => d,
Err(e) => {
let _ = ev_tx.send_blocking(SessionEvent::Ended(Some(format!("video decoder: {e}"))));
clients/linux/src/trust.rs
+21
View File
@@ -135,6 +135,26 @@ pub struct Settings {
/// Requested audio channel count: 2 (stereo), 6 (5.1) or 8 (7.1). The host clamps to what it
/// can capture; the resolved count drives the decoder + playback layout.
pub audio_channels: u8,
/// Preferred video codec: `"auto"` (host decides), `"hevc"`, `"h264"`, or `"av1"`. A soft
/// preference — the host honors it when it can emit it, else falls back to the best shared codec.
#[serde(default = "default_codec")]
pub codec: String,
}
fn default_codec() -> String {
"auto".into()
}
impl Settings {
/// The `codec` setting as a `quic::CODEC_*` preference bit (`0` = auto).
pub fn preferred_codec(&self) -> u8 {
match self.codec.as_str() {
"h264" | "avc" => punktfunk_core::quic::CODEC_H264,
"hevc" | "h265" => punktfunk_core::quic::CODEC_HEVC,
"av1" => punktfunk_core::quic::CODEC_AV1,
_ => 0,
}
}
}
impl Default for Settings {
@@ -149,6 +169,7 @@ impl Default for Settings {
inhibit_shortcuts: true,
mic_enabled: false,
audio_channels: 2,
codec: "auto".into(),
}
}
}
clients/linux/src/ui_settings.rs
+12
View File
@@ -18,6 +18,9 @@ const RESOLUTIONS: &[(u32, u32)] = &[
const REFRESH: &[u32] = &[0, 30, 60, 90, 120, 144, 165, 240];
const GAMEPADS: &[&str] = &["auto", "xbox360", "dualsense", "xboxone", "dualshock4"];
const COMPOSITORS: &[&str] = &["auto", "kwin", "wlroots", "mutter", "gamescope"];
/// Codec setting values (persisted) paired with their display labels below.
const CODECS: &[&str] = &["auto", "hevc", "h264", "av1"];
const CODEC_LABELS: &[&str] = &["Automatic", "HEVC (H.265)", "H.264 (AVC)", "AV1"];
/// punktfunk's own license (MIT OR Apache-2.0), shown on the About dialog's Legal page.
const APP_LICENSE: &str = concat!(
@@ -193,6 +196,12 @@ pub fn show(
]))
.build();
audio.add(&surround_row);
let codec_row = adw::ComboRow::builder()
.title("Video codec")
.subtitle("Preferred codec — the host falls back if it can't encode this one")
.model(&gtk::StringList::new(CODEC_LABELS))
.build();
stream.add(&codec_row);
let mic_row = adw::SwitchRow::builder()
.title("Stream microphone")
.subtitle("Send the default input device to the host's virtual microphone")
@@ -242,6 +251,8 @@ pub fn show(
8 => 2,
_ => 0,
});
let codec_i = CODECS.iter().position(|&c| c == s.codec).unwrap_or(0);
codec_row.set_selected(codec_i as u32);
}
let dialog = adw::PreferencesDialog::new();
@@ -263,6 +274,7 @@ pub fn show(
2 => 8,
_ => 2,
};
s.codec = CODECS[(codec_row.selected() as usize).min(CODECS.len() - 1)].to_string();
s.save();
});
dialog.present(Some(parent));
clients/linux/src/video.rs
+44 -12
View File
@@ -76,18 +76,48 @@ enum Backend {
pub struct Decoder {
backend: Backend,
/// The negotiated codec (from the host's Welcome), so a mid-session VAAPI→software demotion
/// rebuilds the software decoder for the SAME codec.
codec_id: ffmpeg::codec::Id,
}
/// Map a negotiated `quic` codec bit to the FFmpeg decoder id the client opens.
pub fn ffmpeg_codec_id(wire: u8) -> ffmpeg::codec::Id {
match wire {
punktfunk_core::quic::CODEC_H264 => ffmpeg::codec::Id::H264,
punktfunk_core::quic::CODEC_AV1 => ffmpeg::codec::Id::AV1,
_ => ffmpeg::codec::Id::HEVC,
}
}
/// The `quic` codec bitfield this client can decode — whatever FFmpeg has a decoder for (HEVC/H.264
/// always; AV1 when built in). Advertised to the host so it never emits a codec we can't decode.
pub fn decodable_codecs() -> u8 {
let _ = ffmpeg::init();
let mut bits = 0u8;
for (id, bit) in [
(ffmpeg::codec::Id::HEVC, punktfunk_core::quic::CODEC_HEVC),
(ffmpeg::codec::Id::H264, punktfunk_core::quic::CODEC_H264),
(ffmpeg::codec::Id::AV1, punktfunk_core::quic::CODEC_AV1),
] {
if ffmpeg::decoder::find(id).is_some() {
bits |= bit;
}
}
bits
}
impl Decoder {
pub fn new() -> Result<Decoder> {
pub fn new(codec_id: ffmpeg::codec::Id) -> Result<Decoder> {
ffmpeg::init().context("ffmpeg init")?;
let choice = std::env::var("PUNKTFUNK_DECODER").unwrap_or_default();
if choice != "software" {
match VaapiDecoder::new() {
match VaapiDecoder::new(codec_id) {
Ok(v) => {
tracing::info!("VAAPI hardware decode active (zero-copy dmabuf)");
tracing::info!(?codec_id, "VAAPI hardware decode active (zero-copy dmabuf)");
return Ok(Decoder {
backend: Backend::Vaapi(v),
codec_id,
});
}
Err(e) => {
@@ -99,7 +129,8 @@ impl Decoder {
}
}
Ok(Decoder {
backend: Backend::Software(SoftwareDecoder::new()?),
backend: Backend::Software(SoftwareDecoder::new(codec_id)?),
codec_id,
})
}
@@ -113,7 +144,7 @@ impl Decoder {
Ok(f) => Ok(f.map(DecodedFrame::Dmabuf)),
Err(e) => {
tracing::warn!(error = %e, "VAAPI decode failed — falling back to software");
self.backend = Backend::Software(SoftwareDecoder::new()?);
self.backend = Backend::Software(SoftwareDecoder::new(self.codec_id)?);
Ok(None)
}
},
@@ -131,9 +162,9 @@ struct SoftwareDecoder {
}
impl SoftwareDecoder {
fn new() -> Result<SoftwareDecoder> {
let codec =
ffmpeg::decoder::find(ffmpeg::codec::Id::HEVC).ok_or(anyhow!("no HEVC decoder"))?;
fn new(codec_id: ffmpeg::codec::Id) -> Result<SoftwareDecoder> {
let codec = ffmpeg::decoder::find(codec_id)
.ok_or_else(|| anyhow!("no {codec_id:?} decoder in libavcodec"))?;
let mut ctx = ffmpeg::codec::Context::new_with_codec(codec);
unsafe {
let raw = ctx.as_mut_ptr();
@@ -142,7 +173,7 @@ impl SoftwareDecoder {
(*raw).thread_type = ffmpeg::ffi::FF_THREAD_SLICE;
(*raw).thread_count = 0; // auto
}
let decoder = ctx.decoder().video().context("open HEVC decoder")?;
let decoder = ctx.decoder().video().context("open video decoder")?;
Ok(SoftwareDecoder { decoder, sws: None })
}
@@ -240,7 +271,7 @@ struct VaapiDecoder {
unsafe impl Send for VaapiDecoder {}
impl VaapiDecoder {
fn new() -> Result<VaapiDecoder> {
fn new(codec_id: ffmpeg::codec::Id) -> Result<VaapiDecoder> {
use ffmpeg::ffi;
unsafe {
let mut hw_device: *mut ffi::AVBufferRef = ptr::null_mut();
@@ -254,10 +285,11 @@ impl VaapiDecoder {
if r < 0 {
bail!("no VAAPI device ({})", ffmpeg::Error::from(r));
}
let codec = ffi::avcodec_find_decoder(ffi::AVCodecID::AV_CODEC_ID_HEVC);
// The negotiated codec's decoder id (av_codec_id maps 1:1 from ffmpeg::codec::Id).
let codec = ffi::avcodec_find_decoder(codec_id.into());
if codec.is_null() {
ffi::av_buffer_unref(&mut hw_device);
bail!("no HEVC decoder");
bail!("no {codec_id:?} decoder");
}
let ctx = ffi::avcodec_alloc_context3(codec);
(*ctx).hw_device_ctx = ffi::av_buffer_ref(hw_device);