refactor(windows-host): confine platform code under windows/ + linux/ folders (Goal-1 stage 6)
Move 36 platform-specific files into per-module `windows/` and `linux/` subfolders (and the
shared HID codecs into `inject/proto/`):
capture/{windows,linux}/ encode/{windows,linux}/ inject/{windows,linux,proto}/
audio/{windows,linux}/ vdisplay/{windows,linux}/
src/windows/ (service, wgc_helper, win_adapter, win_display)
src/linux/ (dmabuf_fence, drm_sync, zerocopy/)
Done with `#[path]`, NOT a module rename: every file moves into its folder while the
`crate::*::*` module names stay FLAT, so all caller paths and every internal `super::`/`crate::`
reference are unchanged — only the parent `mod` decls gained `#[path = "..."]`. This is the
codebase's existing pattern (inject's gamepad_windows) and makes the move byte-identical in
behaviour with ZERO reference churn, far lower risk than collapsing to a single
`crate::capture::windows::` namespace (that deeper rename is an optional follow-on; this delivers
the cfg-sprawl folder confinement the stage is about). Done LAST, after the semantic stages, so
the path churn didn't fight them.
Verified: Linux cargo check + clippy (-D warnings) clean; my mod-decl changes fmt-clean (the 3
remaining fmt diffs are pre-existing local-rustfmt-version skew that moved with their files); all
36 `#[path]` targets exist; no internal `#[path]`/`include!`/file-child-mod in any moved file
(the inline `mod X {` blocks are self-contained). Box build to follow.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,475 @@
|
||||
//! NVENC encoder via `ffmpeg-next` (binds the system FFmpeg — `ffmpeg-sys-next` auto-detects the
|
||||
//! installed version, so this builds against FFmpeg 7.x/libavcodec 61 *or* 8.x/libavcodec 62;
|
||||
//! validated live on Ubuntu 26.04 (FFmpeg 8) and Bazzite F43 (FFmpeg 7.1)).
|
||||
//!
|
||||
//! Input is a packed RGB/BGR CPU frame; `*_nvenc` accepts `rgb0`/`bgr0`/`rgba`/`bgra`
|
||||
//! directly and does the RGB→YUV conversion on the GPU, so the host stays off the
|
||||
//! colour-conversion path. The portal commonly negotiates packed 24-bit `RGB`, which NVENC
|
||||
//! does *not* accept — we expand it to `rgb0` (one padding byte/pixel, no colour math).
|
||||
//! The encoder is opened *without* a global header so VPS/SPS/PPS are emitted in-band on
|
||||
//! every IDR — the output is both a playable raw Annex-B stream and self-contained AUs.
|
||||
|
||||
use super::{Codec, EncodedFrame, Encoder};
|
||||
use crate::capture::{CapturedFrame, FramePayload, PixelFormat};
|
||||
use anyhow::{anyhow, bail, Context, Result};
|
||||
use ffmpeg::format::Pixel;
|
||||
use ffmpeg::util::frame::Video as VideoFrame;
|
||||
use ffmpeg::{codec, encoder, Dictionary, Packet, Rational};
|
||||
use ffmpeg_next as ffmpeg;
|
||||
use std::os::raw::c_int;
|
||||
|
||||
use ffmpeg::ffi; // = ffmpeg_sys_next
|
||||
|
||||
/// `AVCUDADeviceContext` (libavutil/hwcontext_cuda.h) — not in the ffmpeg-sys bindings (the
|
||||
/// crate doesn't allowlist that header), so mirror its stable 3-pointer layout. We set the
|
||||
/// first field to *our* `CUcontext` so NVENC shares the context the EGL importer maps into.
|
||||
#[repr(C)]
|
||||
struct AVCUDADeviceContext {
|
||||
cuda_ctx: *mut std::ffi::c_void, // CUcontext
|
||||
stream: *mut std::ffi::c_void, // CUstream (null = default)
|
||||
internal: *mut std::ffi::c_void, // filled by ctx_init
|
||||
}
|
||||
|
||||
/// CUDA hardware-frame contexts that wrap our shared `CUcontext`, so `hevc_nvenc` reads the
|
||||
/// imported device buffer directly. Owns two `AVBufferRef`s, unref'd on drop.
|
||||
struct CudaHw {
|
||||
device_ref: *mut ffi::AVBufferRef,
|
||||
frames_ref: *mut ffi::AVBufferRef,
|
||||
}
|
||||
|
||||
impl CudaHw {
|
||||
/// Build a CUDA hwdevice wrapping `cu_ctx` and a frames pool (`sw_format` = `pixel`).
|
||||
unsafe fn new(cu_ctx: *mut std::ffi::c_void, sw_format: Pixel, w: u32, h: u32) -> Result<Self> {
|
||||
let mut device_ref = ffi::av_hwdevice_ctx_alloc(ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA);
|
||||
if device_ref.is_null() {
|
||||
bail!("av_hwdevice_ctx_alloc(CUDA) failed");
|
||||
}
|
||||
let dev_ctx = (*device_ref).data as *mut ffi::AVHWDeviceContext;
|
||||
let cu = (*dev_ctx).hwctx as *mut AVCUDADeviceContext;
|
||||
(*cu).cuda_ctx = cu_ctx; // share the importer's context
|
||||
let r = ffi::av_hwdevice_ctx_init(device_ref);
|
||||
if r < 0 {
|
||||
ffi::av_buffer_unref(&mut device_ref);
|
||||
bail!("av_hwdevice_ctx_init failed ({r})");
|
||||
}
|
||||
|
||||
let mut frames_ref = ffi::av_hwframe_ctx_alloc(device_ref);
|
||||
if frames_ref.is_null() {
|
||||
ffi::av_buffer_unref(&mut device_ref);
|
||||
bail!("av_hwframe_ctx_alloc failed");
|
||||
}
|
||||
let fc = (*frames_ref).data as *mut ffi::AVHWFramesContext;
|
||||
(*fc).format = ffi::AVPixelFormat::AV_PIX_FMT_CUDA;
|
||||
(*fc).sw_format = pixel_to_av(sw_format);
|
||||
(*fc).width = w as c_int;
|
||||
(*fc).height = h as c_int;
|
||||
(*fc).initial_pool_size = 0; // we supply the device pointers
|
||||
let r = ffi::av_hwframe_ctx_init(frames_ref);
|
||||
if r < 0 {
|
||||
ffi::av_buffer_unref(&mut frames_ref);
|
||||
ffi::av_buffer_unref(&mut device_ref);
|
||||
bail!("av_hwframe_ctx_init failed ({r})");
|
||||
}
|
||||
Ok(CudaHw {
|
||||
device_ref,
|
||||
frames_ref,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CudaHw {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
ffi::av_buffer_unref(&mut self.frames_ref);
|
||||
ffi::av_buffer_unref(&mut self.device_ref);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `ffmpeg::format::Pixel` → raw `AVPixelFormat`.
|
||||
fn pixel_to_av(p: Pixel) -> ffi::AVPixelFormat {
|
||||
// `Pixel` is `#[repr(i32)]`-compatible with `AVPixelFormat` (the bindgen enum) via this
|
||||
// documented conversion in ffmpeg-next.
|
||||
ffi::AVPixelFormat::from(p)
|
||||
}
|
||||
|
||||
/// Map a captured layout to the NVENC input pixel format, and whether a 3→4 byte expand is
|
||||
/// needed (packed RGB/BGR have no padding byte; the NVENC `*0` formats do).
|
||||
fn nvenc_input(format: PixelFormat) -> (Pixel, bool) {
|
||||
match format {
|
||||
PixelFormat::Bgrx => (Pixel::BGRZ, false), // bgr0
|
||||
PixelFormat::Rgbx => (Pixel::RGBZ, false), // rgb0
|
||||
PixelFormat::Bgra => (Pixel::BGRA, false),
|
||||
PixelFormat::Rgba => (Pixel::RGBA, false),
|
||||
PixelFormat::Rgb => (Pixel::RGBZ, true), // RGB -> rgb0
|
||||
PixelFormat::Bgr => (Pixel::BGRZ, true), // BGR -> bgr0
|
||||
// NV12 is native YUV: NVENC encodes it with NO internal RGB→YUV CSC (the Tier 2A win). On
|
||||
// Linux it's produced by the GPU convert on the zero-copy tiled path (`PUNKTFUNK_NV12`); on
|
||||
// Windows by the D3D11 video processor.
|
||||
PixelFormat::Nv12 => (Pixel::NV12, false),
|
||||
// Rgb10a2 (HDR) and P010 (the Windows 10-bit video-processor output) are produced only by
|
||||
// the Windows paths; the Linux capturer never emits them. Map to BGRA so the match is
|
||||
// exhaustive — unreachable here.
|
||||
PixelFormat::Rgb10a2 | PixelFormat::P010 => (Pixel::BGRA, false),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct NvencEncoder {
|
||||
enc: encoder::video::Encoder,
|
||||
/// Reusable 4-bpp CPU input frame (CPU path only; `None` for the zero-copy/CUDA path).
|
||||
/// Mutating it in place across frames is sound only because the encoder is opened with
|
||||
/// `delay=0`/`bf=0`/`max_b_frames=0` and the caller drains `poll()` after each `submit`,
|
||||
/// so libavcodec holds no reference to the previous frame's buffer when we overwrite it.
|
||||
frame: Option<VideoFrame>,
|
||||
/// Zero-copy path: CUDA hwdevice/hwframes contexts (the encoder takes `AV_PIX_FMT_CUDA`).
|
||||
cuda: Option<CudaHw>,
|
||||
src_format: PixelFormat,
|
||||
expand: bool,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
/// Monotonic presentation index, in `1/fps` time-base units.
|
||||
frame_idx: i64,
|
||||
/// Force the next submitted frame to be an IDR (set by [`request_keyframe`]).
|
||||
force_kf: bool,
|
||||
}
|
||||
|
||||
// `CudaHw` holds raw `AVBufferRef`s; the encoder lives on a single thread. The CPU encoder is
|
||||
// already `Send` via ffmpeg-next; assert it for the CUDA fields too.
|
||||
unsafe impl Send for NvencEncoder {}
|
||||
|
||||
impl NvencEncoder {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn open(
|
||||
codec: Codec,
|
||||
format: PixelFormat,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
bitrate_bps: u64,
|
||||
cuda: bool,
|
||||
bit_depth: u8,
|
||||
) -> Result<Self> {
|
||||
// TODO(hdr): Linux 10-bit parity. Unlike the Windows raw-SDK path (which upconverts 8-bit
|
||||
// ARGB → Main10 via pixelBitDepthMinus8), libavcodec hevc_nvenc needs a 10-bit input pixel
|
||||
// format (p010) for Main10, so it's a bigger change; deferred until a Linux GPU box is
|
||||
// available to validate. The Linux host stays 8-bit for now.
|
||||
if bit_depth != 8 {
|
||||
tracing::warn!(
|
||||
bit_depth,
|
||||
"Linux NVENC 10-bit not yet wired — encoding 8-bit"
|
||||
);
|
||||
}
|
||||
ffmpeg::init().context("ffmpeg init")?;
|
||||
if std::env::var_os("PUNKTFUNK_FFMPEG_DEBUG").is_some() {
|
||||
unsafe { ffi::av_log_set_level(48) }; // AV_LOG_DEBUG — surface NVENC hw-frame rejects
|
||||
}
|
||||
let name = codec.nvenc_name();
|
||||
let av_codec = encoder::find_by_name(name)
|
||||
.ok_or_else(|| anyhow!("{name} not built into libavcodec"))?;
|
||||
let (nvenc_pixel, expand) = nvenc_input(format);
|
||||
|
||||
let mut video = codec::context::Context::new_with_codec(av_codec)
|
||||
.encoder()
|
||||
.video()
|
||||
.context("alloc video encoder")?;
|
||||
video.set_width(width);
|
||||
video.set_height(height);
|
||||
video.set_format(nvenc_pixel); // NVENC converts RGB→YUV internally
|
||||
video.set_time_base(Rational(1, fps as i32));
|
||||
video.set_frame_rate(Some(Rational(fps as i32, 1)));
|
||||
video.set_bit_rate(bitrate_bps as usize);
|
||||
video.set_max_bit_rate(bitrate_bps as usize);
|
||||
// VBV/HRD buffer — bound the SIZE of any single frame. Under CBR with no buffer set, NVENC
|
||||
// uses a loose default VBV, so a high-motion P-frame is allowed to balloon to many times the
|
||||
// average; those extra packets overflow the bounded send queue + kernel socket buffer and
|
||||
// get dropped, which the client sees as framedrops/jitter (and, on the infinite-GOP path, as
|
||||
// old/stale frames flashing until the next RFI). A tight ~1-frame buffer makes the encoder
|
||||
// hold frame size roughly constant and absorb motion as a momentary QP (quality) dip instead
|
||||
// — the trade we want. Default = 1 frame of bits (bitrate/fps); PUNKTFUNK_VBV_FRAMES tunes it
|
||||
// (larger = better motion quality but bigger per-frame bursts).
|
||||
let vbv_frames = std::env::var("PUNKTFUNK_VBV_FRAMES")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<f32>().ok())
|
||||
.filter(|v| v.is_finite() && *v > 0.0)
|
||||
.unwrap_or(1.0);
|
||||
let vbv_bits = ((bitrate_bps as f64 / fps.max(1) as f64) * vbv_frames as f64)
|
||||
.clamp(1.0, i32::MAX as f64);
|
||||
unsafe {
|
||||
(*video.as_mut_ptr()).rc_buffer_size = vbv_bits as i32;
|
||||
}
|
||||
video.set_max_b_frames(0);
|
||||
// Infinite GOP — NO periodic IDR. A keyframe at 5120x1440 is ~20-40x a P-frame, so a
|
||||
// periodic IDR is a recurring multi-millisecond encode+packetize+send spike — the ~2s
|
||||
// "freeze". NVENC emits one IDR at stream start, then P-frames only; `forced-idr` (below)
|
||||
// turns a client recovery request (RFI, via `request_keyframe`) into an IDR on demand.
|
||||
// This is the Moonlight/Sunshine low-latency model.
|
||||
unsafe {
|
||||
(*video.as_mut_ptr()).gop_size = -1;
|
||||
}
|
||||
|
||||
// NV12 path: we did the RGB→YUV conversion ourselves as BT.709 *limited* range, so signal
|
||||
// that in the bitstream VUI (colorspace/range/primaries/transfer) — otherwise the client
|
||||
// decoder assumes a default and the picture comes out washed-out / wrong-contrast. The
|
||||
// RGB-input paths leave these unset (NVENC's internal CSC writes its own VUI). Matches the
|
||||
// Windows NV12 path's BT.709 limited-range signalling.
|
||||
if matches!(format, PixelFormat::Nv12) {
|
||||
unsafe {
|
||||
let raw = video.as_mut_ptr();
|
||||
(*raw).colorspace = ffi::AVColorSpace::AVCOL_SPC_BT709;
|
||||
(*raw).color_range = ffi::AVColorRange::AVCOL_RANGE_MPEG; // limited/studio
|
||||
(*raw).color_primaries = ffi::AVColorPrimaries::AVCOL_PRI_BT709;
|
||||
(*raw).color_trc = ffi::AVColorTransferCharacteristic::AVCOL_TRC_BT709;
|
||||
}
|
||||
}
|
||||
|
||||
// For the zero-copy path, take CUDA surfaces: wrap the shared CUcontext in CUDA
|
||||
// hwdevice/hwframes contexts and set `pix_fmt = CUDA` on the raw encoder context
|
||||
// *before* open (NVENC derives the device from `hw_frames_ctx`).
|
||||
let cuda_hw = if cuda {
|
||||
let cu_ctx = crate::zerocopy::cuda::context().context("shared CUDA context")?;
|
||||
let hw = unsafe { CudaHw::new(cu_ctx, nvenc_pixel, width, height)? };
|
||||
unsafe {
|
||||
let raw = video.as_mut_ptr();
|
||||
(*raw).pix_fmt = ffi::AVPixelFormat::AV_PIX_FMT_CUDA;
|
||||
(*raw).hw_device_ctx = ffi::av_buffer_ref(hw.device_ref);
|
||||
(*raw).hw_frames_ctx = ffi::av_buffer_ref(hw.frames_ref);
|
||||
}
|
||||
Some(hw)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Low-latency NVENC tuning (plan §7 / linux-setup doc).
|
||||
let mut opts = Dictionary::new();
|
||||
opts.set("preset", "p1"); // fastest
|
||||
opts.set("tune", "ull"); // ultra-low-latency
|
||||
opts.set("rc", "cbr");
|
||||
opts.set("bf", "0");
|
||||
opts.set("delay", "0");
|
||||
opts.set("forced-idr", "1"); // RFI/request_keyframe → real IDR under the infinite GOP
|
||||
|
||||
// Split-frame encode across both NVENC engines (GB203 has 2) when the pixel rate exceeds
|
||||
// a single engine's HEVC capacity (~1 Gpix/s); e.g. 5120x1440@240 = 1.77 Gpix/s needs it,
|
||||
// @120 = 0.88 Gpix/s does not. HEVC/AV1 only (not H.264). AUTO won't engage below ~2112px
|
||||
// height, so we force `2`; below the threshold we leave it AUTO (split costs ~2% BD-rate).
|
||||
// Output is standard HEVC — transparent to the client. Override with PUNKTFUNK_SPLIT_ENCODE.
|
||||
let pix_rate = width as u64 * height as u64 * fps as u64;
|
||||
let split = std::env::var("PUNKTFUNK_SPLIT_ENCODE").ok();
|
||||
match split.as_deref() {
|
||||
Some(mode) => opts.set("split_encode_mode", mode),
|
||||
None if matches!(codec, Codec::H265 | Codec::Av1) && pix_rate > 1_000_000_000 => {
|
||||
opts.set("split_encode_mode", "2");
|
||||
tracing::info!(
|
||||
pix_rate,
|
||||
"NVENC: forcing 2-way split encode (high pixel rate)"
|
||||
);
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
|
||||
let enc = video
|
||||
.open_with(opts)
|
||||
.with_context(|| format!("open {name} ({width}x{height}@{fps}, {bitrate_bps} bps)"))?;
|
||||
|
||||
let frame = if cuda {
|
||||
None
|
||||
} else {
|
||||
Some(VideoFrame::new(nvenc_pixel, width, height))
|
||||
};
|
||||
Ok(NvencEncoder {
|
||||
enc,
|
||||
frame,
|
||||
cuda: cuda_hw,
|
||||
src_format: format,
|
||||
expand,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
frame_idx: 0,
|
||||
force_kf: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Encoder for NvencEncoder {
|
||||
fn submit(&mut self, captured: &CapturedFrame) -> Result<()> {
|
||||
anyhow::ensure!(
|
||||
captured.width == self.width && captured.height == self.height,
|
||||
"captured frame {}x{} != encoder {}x{}",
|
||||
captured.width,
|
||||
captured.height,
|
||||
self.width,
|
||||
self.height
|
||||
);
|
||||
let pts = self.frame_idx;
|
||||
self.frame_idx += 1;
|
||||
// Force an IDR when requested (client RFI); otherwise let NVENC pick (GOP/P-frame).
|
||||
let idr = self.force_kf;
|
||||
self.force_kf = false;
|
||||
match &captured.payload {
|
||||
FramePayload::Cuda(buf) => self.submit_cuda(buf, pts, idr),
|
||||
FramePayload::Cpu(bytes) => self.submit_cpu(bytes, captured.format, pts, idr),
|
||||
FramePayload::Dmabuf(_) => {
|
||||
bail!("NVENC got a VAAPI dmabuf frame — capture/encoder backend mismatch")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn request_keyframe(&mut self) {
|
||||
self.force_kf = true;
|
||||
}
|
||||
|
||||
fn poll(&mut self) -> Result<Option<EncodedFrame>> {
|
||||
let mut pkt = Packet::empty();
|
||||
match self.enc.receive_packet(&mut pkt) {
|
||||
Ok(()) => {
|
||||
let data = pkt.data().map(|d| d.to_vec()).unwrap_or_default();
|
||||
let pts = pkt.pts().unwrap_or(0).max(0) as u64;
|
||||
let pts_ns = pts * 1_000_000_000 / self.fps as u64;
|
||||
Ok(Some(EncodedFrame {
|
||||
data,
|
||||
pts_ns,
|
||||
keyframe: pkt.is_key(),
|
||||
}))
|
||||
}
|
||||
// No packet ready yet (need another input frame).
|
||||
Err(ffmpeg::Error::Other { errno })
|
||||
if errno == ffmpeg::util::error::EAGAIN
|
||||
|| errno == ffmpeg::util::error::EWOULDBLOCK =>
|
||||
{
|
||||
Ok(None)
|
||||
}
|
||||
// Fully drained after flush().
|
||||
Err(ffmpeg::Error::Eof) => Ok(None),
|
||||
Err(e) => Err(e).context("receive_packet"),
|
||||
}
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Result<()> {
|
||||
self.enc.send_eof().context("send_eof")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl NvencEncoder {
|
||||
/// CPU path: expand/copy the packed RGB/BGR bytes into the reusable 4-bpp frame, then send.
|
||||
fn submit_cpu(&mut self, bytes: &[u8], format: PixelFormat, pts: i64, idr: bool) -> Result<()> {
|
||||
anyhow::ensure!(
|
||||
format == self.src_format,
|
||||
"captured format {:?} != encoder source {:?}",
|
||||
format,
|
||||
self.src_format
|
||||
);
|
||||
let w = self.width as usize;
|
||||
let h = self.height as usize;
|
||||
let src_bpp = self.src_format.bytes_per_pixel();
|
||||
let src_row = w * src_bpp;
|
||||
anyhow::ensure!(
|
||||
bytes.len() >= src_row * h,
|
||||
"captured buffer {} bytes < required {}",
|
||||
bytes.len(),
|
||||
src_row * h
|
||||
);
|
||||
let frame = self
|
||||
.frame
|
||||
.as_mut()
|
||||
.context("CPU frame missing (encoder opened in CUDA mode)")?;
|
||||
let stride = frame.stride(0); // dst is 4-bpp, aligned
|
||||
let dst = frame.data_mut(0);
|
||||
if self.expand {
|
||||
// packed 3-bpp RGB/BGR → 4-bpp *0 (copy 3 bytes, zero the pad byte)
|
||||
for y in 0..h {
|
||||
let s = &bytes[y * src_row..y * src_row + src_row];
|
||||
let drow = &mut dst[y * stride..y * stride + w * 4];
|
||||
for x in 0..w {
|
||||
drow[x * 4..x * 4 + 3].copy_from_slice(&s[x * 3..x * 3 + 3]);
|
||||
drow[x * 4 + 3] = 0;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 4-bpp → 4-bpp, honoring the (possibly larger) dst stride
|
||||
for y in 0..h {
|
||||
dst[y * stride..y * stride + src_row]
|
||||
.copy_from_slice(&bytes[y * src_row..y * src_row + src_row]);
|
||||
}
|
||||
}
|
||||
frame.set_pts(Some(pts));
|
||||
frame.set_kind(if idr {
|
||||
ffmpeg::picture::Type::I
|
||||
} else {
|
||||
ffmpeg::picture::Type::None
|
||||
});
|
||||
self.enc.send_frame(frame).context("send_frame")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Zero-copy path: hand the imported CUDA device buffer to NVENC with no CPU touch.
|
||||
///
|
||||
/// We take a *pooled* surface from the CUDA hwframes context (`av_hwframe_get_buffer`) and
|
||||
/// device→device-copy our imported buffer into it, rather than wrapping our own pointer in a
|
||||
/// bare frame. Two reasons: (1) NVENC's `nvenc_send_frame` ignores frames whose `buf[0]` is
|
||||
/// null and the generic encode path's `av_frame_ref` needs a refcounted buffer — a bare
|
||||
/// frame is rejected with `EINVAL`; (2) NVENC caches CUDA-resource *registrations* keyed by
|
||||
/// device pointer with a bounded table, so a fresh pointer every frame would thrash/overflow
|
||||
/// it — the pool recycles a small set of pointers. The extra copy is device-local (~8 MB at
|
||||
/// 1080p, sub-millisecond on the GPU) and keeps the host fully off the pixel path.
|
||||
fn submit_cuda(
|
||||
&mut self,
|
||||
buf: &crate::zerocopy::DeviceBuffer,
|
||||
pts: i64,
|
||||
idr: bool,
|
||||
) -> Result<()> {
|
||||
let frames_ref = self
|
||||
.cuda
|
||||
.as_ref()
|
||||
.context("CUDA hw context missing (encoder opened in CPU mode)")?
|
||||
.frames_ref;
|
||||
// The device→device copy below uses our shared context directly; make it current on the
|
||||
// encode thread (ffmpeg pushes its own around the pool alloc, so order is fine).
|
||||
crate::zerocopy::cuda::make_current().context("CUDA context current (encode thread)")?;
|
||||
unsafe {
|
||||
let mut f = ffi::av_frame_alloc();
|
||||
if f.is_null() {
|
||||
bail!("av_frame_alloc failed");
|
||||
}
|
||||
// Pooled CUDA surface: sets format, width/height, data[0]/linesize[0], buf[0] and
|
||||
// hw_frames_ctx. Reused across frames (the pool recycles), keeping NVENC's
|
||||
// registration cache warm.
|
||||
let r = ffi::av_hwframe_get_buffer(frames_ref, f, 0);
|
||||
if r < 0 {
|
||||
ffi::av_frame_free(&mut f);
|
||||
bail!("av_hwframe_get_buffer(CUDA) failed ({r})");
|
||||
}
|
||||
// NV12 surfaces are two-plane (Y in data[0], interleaved UV in data[1]); the RGB
|
||||
// surfaces are single-plane. Copy the matching layout into NVENC's pooled surface.
|
||||
let copy_res = if buf.is_nv12() {
|
||||
let y_ptr = (*f).data[0] as crate::zerocopy::cuda::CUdeviceptr;
|
||||
let y_pitch = (*f).linesize[0] as usize;
|
||||
let uv_ptr = (*f).data[1] as crate::zerocopy::cuda::CUdeviceptr;
|
||||
let uv_pitch = (*f).linesize[1] as usize;
|
||||
crate::zerocopy::cuda::copy_nv12_to_device(buf, y_ptr, y_pitch, uv_ptr, uv_pitch)
|
||||
} else {
|
||||
let dst_ptr = (*f).data[0] as crate::zerocopy::cuda::CUdeviceptr;
|
||||
let dst_pitch = (*f).linesize[0] as usize;
|
||||
crate::zerocopy::cuda::copy_device_to_device(buf, dst_ptr, dst_pitch)
|
||||
};
|
||||
if let Err(e) = copy_res {
|
||||
ffi::av_frame_free(&mut f);
|
||||
return Err(e).context("copy imported buffer into NVENC surface");
|
||||
}
|
||||
(*f).pts = pts;
|
||||
(*f).pict_type = if idr {
|
||||
ffi::AVPictureType::AV_PICTURE_TYPE_I
|
||||
} else {
|
||||
ffi::AVPictureType::AV_PICTURE_TYPE_NONE
|
||||
};
|
||||
let r = ffi::avcodec_send_frame(self.enc.as_mut_ptr(), f);
|
||||
ffi::av_frame_free(&mut f);
|
||||
if r < 0 {
|
||||
bail!("avcodec_send_frame(CUDA) failed ({r})");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,817 @@
|
||||
//! VAAPI encoder via `ffmpeg-next` — AMD (Mesa `radeonsi`) and Intel (`iHD`/`i965`) over one
|
||||
//! libavcodec backend (`h264_vaapi`/`hevc_vaapi`/`av1_vaapi`). The kernel driver differs per
|
||||
//! vendor; the libva userspace API is identical, so a single encoder covers both. This is the
|
||||
//! sibling of [`super::linux`] (NVENC/CUDA) behind the shared [`Encoder`] trait — selected in
|
||||
//! [`super::open_video`] (NVIDIA → NVENC, AMD/Intel → here).
|
||||
//!
|
||||
//! Two input paths, chosen lazily from the FIRST frame's payload (so `open_video`'s signature
|
||||
//! is unchanged and the encoder self-configures for whatever the capturer produces):
|
||||
//! * **CPU upload** ([`CpuInner`]): the portal hands packed RGB/BGR CPU frames; we swscale to
|
||||
//! BT.709-limited NV12 and `av_hwframe_transfer_data` it into a pooled VA surface. Works on any
|
||||
//! VAAPI GPU with no capture changes (the capturer falls back to CPU frames on non-NVIDIA).
|
||||
//! * **Zero-copy dmabuf** ([`DmabufInner`], `PUNKTFUNK_ZEROCOPY=1`): the capturer hands a packed-RGB
|
||||
//! dmabuf. We wrap it as an `AV_PIX_FMT_DRM_PRIME` frame and push it through a tiny filter graph
|
||||
//! `buffer(drm_prime) → hwmap=derive_device=vaapi → scale_vaapi=format=nv12 → buffersink`, so
|
||||
//! the import AND the RGB→NV12 colour conversion run on the GPU's video engine — no host CSC, no
|
||||
//! upload. The encoder takes the NV12 surfaces straight from the filter sink.
|
||||
//!
|
||||
//! Raw FFI: `ffmpeg-next` has no hwcontext/filter wrappers for what we need, so the
|
||||
//! hwdevice/hwframes/buffersrc/buffersink calls go through `ffmpeg::ffi` (= `ffmpeg_sys_next`),
|
||||
//! as the CUDA encode path and the clients' decode paths already do. The encoder is opened
|
||||
//! *without* a global header, so VPS/SPS/PPS are in-band on every IDR.
|
||||
|
||||
use super::{Codec, EncodedFrame, Encoder};
|
||||
use crate::capture::{CapturedFrame, DmabufFrame, FramePayload, PixelFormat};
|
||||
use anyhow::{anyhow, bail, Context, Result};
|
||||
use ffmpeg::format::Pixel;
|
||||
use ffmpeg::{codec, encoder, Dictionary, Packet, Rational};
|
||||
use ffmpeg_next as ffmpeg;
|
||||
use std::ffi::{CStr, CString};
|
||||
use std::os::fd::AsRawFd;
|
||||
use std::os::raw::c_int;
|
||||
use std::ptr;
|
||||
|
||||
use ffmpeg::ffi; // = ffmpeg_sys_next
|
||||
|
||||
// libswscale scaler-flag + colour-space constants (not exported as Rust consts by the bindings;
|
||||
// these are the stable `<libswscale/swscale.h>` #defines). No-rescale → POINT is cheapest.
|
||||
const SWS_POINT: c_int = 0x10;
|
||||
const SWS_CS_ITU709: c_int = 1;
|
||||
|
||||
/// `ffmpeg::format::Pixel` → raw `AVPixelFormat` (the documented ffmpeg-next conversion).
|
||||
fn pixel_to_av(p: Pixel) -> ffi::AVPixelFormat {
|
||||
ffi::AVPixelFormat::from(p)
|
||||
}
|
||||
|
||||
/// `fourcc(a,b,c,d)` — DRM FourCC packing (`a | b<<8 | c<<16 | d<<24`).
|
||||
const fn fourcc(a: u8, b: u8, c: u8, d: u8) -> u32 {
|
||||
(a as u32) | ((b as u32) << 8) | ((c as u32) << 16) | ((d as u32) << 24)
|
||||
}
|
||||
|
||||
/// The render node a VAAPI/DRM device should open. `PUNKTFUNK_RENDER_NODE` pins it on a multi-GPU
|
||||
/// box; the default is correct on a single-GPU host.
|
||||
fn render_node() -> CString {
|
||||
let p = std::env::var("PUNKTFUNK_RENDER_NODE").unwrap_or_else(|_| "/dev/dri/renderD128".into());
|
||||
CString::new(p).unwrap_or_else(|_| CString::new("/dev/dri/renderD128").unwrap())
|
||||
}
|
||||
|
||||
/// The swscale *source* pixel format for a captured CPU layout (packed RGB/BGR only).
|
||||
fn vaapi_sws_src(format: PixelFormat) -> Result<Pixel> {
|
||||
Ok(match format {
|
||||
PixelFormat::Bgrx => Pixel::BGRZ, // bgr0
|
||||
PixelFormat::Rgbx => Pixel::RGBZ, // rgb0
|
||||
PixelFormat::Bgra => Pixel::BGRA,
|
||||
PixelFormat::Rgba => Pixel::RGBA,
|
||||
PixelFormat::Rgb => Pixel::RGB24,
|
||||
PixelFormat::Bgr => Pixel::BGR24,
|
||||
PixelFormat::Nv12 | PixelFormat::P010 | PixelFormat::Rgb10a2 => {
|
||||
bail!("VAAPI CPU-input path supports packed RGB/BGR only; got {format:?}")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Build the FFmpeg encoder context (shared by both inner paths): name, mode, low-latency RC,
|
||||
/// infinite GOP, BT.709-limited VUI, `pix_fmt=VAAPI`, and the given hw device + frames contexts.
|
||||
/// Returns the opened encoder. `device_ref`/`frames_ref` are borrowed (ref'd into the context).
|
||||
unsafe fn open_vaapi_encoder(
|
||||
codec: Codec,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
bitrate_bps: u64,
|
||||
device_ref: *mut ffi::AVBufferRef,
|
||||
frames_ref: *mut ffi::AVBufferRef,
|
||||
) -> Result<encoder::video::Encoder> {
|
||||
let name = codec.vaapi_name();
|
||||
let av_codec = encoder::find_by_name(name).ok_or_else(|| {
|
||||
anyhow!("{name} not built into libavcodec (no VAAPI encoder for {codec:?})")
|
||||
})?;
|
||||
let mut video = codec::context::Context::new_with_codec(av_codec)
|
||||
.encoder()
|
||||
.video()
|
||||
.context("alloc video encoder")?;
|
||||
video.set_width(width);
|
||||
video.set_height(height);
|
||||
video.set_format(Pixel::NV12); // sw view; pix_fmt overridden to VAAPI below
|
||||
video.set_time_base(Rational(1, fps as i32));
|
||||
video.set_frame_rate(Some(Rational(fps as i32, 1)));
|
||||
video.set_bit_rate(bitrate_bps as usize);
|
||||
video.set_max_bit_rate(bitrate_bps as usize); // == target → vaapi_encode picks CBR when supported
|
||||
let vbv_frames = std::env::var("PUNKTFUNK_VBV_FRAMES")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<f32>().ok())
|
||||
.filter(|v| v.is_finite() && *v > 0.0)
|
||||
.unwrap_or(1.0);
|
||||
let vbv_bits =
|
||||
((bitrate_bps as f64 / fps.max(1) as f64) * vbv_frames as f64).clamp(1.0, i32::MAX as f64);
|
||||
video.set_max_b_frames(0);
|
||||
let raw = video.as_mut_ptr();
|
||||
(*raw).rc_buffer_size = vbv_bits as i32;
|
||||
(*raw).gop_size = i32::MAX; // no periodic IDR (forced-IDR via pict_type=I on RFI)
|
||||
// We hand the encoder BT.709 *limited* NV12 (swscale CSC, or scale_vaapi which preserves the
|
||||
// input range we tag), so signal that VUI — else the client decoder washes the picture out.
|
||||
(*raw).colorspace = ffi::AVColorSpace::AVCOL_SPC_BT709;
|
||||
(*raw).color_range = ffi::AVColorRange::AVCOL_RANGE_MPEG;
|
||||
(*raw).color_primaries = ffi::AVColorPrimaries::AVCOL_PRI_BT709;
|
||||
(*raw).color_trc = ffi::AVColorTransferCharacteristic::AVCOL_TRC_BT709;
|
||||
(*raw).pix_fmt = ffi::AVPixelFormat::AV_PIX_FMT_VAAPI;
|
||||
(*raw).hw_device_ctx = ffi::av_buffer_ref(device_ref);
|
||||
(*raw).hw_frames_ctx = ffi::av_buffer_ref(frames_ref);
|
||||
|
||||
let mut opts = Dictionary::new();
|
||||
opts.set("async_depth", "1"); // one-in/one-out — minimal encode-pipeline latency
|
||||
video
|
||||
.open_with(opts)
|
||||
.with_context(|| format!("open {name} ({width}x{height}@{fps}, {bitrate_bps} bps)"))
|
||||
}
|
||||
|
||||
/// Probe whether THIS GPU can VAAPI-encode `codec`, by opening a tiny encoder: the driver rejects
|
||||
/// codecs its video engine can't do (e.g. AV1 on pre-RDNA3 AMD / pre-Arc Intel). Used to build the
|
||||
/// GameStream codec advertisement so a client never negotiates a codec the GPU can't encode. The
|
||||
/// device + encoder are torn down immediately (RAII).
|
||||
pub fn probe_can_encode(codec: Codec) -> bool {
|
||||
if ffmpeg::init().is_err() {
|
||||
return false;
|
||||
}
|
||||
unsafe {
|
||||
// A missing VA device (non-VAAPI host, GPU-less CI) is an expected probe outcome — quiet
|
||||
// ffmpeg's "No VA display found" error for the probe, then restore the level.
|
||||
let prev = ffi::av_log_get_level();
|
||||
ffi::av_log_set_level(ffi::AV_LOG_FATAL);
|
||||
let ok = match VaapiHw::new(ffi::AVPixelFormat::AV_PIX_FMT_NV12, 640, 480, 2) {
|
||||
Ok(hw) => {
|
||||
open_vaapi_encoder(codec, 640, 480, 30, 2_000_000, hw.device_ref, hw.frames_ref)
|
||||
.is_ok()
|
||||
}
|
||||
Err(_) => false,
|
||||
};
|
||||
ffi::av_log_set_level(prev);
|
||||
ok
|
||||
}
|
||||
}
|
||||
|
||||
/// Drain the encoder for one packet (shared poll logic).
|
||||
fn poll_encoder(enc: &mut encoder::video::Encoder, fps: u32) -> Result<Option<EncodedFrame>> {
|
||||
let mut pkt = Packet::empty();
|
||||
match enc.receive_packet(&mut pkt) {
|
||||
Ok(()) => {
|
||||
let data = pkt.data().map(|d| d.to_vec()).unwrap_or_default();
|
||||
let pts = pkt.pts().unwrap_or(0).max(0) as u64;
|
||||
Ok(Some(EncodedFrame {
|
||||
data,
|
||||
pts_ns: pts * 1_000_000_000 / fps as u64,
|
||||
keyframe: pkt.is_key(),
|
||||
}))
|
||||
}
|
||||
Err(ffmpeg::Error::Other { errno })
|
||||
if errno == ffmpeg::util::error::EAGAIN
|
||||
|| errno == ffmpeg::util::error::EWOULDBLOCK =>
|
||||
{
|
||||
Ok(None)
|
||||
}
|
||||
Err(ffmpeg::Error::Eof) => Ok(None),
|
||||
Err(e) => Err(e).context("receive_packet"),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// CPU upload path (Phase 1): swscale RGB→NV12 → upload into a pooled VA surface → encode.
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
/// VAAPI device + NV12 frames pool (the encoder's input surfaces for the CPU path).
|
||||
struct VaapiHw {
|
||||
device_ref: *mut ffi::AVBufferRef,
|
||||
frames_ref: *mut ffi::AVBufferRef,
|
||||
}
|
||||
|
||||
impl VaapiHw {
|
||||
unsafe fn new(sw_format: ffi::AVPixelFormat, w: u32, h: u32, pool: c_int) -> Result<Self> {
|
||||
let mut device_ref: *mut ffi::AVBufferRef = ptr::null_mut();
|
||||
let node = render_node();
|
||||
let r = ffi::av_hwdevice_ctx_create(
|
||||
&mut device_ref,
|
||||
ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI,
|
||||
node.as_ptr(),
|
||||
ptr::null_mut(),
|
||||
0,
|
||||
);
|
||||
if r < 0 {
|
||||
bail!("no VAAPI device ({:?}): {}", node, ffmpeg::Error::from(r));
|
||||
}
|
||||
let mut frames_ref = ffi::av_hwframe_ctx_alloc(device_ref);
|
||||
if frames_ref.is_null() {
|
||||
ffi::av_buffer_unref(&mut device_ref);
|
||||
bail!("av_hwframe_ctx_alloc(VAAPI) failed");
|
||||
}
|
||||
let fc = (*frames_ref).data as *mut ffi::AVHWFramesContext;
|
||||
(*fc).format = ffi::AVPixelFormat::AV_PIX_FMT_VAAPI;
|
||||
(*fc).sw_format = sw_format;
|
||||
(*fc).width = w as c_int;
|
||||
(*fc).height = h as c_int;
|
||||
(*fc).initial_pool_size = pool;
|
||||
let r = ffi::av_hwframe_ctx_init(frames_ref);
|
||||
if r < 0 {
|
||||
ffi::av_buffer_unref(&mut frames_ref);
|
||||
ffi::av_buffer_unref(&mut device_ref);
|
||||
bail!("av_hwframe_ctx_init(VAAPI) failed ({r})");
|
||||
}
|
||||
Ok(VaapiHw {
|
||||
device_ref,
|
||||
frames_ref,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for VaapiHw {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
ffi::av_buffer_unref(&mut self.frames_ref);
|
||||
ffi::av_buffer_unref(&mut self.device_ref);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct CpuInner {
|
||||
enc: encoder::video::Encoder,
|
||||
hw: VaapiHw,
|
||||
sws: *mut ffi::SwsContext,
|
||||
nv12: *mut ffi::AVFrame, // reusable software NV12 staging frame (swscale dst → upload src)
|
||||
src_format: PixelFormat,
|
||||
width: u32,
|
||||
height: u32,
|
||||
}
|
||||
|
||||
impl CpuInner {
|
||||
fn open(
|
||||
codec: Codec,
|
||||
format: PixelFormat,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
bitrate_bps: u64,
|
||||
) -> Result<Self> {
|
||||
let src_pixel = vaapi_sws_src(format)?;
|
||||
const POOL: c_int = 16;
|
||||
let hw = unsafe { VaapiHw::new(ffi::AVPixelFormat::AV_PIX_FMT_NV12, width, height, POOL)? };
|
||||
let enc = unsafe {
|
||||
open_vaapi_encoder(
|
||||
codec,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
bitrate_bps,
|
||||
hw.device_ref,
|
||||
hw.frames_ref,
|
||||
)?
|
||||
};
|
||||
// swscale RGB→NV12, BT.709 limited (matches the VUI), no rescale.
|
||||
let src_av = pixel_to_av(src_pixel);
|
||||
let sws = unsafe {
|
||||
ffi::sws_getContext(
|
||||
width as c_int,
|
||||
height as c_int,
|
||||
src_av,
|
||||
width as c_int,
|
||||
height as c_int,
|
||||
ffi::AVPixelFormat::AV_PIX_FMT_NV12,
|
||||
SWS_POINT,
|
||||
ptr::null_mut(),
|
||||
ptr::null_mut(),
|
||||
ptr::null(),
|
||||
)
|
||||
};
|
||||
if sws.is_null() {
|
||||
bail!("sws_getContext(RGB→NV12) failed");
|
||||
}
|
||||
unsafe {
|
||||
let cs709 = ffi::sws_getCoefficients(SWS_CS_ITU709);
|
||||
ffi::sws_setColorspaceDetails(sws, cs709, 1, cs709, 0, 0, 1 << 16, 1 << 16);
|
||||
}
|
||||
let nv12 = unsafe {
|
||||
let f = ffi::av_frame_alloc();
|
||||
if f.is_null() {
|
||||
ffi::sws_freeContext(sws);
|
||||
bail!("av_frame_alloc(NV12) failed");
|
||||
}
|
||||
(*f).format = ffi::AVPixelFormat::AV_PIX_FMT_NV12 as c_int;
|
||||
(*f).width = width as c_int;
|
||||
(*f).height = height as c_int;
|
||||
if ffi::av_frame_get_buffer(f, 0) < 0 {
|
||||
let mut f = f;
|
||||
ffi::av_frame_free(&mut f);
|
||||
ffi::sws_freeContext(sws);
|
||||
bail!("av_frame_get_buffer(NV12) failed");
|
||||
}
|
||||
f
|
||||
};
|
||||
tracing::info!(
|
||||
encoder = codec.vaapi_name(),
|
||||
"VAAPI encode active ({width}x{height}@{fps}, CPU→NV12 upload path)"
|
||||
);
|
||||
Ok(CpuInner {
|
||||
enc,
|
||||
hw,
|
||||
sws,
|
||||
nv12,
|
||||
src_format: format,
|
||||
width,
|
||||
height,
|
||||
})
|
||||
}
|
||||
|
||||
fn submit(&mut self, bytes: &[u8], format: PixelFormat, pts: i64, idr: bool) -> Result<()> {
|
||||
anyhow::ensure!(
|
||||
format == self.src_format,
|
||||
"captured format {format:?} != encoder source {:?}",
|
||||
self.src_format
|
||||
);
|
||||
let w = self.width as usize;
|
||||
let h = self.height as usize;
|
||||
let src_row = w * self.src_format.bytes_per_pixel();
|
||||
anyhow::ensure!(bytes.len() >= src_row * h, "captured buffer too small");
|
||||
unsafe {
|
||||
let src_data: [*const u8; 4] = [bytes.as_ptr(), ptr::null(), ptr::null(), ptr::null()];
|
||||
let src_stride: [c_int; 4] = [src_row as c_int, 0, 0, 0];
|
||||
if ffi::sws_scale(
|
||||
self.sws,
|
||||
src_data.as_ptr(),
|
||||
src_stride.as_ptr(),
|
||||
0,
|
||||
h as c_int,
|
||||
(*self.nv12).data.as_ptr(),
|
||||
(*self.nv12).linesize.as_ptr(),
|
||||
) < 0
|
||||
{
|
||||
bail!("sws_scale RGB→NV12 failed");
|
||||
}
|
||||
let mut hwf = ffi::av_frame_alloc();
|
||||
if hwf.is_null() {
|
||||
bail!("av_frame_alloc(hw) failed");
|
||||
}
|
||||
if ffi::av_hwframe_get_buffer(self.hw.frames_ref, hwf, 0) < 0 {
|
||||
ffi::av_frame_free(&mut hwf);
|
||||
bail!("av_hwframe_get_buffer(VAAPI) failed");
|
||||
}
|
||||
if ffi::av_hwframe_transfer_data(hwf, self.nv12, 0) < 0 {
|
||||
ffi::av_frame_free(&mut hwf);
|
||||
bail!("av_hwframe_transfer_data(→VAAPI) failed");
|
||||
}
|
||||
(*hwf).pts = pts;
|
||||
(*hwf).pict_type = if idr {
|
||||
ffi::AVPictureType::AV_PICTURE_TYPE_I
|
||||
} else {
|
||||
ffi::AVPictureType::AV_PICTURE_TYPE_NONE
|
||||
};
|
||||
let r = ffi::avcodec_send_frame(self.enc.as_mut_ptr(), hwf);
|
||||
ffi::av_frame_free(&mut hwf);
|
||||
if r < 0 {
|
||||
bail!("avcodec_send_frame(VAAPI) failed ({r})");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CpuInner {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
if !self.nv12.is_null() {
|
||||
ffi::av_frame_free(&mut self.nv12);
|
||||
}
|
||||
if !self.sws.is_null() {
|
||||
ffi::sws_freeContext(self.sws);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Zero-copy dmabuf path: DRM-PRIME → hwmap(vaapi) → scale_vaapi(nv12) filter graph → encode.
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
struct DmabufInner {
|
||||
enc: encoder::video::Encoder,
|
||||
/// DRM device the source dmabuf frames reference (the buffersrc's `hw_frames_ctx` device).
|
||||
drm_device: *mut ffi::AVBufferRef,
|
||||
/// VAAPI device driving `hwmap`/`scale_vaapi`/the encoder.
|
||||
vaapi_device: *mut ffi::AVBufferRef,
|
||||
/// DRM-PRIME frames context for the imported dmabufs (buffersrc input).
|
||||
drm_frames: *mut ffi::AVBufferRef,
|
||||
graph: *mut ffi::AVFilterGraph,
|
||||
src: *mut ffi::AVFilterContext,
|
||||
sink: *mut ffi::AVFilterContext,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fourcc: u32,
|
||||
}
|
||||
|
||||
impl DmabufInner {
|
||||
fn open(
|
||||
codec: Codec,
|
||||
format: PixelFormat,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
bitrate_bps: u64,
|
||||
) -> Result<Self> {
|
||||
let drm_fourcc = crate::zerocopy::drm_fourcc(format)
|
||||
.ok_or_else(|| anyhow!("no DRM fourcc for {format:?} (VAAPI zero-copy)"))?;
|
||||
let node = render_node();
|
||||
unsafe {
|
||||
// DRM device (source dmabuf frames) + a VAAPI device derived from it (same GPU) for
|
||||
// hwmap/scale_vaapi/the encoder.
|
||||
let mut drm_device: *mut ffi::AVBufferRef = ptr::null_mut();
|
||||
let r = ffi::av_hwdevice_ctx_create(
|
||||
&mut drm_device,
|
||||
ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_DRM,
|
||||
node.as_ptr(),
|
||||
ptr::null_mut(),
|
||||
0,
|
||||
);
|
||||
if r < 0 {
|
||||
bail!(
|
||||
"av_hwdevice_ctx_create(DRM {:?}): {}",
|
||||
node,
|
||||
ffmpeg::Error::from(r)
|
||||
);
|
||||
}
|
||||
let mut vaapi_device: *mut ffi::AVBufferRef = ptr::null_mut();
|
||||
let r = ffi::av_hwdevice_ctx_create_derived(
|
||||
&mut vaapi_device,
|
||||
ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI,
|
||||
drm_device,
|
||||
0,
|
||||
);
|
||||
if r < 0 {
|
||||
ffi::av_buffer_unref(&mut drm_device);
|
||||
bail!("derive VAAPI from DRM: {}", ffmpeg::Error::from(r));
|
||||
}
|
||||
|
||||
// DRM-PRIME frames context for the imported dmabufs.
|
||||
let mut drm_frames = ffi::av_hwframe_ctx_alloc(drm_device);
|
||||
if drm_frames.is_null() {
|
||||
ffi::av_buffer_unref(&mut vaapi_device);
|
||||
ffi::av_buffer_unref(&mut drm_device);
|
||||
bail!("av_hwframe_ctx_alloc(DRM) failed");
|
||||
}
|
||||
let fc = (*drm_frames).data as *mut ffi::AVHWFramesContext;
|
||||
(*fc).format = ffi::AVPixelFormat::AV_PIX_FMT_DRM_PRIME;
|
||||
(*fc).sw_format = ffi::AVPixelFormat::AV_PIX_FMT_BGR0; // packed XR24 RGB plane
|
||||
(*fc).width = width as c_int;
|
||||
(*fc).height = height as c_int;
|
||||
if ffi::av_hwframe_ctx_init(drm_frames) < 0 {
|
||||
ffi::av_buffer_unref(&mut drm_frames);
|
||||
ffi::av_buffer_unref(&mut vaapi_device);
|
||||
ffi::av_buffer_unref(&mut drm_device);
|
||||
bail!("av_hwframe_ctx_init(DRM) failed");
|
||||
}
|
||||
|
||||
// Filter graph: buffer(drm_prime) → hwmap=derive_device=vaapi:mode=read →
|
||||
// scale_vaapi=format=nv12 → buffersink.
|
||||
let mut graph = ffi::avfilter_graph_alloc();
|
||||
if graph.is_null() {
|
||||
ffi::av_buffer_unref(&mut drm_frames);
|
||||
ffi::av_buffer_unref(&mut vaapi_device);
|
||||
ffi::av_buffer_unref(&mut drm_device);
|
||||
bail!("avfilter_graph_alloc failed");
|
||||
}
|
||||
|
||||
let mk = |name: &CStr, inst: &CStr| -> *mut ffi::AVFilterContext {
|
||||
let f = ffi::avfilter_get_by_name(name.as_ptr());
|
||||
if f.is_null() {
|
||||
return ptr::null_mut();
|
||||
}
|
||||
ffi::avfilter_graph_alloc_filter(graph, f, inst.as_ptr())
|
||||
};
|
||||
let src = mk(c"buffer", c"in");
|
||||
let hwmap = mk(c"hwmap", c"map");
|
||||
let scale = mk(c"scale_vaapi", c"csc");
|
||||
let sink = mk(c"buffersink", c"out");
|
||||
if src.is_null() || hwmap.is_null() || scale.is_null() || sink.is_null() {
|
||||
ffi::avfilter_graph_free(&mut graph);
|
||||
ffi::av_buffer_unref(&mut drm_frames);
|
||||
ffi::av_buffer_unref(&mut vaapi_device);
|
||||
ffi::av_buffer_unref(&mut drm_device);
|
||||
bail!("a VAAPI filter (buffer/hwmap/scale_vaapi/buffersink) is missing");
|
||||
}
|
||||
// hwmap maps the DRM-PRIME input onto THIS vaapi device; scale_vaapi runs the CSC on
|
||||
// it. Giving both our device (rather than `hwmap=derive_device`) keeps every surface —
|
||||
// and the sink's output frames ctx the encoder adopts — on one VADisplay.
|
||||
(*hwmap).hw_device_ctx = ffi::av_buffer_ref(vaapi_device);
|
||||
(*scale).hw_device_ctx = ffi::av_buffer_ref(vaapi_device);
|
||||
|
||||
// buffersrc params: DRM-PRIME frames, the drm_frames ctx.
|
||||
let par = ffi::av_buffersrc_parameters_alloc();
|
||||
(*par).format = ffi::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as c_int;
|
||||
(*par).width = width as c_int;
|
||||
(*par).height = height as c_int;
|
||||
(*par).time_base = ffi::AVRational {
|
||||
num: 1,
|
||||
den: fps as c_int,
|
||||
};
|
||||
(*par).hw_frames_ctx = ffi::av_buffer_ref(drm_frames);
|
||||
let r = ffi::av_buffersrc_parameters_set(src, par);
|
||||
ffi::av_free(par as *mut _);
|
||||
if r < 0 {
|
||||
ffi::avfilter_graph_free(&mut graph);
|
||||
ffi::av_buffer_unref(&mut drm_frames);
|
||||
ffi::av_buffer_unref(&mut vaapi_device);
|
||||
ffi::av_buffer_unref(&mut drm_device);
|
||||
bail!("av_buffersrc_parameters_set failed ({r})");
|
||||
}
|
||||
macro_rules! init {
|
||||
($ctx:expr, $args:expr, $what:literal) => {{
|
||||
let r = ffi::avfilter_init_str($ctx, $args);
|
||||
if r < 0 {
|
||||
ffi::avfilter_graph_free(&mut graph);
|
||||
ffi::av_buffer_unref(&mut drm_frames);
|
||||
ffi::av_buffer_unref(&mut vaapi_device);
|
||||
ffi::av_buffer_unref(&mut drm_device);
|
||||
bail!(concat!("init ", $what, " failed ({})"), r);
|
||||
}
|
||||
}};
|
||||
}
|
||||
init!(src, ptr::null(), "buffer");
|
||||
init!(hwmap, c"mode=read".as_ptr(), "hwmap");
|
||||
init!(scale, c"format=nv12".as_ptr(), "scale_vaapi");
|
||||
init!(sink, ptr::null(), "buffersink");
|
||||
|
||||
let link = |a: *mut ffi::AVFilterContext, b: *mut ffi::AVFilterContext| -> c_int {
|
||||
ffi::avfilter_link(a, 0, b, 0)
|
||||
};
|
||||
if link(src, hwmap) < 0 || link(hwmap, scale) < 0 || link(scale, sink) < 0 {
|
||||
ffi::avfilter_graph_free(&mut graph);
|
||||
ffi::av_buffer_unref(&mut drm_frames);
|
||||
ffi::av_buffer_unref(&mut vaapi_device);
|
||||
ffi::av_buffer_unref(&mut drm_device);
|
||||
bail!("avfilter_link failed");
|
||||
}
|
||||
let r = ffi::avfilter_graph_config(graph, ptr::null_mut());
|
||||
if r < 0 {
|
||||
ffi::avfilter_graph_free(&mut graph);
|
||||
ffi::av_buffer_unref(&mut drm_frames);
|
||||
ffi::av_buffer_unref(&mut vaapi_device);
|
||||
ffi::av_buffer_unref(&mut drm_device);
|
||||
bail!("avfilter_graph_config failed ({r})");
|
||||
}
|
||||
|
||||
// The encoder takes NV12 surfaces from the sink's output frames context.
|
||||
let nv12_ctx = ffi::av_buffersink_get_hw_frames_ctx(sink);
|
||||
if nv12_ctx.is_null() {
|
||||
ffi::avfilter_graph_free(&mut graph);
|
||||
ffi::av_buffer_unref(&mut drm_frames);
|
||||
ffi::av_buffer_unref(&mut vaapi_device);
|
||||
ffi::av_buffer_unref(&mut drm_device);
|
||||
bail!("filter sink has no VAAPI frames context");
|
||||
}
|
||||
let enc = open_vaapi_encoder(
|
||||
codec,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
bitrate_bps,
|
||||
vaapi_device,
|
||||
nv12_ctx,
|
||||
)?;
|
||||
|
||||
tracing::info!(
|
||||
encoder = codec.vaapi_name(),
|
||||
"VAAPI encode active ({width}x{height}@{fps}, zero-copy dmabuf → GPU NV12)"
|
||||
);
|
||||
Ok(DmabufInner {
|
||||
enc,
|
||||
drm_device,
|
||||
vaapi_device,
|
||||
drm_frames,
|
||||
graph,
|
||||
src,
|
||||
sink,
|
||||
width,
|
||||
height,
|
||||
fourcc: drm_fourcc,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn submit(&mut self, dmabuf: &DmabufFrame, pts: i64, idr: bool) -> Result<()> {
|
||||
anyhow::ensure!(
|
||||
dmabuf.fourcc == self.fourcc,
|
||||
"dmabuf fourcc {:#x} != encoder {:#x}",
|
||||
dmabuf.fourcc,
|
||||
self.fourcc
|
||||
);
|
||||
unsafe {
|
||||
// Build a DRM-PRIME AVFrame describing the dmabuf (one object/fd, one layer/plane).
|
||||
let mut desc: Box<ffi::AVDRMFrameDescriptor> = Box::new(std::mem::zeroed());
|
||||
desc.nb_objects = 1;
|
||||
desc.objects[0].fd = dmabuf.fd.as_raw_fd();
|
||||
desc.objects[0].size = 0;
|
||||
desc.objects[0].format_modifier = dmabuf.modifier;
|
||||
desc.nb_layers = 1;
|
||||
desc.layers[0].format = self.fourcc;
|
||||
desc.layers[0].nb_planes = 1;
|
||||
desc.layers[0].planes[0].object_index = 0;
|
||||
desc.layers[0].planes[0].offset = dmabuf.offset as isize;
|
||||
desc.layers[0].planes[0].pitch = dmabuf.stride as isize;
|
||||
|
||||
let mut drm = ffi::av_frame_alloc();
|
||||
if drm.is_null() {
|
||||
bail!("av_frame_alloc(drm) failed");
|
||||
}
|
||||
(*drm).format = ffi::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as c_int;
|
||||
(*drm).width = self.width as c_int;
|
||||
(*drm).height = self.height as c_int;
|
||||
(*drm).hw_frames_ctx = ffi::av_buffer_ref(self.drm_frames);
|
||||
(*drm).data[0] = Box::into_raw(desc) as *mut u8;
|
||||
// Own the descriptor so it frees with the frame (the fd is owned by the DmabufFrame,
|
||||
// which outlives this call — the graph reads the surface before submit returns).
|
||||
extern "C" fn free_desc(_opaque: *mut std::ffi::c_void, data: *mut u8) {
|
||||
unsafe { drop(Box::from_raw(data as *mut ffi::AVDRMFrameDescriptor)) };
|
||||
}
|
||||
(*drm).buf[0] = ffi::av_buffer_create(
|
||||
(*drm).data[0],
|
||||
std::mem::size_of::<ffi::AVDRMFrameDescriptor>(),
|
||||
Some(free_desc),
|
||||
ptr::null_mut(),
|
||||
0,
|
||||
);
|
||||
|
||||
// Push through hwmap → scale_vaapi; pull the NV12 surface back out.
|
||||
let r = ffi::av_buffersrc_add_frame_flags(
|
||||
self.src,
|
||||
drm,
|
||||
ffi::AV_BUFFERSRC_FLAG_KEEP_REF as c_int,
|
||||
);
|
||||
ffi::av_frame_free(&mut drm);
|
||||
if r < 0 {
|
||||
bail!("av_buffersrc_add_frame failed ({r})");
|
||||
}
|
||||
let mut nv12 = ffi::av_frame_alloc();
|
||||
if nv12.is_null() {
|
||||
bail!("av_frame_alloc(nv12) failed");
|
||||
}
|
||||
let r = ffi::av_buffersink_get_frame(self.sink, nv12);
|
||||
if r < 0 {
|
||||
ffi::av_frame_free(&mut nv12);
|
||||
bail!("av_buffersink_get_frame failed ({r})");
|
||||
}
|
||||
(*nv12).pts = pts;
|
||||
(*nv12).pict_type = if idr {
|
||||
ffi::AVPictureType::AV_PICTURE_TYPE_I
|
||||
} else {
|
||||
ffi::AVPictureType::AV_PICTURE_TYPE_NONE
|
||||
};
|
||||
let r = ffi::avcodec_send_frame(self.enc.as_mut_ptr(), nv12);
|
||||
ffi::av_frame_free(&mut nv12);
|
||||
if r < 0 {
|
||||
bail!("avcodec_send_frame(VAAPI) failed ({r})");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DmabufInner {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
ffi::avfilter_graph_free(&mut self.graph);
|
||||
ffi::av_buffer_unref(&mut self.drm_frames);
|
||||
ffi::av_buffer_unref(&mut self.vaapi_device);
|
||||
ffi::av_buffer_unref(&mut self.drm_device);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
enum Inner {
|
||||
Cpu(CpuInner),
|
||||
Dmabuf(DmabufInner),
|
||||
}
|
||||
|
||||
pub struct VaapiEncoder {
|
||||
codec: Codec,
|
||||
format: PixelFormat,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
bitrate_bps: u64,
|
||||
/// Built lazily from the first frame's payload (CPU upload vs zero-copy dmabuf).
|
||||
inner: Option<Inner>,
|
||||
frame_idx: i64,
|
||||
force_kf: bool,
|
||||
}
|
||||
|
||||
// Raw FFI pointers; the encoder lives on a single thread (same contract as `NvencEncoder`).
|
||||
unsafe impl Send for VaapiEncoder {}
|
||||
|
||||
impl VaapiEncoder {
|
||||
pub fn open(
|
||||
codec: Codec,
|
||||
format: PixelFormat,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
bitrate_bps: u64,
|
||||
bit_depth: u8,
|
||||
) -> Result<Self> {
|
||||
if bit_depth != 8 {
|
||||
tracing::warn!(bit_depth, "VAAPI 10-bit not yet wired — encoding 8-bit");
|
||||
}
|
||||
ffmpeg::init().context("ffmpeg init")?;
|
||||
if std::env::var_os("PUNKTFUNK_FFMPEG_DEBUG").is_some() {
|
||||
unsafe { ffi::av_log_set_level(48) };
|
||||
}
|
||||
// Validate the codec/format up front so a bad request fails at open, not on the first frame.
|
||||
let _ = vaapi_sws_src(format)?;
|
||||
Ok(VaapiEncoder {
|
||||
codec,
|
||||
format,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
bitrate_bps,
|
||||
inner: None,
|
||||
frame_idx: 0,
|
||||
force_kf: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn ensure_inner(&mut self, want_dmabuf: bool) -> Result<&mut Inner> {
|
||||
if self.inner.is_none() {
|
||||
let inner = if want_dmabuf {
|
||||
Inner::Dmabuf(DmabufInner::open(
|
||||
self.codec,
|
||||
self.format,
|
||||
self.width,
|
||||
self.height,
|
||||
self.fps,
|
||||
self.bitrate_bps,
|
||||
)?)
|
||||
} else {
|
||||
Inner::Cpu(CpuInner::open(
|
||||
self.codec,
|
||||
self.format,
|
||||
self.width,
|
||||
self.height,
|
||||
self.fps,
|
||||
self.bitrate_bps,
|
||||
)?)
|
||||
};
|
||||
self.inner = Some(inner);
|
||||
}
|
||||
Ok(self.inner.as_mut().unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
impl Encoder for VaapiEncoder {
|
||||
fn submit(&mut self, captured: &CapturedFrame) -> Result<()> {
|
||||
anyhow::ensure!(
|
||||
captured.width == self.width && captured.height == self.height,
|
||||
"captured frame {}x{} != encoder {}x{}",
|
||||
captured.width,
|
||||
captured.height,
|
||||
self.width,
|
||||
self.height
|
||||
);
|
||||
let pts = self.frame_idx;
|
||||
self.frame_idx += 1;
|
||||
let idr = self.force_kf;
|
||||
self.force_kf = false;
|
||||
match &captured.payload {
|
||||
FramePayload::Cpu(bytes) => match self.ensure_inner(false)? {
|
||||
Inner::Cpu(c) => c.submit(bytes, captured.format, pts, idr),
|
||||
Inner::Dmabuf(_) => bail!("VAAPI encoder built for dmabuf got a CPU frame"),
|
||||
},
|
||||
FramePayload::Dmabuf(d) => match self.ensure_inner(true)? {
|
||||
Inner::Dmabuf(dm) => dm.submit(d, pts, idr),
|
||||
Inner::Cpu(_) => bail!("VAAPI encoder built for CPU got a dmabuf frame"),
|
||||
},
|
||||
FramePayload::Cuda(_) => bail!(
|
||||
"VAAPI encoder received a CUDA frame — that payload is NVENC-only; \
|
||||
unset PUNKTFUNK_ZEROCOPY or don't force PUNKTFUNK_ENCODER=vaapi on an NVIDIA host"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn request_keyframe(&mut self) {
|
||||
self.force_kf = true;
|
||||
}
|
||||
|
||||
fn poll(&mut self) -> Result<Option<EncodedFrame>> {
|
||||
match &mut self.inner {
|
||||
Some(Inner::Cpu(c)) => poll_encoder(&mut c.enc, self.fps),
|
||||
Some(Inner::Dmabuf(d)) => poll_encoder(&mut d.enc, self.fps),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Result<()> {
|
||||
match &mut self.inner {
|
||||
Some(Inner::Cpu(c)) => c.enc.send_eof().context("send_eof")?,
|
||||
Some(Inner::Dmabuf(d)) => d.enc.send_eof().context("send_eof")?,
|
||||
None => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user