refactor(host/W6.2): extract the video encode backends into the pf-encode crate
encode.rs + encode/* (NVENC, VAAPI, native AMF, AMF/QSV ffmpeg, direct-SDK NVENC/CUDA, raw Vulkan-Video, PyroWave, openh264) move into crates/pf-encode behind one Encoder trait + open_video selector (plan §W6). The crate speaks the shared frame vocabulary (pf-frame: CapturedFrame/PixelFormat + the DXGI identity D3d11Frame/make_device) and pf-zerocopy (CUDA context/buffers), and NEVER pf-capture — the capture→encode edge is one-way (ZeroCopyPolicy, prior commit). Dep moves: the heavy encoder deps (ffmpeg-next, the NVENC SDK, openh264, pyrowave-sys) move from the host to pf-encode; the host's nvenc/amf-qsv/vulkan-encode/pyrowave features now FORWARD to pf-encode/*. The host keeps a mod-encode shim (pub use pf_encode) so every crate::encode::* path (negotiator + GameStream/native/mgmt planes) is unchanged. resolve_render_adapter_luid moves from the host's windows/win_adapter.rs into pf-gpu (both pf-encode and pf-capture need it as a peer of GPU selection); its 5 call sites (encode amf/nvenc, capture idd_push/synthetic_nv12, vdisplay manager) rewire to pf_gpu::resolve_render_adapter_luid and win_adapter.rs is deleted. pf-frame's make_device gains a # Safety section (public-unsafe-fn lint, latent since the pf-frame carve — a full-workspace -D warnings clippy catches it). Verified: Linux clippy -D warnings (pf-encode + host nvenc,vulkan-encode,pyrowave --all-targets) + 13/13 pf-encode + 299/299 host tests; Windows clippy -D warnings (pf-encode nvenc,amf-qsv --all-targets + host nvenc,amf-qsv --all-targets) Finished exit 0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
# Hardware/software video encode (plan §7 / §W6): the per-vendor backends (NVENC, VAAPI, AMF, QSV,
|
||||
# Vulkan-Video, PyroWave, openh264) behind one `Encoder` trait + `open_video` selector, extracted
|
||||
# from the host so it depends on the shared frame vocabulary (pf-frame) rather than living inside
|
||||
# the orchestrator. Speaks pf-frame (CapturedFrame/PixelFormat/dxgi identity) and pf-zerocopy
|
||||
# (CUDA), never pf-capture — the capture→encode edge is one-way (plan §2.4).
|
||||
[package]
|
||||
name = "pf-encode"
|
||||
version = "0.12.0"
|
||||
edition = "2021"
|
||||
rust-version.workspace = true
|
||||
license = "MIT OR Apache-2.0"
|
||||
description = "punktfunk host video encode: NVENC/VAAPI/AMF/QSV/Vulkan-Video/PyroWave/openh264 backends behind one Encoder trait."
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
punktfunk-core = { path = "../punktfunk-core", features = ["quic"] }
|
||||
pf-frame = { path = "../pf-frame" }
|
||||
pf-gpu = { path = "../pf-gpu" }
|
||||
pf-host-config = { path = "../pf-host-config" }
|
||||
pf-zerocopy = { path = "../pf-zerocopy" }
|
||||
anyhow = "1"
|
||||
tracing = "0.1"
|
||||
|
||||
[dev-dependencies]
|
||||
# A test writer for the NVENC backend's unit tests (`with_test_writer().try_init()`).
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
[target.'cfg(any(target_os = "linux", target_os = "windows"))'.dependencies]
|
||||
# Software H.264 (openh264, BSD-2) — the GPU-less encode path on both platforms.
|
||||
openh264 = "0.9"
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
# libavcodec (NVENC libav + VAAPI backends). `ffmpeg-sys-next` auto-detects the FFmpeg version.
|
||||
ffmpeg-next = "8"
|
||||
libc = "0.2"
|
||||
# Vulkan bindings for the raw Vulkan-Video encode + PyroWave compute backends (feature-gated below;
|
||||
# the dep stays unconditional to mirror the host's Linux target — unused-but-declared is harmless).
|
||||
ash = "0.38"
|
||||
# `libnvidia-encode.so.1` is dlopen'd at runtime for the direct-SDK NVENC/CUDA backend.
|
||||
libloading = "0.8"
|
||||
# Direct-SDK NVENC (raw `sys::nvEncodeAPI` types; entry points resolved at runtime). `ci-check` =
|
||||
# vendored bindings, no CUDA toolkit at build.
|
||||
nvidia-video-codec-sdk = { version = "0.4", features = ["ci-check"], optional = true }
|
||||
# PyroWave (opt-in wired-LAN wavelet codec) — vendored codec + bindgen'd C API, only under `pyrowave`.
|
||||
pyrowave-sys = { path = "../pyrowave-sys", optional = true }
|
||||
|
||||
[target.'cfg(target_os = "windows")'.dependencies]
|
||||
# NVENC (direct SDK, D3D11 input) + the shared D3D11/DXGI vocabulary via pf-frame.
|
||||
nvidia-video-codec-sdk = { version = "0.4", features = ["ci-check"], optional = true }
|
||||
# AMD (AMF) + Intel (QSV) hardware encode via libavcodec (behind `amf-qsv`; link-imports FFmpeg).
|
||||
ffmpeg-next = { version = "8", optional = true }
|
||||
# `libnvidia-encode`/`nvEncodeAPI64.dll` resolved at runtime; the NVENC status→cause table dlopen.
|
||||
libloading = "0.8"
|
||||
windows = { version = "0.62", features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_Graphics_Direct3D",
|
||||
"Win32_Graphics_Direct3D11",
|
||||
"Win32_Graphics_Dxgi",
|
||||
"Win32_Graphics_Dxgi_Common",
|
||||
"Win32_Storage_FileSystem",
|
||||
"Win32_System_LibraryLoader",
|
||||
"Win32_System_Threading",
|
||||
] }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# NVENC hardware encode (Linux CUDA + Windows D3D11); entry points resolved at runtime.
|
||||
nvenc = ["dep:nvidia-video-codec-sdk"]
|
||||
# AMD (AMF) + Intel (QSV) hardware encode on Windows via libavcodec.
|
||||
amf-qsv = ["dep:ffmpeg-next"]
|
||||
# Raw Vulkan-Video HEVC/AV1 encode on Linux (reuses the `ash` bindings; no new dep).
|
||||
vulkan-encode = []
|
||||
# PyroWave — the opt-in wired-LAN intra-only wavelet codec (Linux encode backend).
|
||||
pyrowave = ["dep:pyrowave-sys"]
|
||||
@@ -0,0 +1,424 @@
|
||||
//! The encoder contract (plan §7, Tier 1): the [`Encoder`] trait plus the plain-data value types its
|
||||
//! signatures use — [`EncodedFrame`], [`Codec`], [`ChromaFormat`], [`EncoderCaps`] — and the
|
||||
//! dimension/VBV helpers [`validate_dimensions`] and [`vbv_frames_env`]. Backend selection, the
|
||||
//! capability probes that mirror it, and `Codec::host_wire_caps` stay in the parent the `pf-encode` crate root
|
||||
//! facade, which re-exports this module (`pub(crate) use codec::*;`) so every `crate::*` path
|
||||
//! is unchanged.
|
||||
use anyhow::Result;
|
||||
use pf_frame::CapturedFrame;
|
||||
|
||||
/// An encoded access unit (one NAL/AU) to hand to `punktfunk_core` for FEC + packetization.
|
||||
/// `data` is in-band Annex-B (the encoder is opened without a global header), so each
|
||||
/// keyframe carries its own VPS/SPS/PPS — the bytes are both a playable elementary
|
||||
/// stream and a self-contained AU for the wire.
|
||||
pub struct EncodedFrame {
|
||||
pub data: Vec<u8>,
|
||||
pub pts_ns: u64,
|
||||
/// True for IDR/keyframes (sets the SOF/keyframe wire flags).
|
||||
pub keyframe: bool,
|
||||
/// True when this AU is a **reference-frame-invalidation recovery frame** — a clean P-frame the
|
||||
/// encoder coded against a known-good reference in response to
|
||||
/// [`invalidate_ref_frames`](Encoder::invalidate_ref_frames). The pump tags it
|
||||
/// [`punktfunk_core::packet::USER_FLAG_RECOVERY_ANCHOR`] so the client lifts its post-loss
|
||||
/// freeze on it without an IDR. Set by BOTH RFI backends: native AMF (the LTR force-reference
|
||||
/// frame) and Windows direct-NVENC (the first frame encoded after `nvEncInvalidateRefFrames` —
|
||||
/// the invalidation applies at the next `encode_picture`, so that AU is by construction the
|
||||
/// clean re-anchor). Without it the client's freeze can only lift on an IDR — which the host
|
||||
/// suppresses after a successful RFI (the cooldown), a ~1 s frozen stall per loss event.
|
||||
pub recovery_anchor: bool,
|
||||
/// The AU is shard-aligned self-delimiting chunks (see [`Encoder::set_wire_chunking`]);
|
||||
/// the session stamps [`punktfunk_core::packet::USER_FLAG_CHUNK_ALIGNED`] so the client
|
||||
/// windows its parse and may opt into partial delivery. Only the PyroWave backend sets it.
|
||||
pub chunk_aligned: bool,
|
||||
}
|
||||
|
||||
/// Codec selection negotiated with the client.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Codec {
|
||||
H264,
|
||||
H265,
|
||||
Av1,
|
||||
/// PyroWave — the opt-in wired-LAN intra-only wavelet codec (design/pyrowave-codec-plan.md).
|
||||
/// Only ever negotiated via the client's explicit `preferred_codec` (never the precedence
|
||||
/// ladder) and only emitted by the `pyrowave`-feature backend; every AU is a keyframe.
|
||||
PyroWave,
|
||||
}
|
||||
|
||||
/// Chroma subsampling the encoder emits, negotiated with the client (the `PUNKTFUNK_444` gate + the
|
||||
/// client's `VIDEO_CAP_444` + a GPU probe). `Yuv420` is the universal default; `Yuv444` is HEVC-only,
|
||||
/// native-protocol-only (GameStream stays 4:2:0), and the host only ever passes it after
|
||||
/// [`can_encode_444`] confirmed the active backend supports it.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)]
|
||||
pub enum ChromaFormat {
|
||||
#[default]
|
||||
Yuv420,
|
||||
Yuv444,
|
||||
}
|
||||
|
||||
impl ChromaFormat {
|
||||
/// The HEVC `chroma_format_idc` this maps to: `1` (4:2:0) or `3` (4:4:4). Also the wire value
|
||||
/// echoed in [`punktfunk_core::quic::Welcome::chroma_format`].
|
||||
pub fn idc(self) -> u8 {
|
||||
match self {
|
||||
ChromaFormat::Yuv420 => punktfunk_core::quic::CHROMA_IDC_420,
|
||||
ChromaFormat::Yuv444 => punktfunk_core::quic::CHROMA_IDC_444,
|
||||
}
|
||||
}
|
||||
|
||||
/// True for full-chroma 4:4:4.
|
||||
pub fn is_444(self) -> bool {
|
||||
matches!(self, ChromaFormat::Yuv444)
|
||||
}
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
/// Map a negotiated `quic` codec bit ([`punktfunk_core::quic::CODEC_H264`] etc.) to the encoder
|
||||
/// [`Codec`]. Unknown / `0` → HEVC (the pre-negotiation default). Inverse of [`Codec::to_wire`].
|
||||
pub fn from_wire(bit: u8) -> Codec {
|
||||
match bit {
|
||||
punktfunk_core::quic::CODEC_H264 => Codec::H264,
|
||||
punktfunk_core::quic::CODEC_AV1 => Codec::Av1,
|
||||
punktfunk_core::quic::CODEC_PYROWAVE => Codec::PyroWave,
|
||||
_ => Codec::H265,
|
||||
}
|
||||
}
|
||||
|
||||
/// The single `quic` codec bit for this codec (echoed in [`punktfunk_core::quic::Welcome::codec`]).
|
||||
pub fn to_wire(self) -> u8 {
|
||||
match self {
|
||||
Codec::H264 => punktfunk_core::quic::CODEC_H264,
|
||||
Codec::H265 => punktfunk_core::quic::CODEC_HEVC,
|
||||
Codec::Av1 => punktfunk_core::quic::CODEC_AV1,
|
||||
Codec::PyroWave => punktfunk_core::quic::CODEC_PYROWAVE,
|
||||
}
|
||||
}
|
||||
|
||||
/// Lowercase stats/console label (`"h264"` / `"hevc"` / `"av1"`) — the codec string seeded into
|
||||
/// the web console's session meta (the host `stats_recorder::StatsRecorder::register_session`).
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
Codec::H264 => "h264",
|
||||
Codec::H265 => "hevc",
|
||||
Codec::Av1 => "av1",
|
||||
Codec::PyroWave => "pyrowave",
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this codec has a negotiable **10-bit** encode path (HEVC Main10 / AV1 10-bit).
|
||||
/// H.264 is always 8-bit (High10 is neither an NVENC nor a VCN encode mode — negotiation
|
||||
/// never asks), and PyroWave's wavelet path ingests 8-bit. `true` here is only the
|
||||
/// *codec-level* gate: the active GPU/backend must still pass
|
||||
/// [`can_encode_10bit`](crate::can_encode_10bit) before the host negotiates 10-bit.
|
||||
pub fn supports_10bit(self) -> bool {
|
||||
matches!(self, Codec::H265 | Codec::Av1)
|
||||
}
|
||||
|
||||
/// The FFmpeg NVENC encoder name (selected by name, not codec id — the latter would
|
||||
/// pick the software encoder).
|
||||
pub fn nvenc_name(self) -> &'static str {
|
||||
match self {
|
||||
Codec::H264 => "h264_nvenc",
|
||||
Codec::H265 => "hevc_nvenc",
|
||||
Codec::Av1 => "av1_nvenc",
|
||||
// Guarded by the open_video dispatch: a PyroWave session never reaches a
|
||||
// libavcodec backend.
|
||||
Codec::PyroWave => unreachable!("PyroWave has no FFmpeg encoder"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The FFmpeg VAAPI encoder name (AMD via Mesa `radeonsi`, Intel via `iHD`/`i965`). One
|
||||
/// libavcodec encoder per codec covers both vendors — the kernel driver differs, the libva
|
||||
/// userspace API is identical. Selected by name (the codec id would pick the SW encoder).
|
||||
/// AV1 VAAPI encode is narrow (Intel Arc/Xe2+, AMD RDNA3+/RDNA4) — gate it on a capability
|
||||
/// probe, never assume it (see [`open_video`]).
|
||||
pub fn vaapi_name(self) -> &'static str {
|
||||
match self {
|
||||
Codec::H264 => "h264_vaapi",
|
||||
Codec::H265 => "hevc_vaapi",
|
||||
Codec::Av1 => "av1_vaapi",
|
||||
// Guarded by the open_video dispatch: a PyroWave session never reaches a
|
||||
// libavcodec backend.
|
||||
Codec::PyroWave => unreachable!("PyroWave has no FFmpeg encoder"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The FFmpeg AMD **AMF** encoder name (the Windows AMD backend). Selected by name (the codec id
|
||||
/// would pick the software encoder). AV1 (`av1_amf`) is RDNA3+/RX 7000+ — probe, never assume.
|
||||
pub fn amf_name(self) -> &'static str {
|
||||
match self {
|
||||
Codec::H264 => "h264_amf",
|
||||
Codec::H265 => "hevc_amf",
|
||||
Codec::Av1 => "av1_amf",
|
||||
// Guarded by the open_video dispatch: a PyroWave session never reaches a
|
||||
// libavcodec backend.
|
||||
Codec::PyroWave => unreachable!("PyroWave has no FFmpeg encoder"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The FFmpeg Intel **QSV** encoder name (the Windows Intel backend). Selected by name. AV1
|
||||
/// (`av1_qsv`) is Arc/Xe2+; HEVC Main10 is Gen9.5+ — probe, never assume.
|
||||
pub fn qsv_name(self) -> &'static str {
|
||||
match self {
|
||||
Codec::H264 => "h264_qsv",
|
||||
Codec::H265 => "hevc_qsv",
|
||||
Codec::Av1 => "av1_qsv",
|
||||
// Guarded by the open_video dispatch: a PyroWave session never reaches a
|
||||
// libavcodec backend.
|
||||
Codec::PyroWave => unreachable!("PyroWave has no FFmpeg encoder"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Static capabilities an [`Encoder`] declares so the session glue routes loss-recovery and HDR
|
||||
/// plumbing by *query* rather than relying on a method's no-op/`false` default. Cheap `Copy`; fixed
|
||||
/// for the session (an HDR toggle re-initialises the encoder — re-query if that matters).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct EncoderCaps {
|
||||
/// The encoder can perform real reference-frame invalidation — i.e.
|
||||
/// [`invalidate_ref_frames`](Encoder::invalidate_ref_frames) can return `true`. When `false`
|
||||
/// the caller skips that always-`false` call and forces a keyframe directly on loss recovery.
|
||||
/// Two backends implement RFI: Windows direct-NVENC (`nvEncInvalidateRefFrames`) and native
|
||||
/// AMF (user-LTR force-reference, when the driver accepted the LTR slots at open). The
|
||||
/// libavcodec paths (Linux NVENC, VAAPI, QSV) can't express it and always keyframe.
|
||||
pub supports_rfi: bool,
|
||||
/// The encoder emits in-band HDR mastering/CLL SEI from [`set_hdr_meta`](Encoder::set_hdr_meta).
|
||||
/// When `false`, `set_hdr_meta` is a no-op and no in-band grade reaches the client. Only the
|
||||
/// Windows direct-NVENC path attaches it today.
|
||||
pub supports_hdr_metadata: bool,
|
||||
/// The opened encoder is actually producing a full-chroma 4:4:4 (`chroma_format_idc = 3`) stream.
|
||||
/// `false` on every 4:2:0 session (the default) and on a backend that declined 4:4:4. Set by the
|
||||
/// NVENC backends (Linux + Windows). The chroma is committed to the wire (`Welcome::chroma_format`)
|
||||
/// from the pre-open probe, so this is a *post-open cross-check*: the session glue logs loudly if
|
||||
/// the encoder's real chroma disagrees with what was negotiated (the in-band SPS is authoritative
|
||||
/// for the decoder either way).
|
||||
pub chroma_444: bool,
|
||||
/// The encoder runs a periodic **intra-refresh wave** — a moving band of intra blocks that
|
||||
/// re-codes the whole picture over ~0.5 s, no periodic IDR. FEC-unrecoverable loss self-heals as
|
||||
/// the band sweeps, so the session glue rate-limits client keyframe requests instead of answering
|
||||
/// each with a full IDR (the 20-40× frame-size spike that cascades under loss). Linux NVENC / AMF
|
||||
/// set it when `PUNKTFUNK_INTRA_REFRESH` opened the encoder in that mode; VAAPI/QSV/software never
|
||||
/// do. NOTE — the wave carries NO decoder-visible clean-point: FFmpeg never sets `AV_FRAME_FLAG_KEY`
|
||||
/// at a recovery point (H.264 flags key only when `recovery_frame_cnt == 0`; HEVC only on IRAP),
|
||||
/// and AMF emits no recovery-point SEI at all. So this cap ALONE does not let the client lift its
|
||||
/// post-loss freeze without an IDR — that needs [`intra_refresh_recovery`](Self::intra_refresh_recovery).
|
||||
pub intra_refresh: bool,
|
||||
/// The intra-refresh wave is a *validated constrained GDR* — verified on real hardware to fully
|
||||
/// heal a lost picture within one wave period with no residual artifacts. Only then does the host
|
||||
/// tag each wave-boundary AU with [`USER_FLAG_RECOVERY_POINT`](punktfunk_core::packet::USER_FLAG_RECOVERY_POINT),
|
||||
/// so the client can lift its freeze on the second mark (a proven clean re-anchor) instead of
|
||||
/// waiting out its backstop and forcing a full IDR. Default `false` on every backend until on-glass
|
||||
/// validation flips it — an un-validated encoder keeps the IDR recovery path, so this is inert and
|
||||
/// cannot regress. Meaningless unless [`intra_refresh`](Self::intra_refresh) is also set.
|
||||
pub intra_refresh_recovery: bool,
|
||||
/// Length of the intra-refresh wave in frames — the boundary period the host marks on (it sets
|
||||
/// `USER_FLAG_RECOVERY_POINT` on every Nth emitted AU, re-phased at each IDR). 0 when intra-refresh
|
||||
/// is off. Only consulted when [`intra_refresh_recovery`](Self::intra_refresh_recovery) is set.
|
||||
pub intra_refresh_period: u32,
|
||||
}
|
||||
|
||||
/// A hardware encoder. One per session; runs on the encode thread.
|
||||
pub trait Encoder: Send {
|
||||
fn submit(&mut self, frame: &CapturedFrame) -> Result<()>;
|
||||
/// [`submit`](Self::submit) with the **wire frame index** this frame's AU will carry — the
|
||||
/// number the packetizer stamps on it and the client's loss reports/RFI requests name. The
|
||||
/// session glue predicts it exactly as `AUs sent so far + frames in flight` (AUs are emitted
|
||||
/// FIFO, one per submission; anything that would break the prediction — an in-place reset, a
|
||||
/// device-change teardown, an encoder rebuild — forfeits the in-flight frames on BOTH sides
|
||||
/// and clears the encoder's reference state, so stale predictions die with it). The RFI
|
||||
/// backends pin their frame numbering (LTR marks, DPB timestamps) to this so
|
||||
/// [`invalidate_ref_frames`](Self::invalidate_ref_frames) compares client frame numbers
|
||||
/// against the same domain — an encoder-internal counter desyncs from the wire on the first
|
||||
/// mid-stream rebuild (adaptive bitrate steps do this under congestion, exactly when losses
|
||||
/// happen). Default: ignore the index and delegate to `submit` (backends without per-frame
|
||||
/// reference bookkeeping don't care).
|
||||
fn submit_indexed(&mut self, frame: &CapturedFrame, wire_index: u32) -> Result<()> {
|
||||
let _ = wire_index;
|
||||
self.submit(frame)
|
||||
}
|
||||
/// This encoder's static [capabilities](EncoderCaps) (RFI, HDR SEI), so the session glue can
|
||||
/// route by query rather than rely on the no-op/`false` defaults of
|
||||
/// [`invalidate_ref_frames`](Self::invalidate_ref_frames) / [`set_hdr_meta`](Self::set_hdr_meta).
|
||||
/// Default: no optional capabilities (the SDR / libavcodec backends) — only the direct-NVENC
|
||||
/// path overrides it.
|
||||
fn caps(&self) -> EncoderCaps {
|
||||
EncoderCaps::default()
|
||||
}
|
||||
/// Force the next submitted frame to be an IDR keyframe (e.g. after a client
|
||||
/// reference-frame-invalidation request). Default: no-op.
|
||||
fn request_keyframe(&mut self) {}
|
||||
/// Set the source's static HDR mastering metadata (from the capturer). An HDR encoder emits it
|
||||
/// as in-band SEI (`mastering_display_colour_volume` + `content_light_level_info`) on each
|
||||
/// keyframe so any decoder — including stock Moonlight — tone-maps from the source's real grade.
|
||||
/// Default: no-op (SDR encoders / libavcodec paths that don't attach it yet). Cheap to call
|
||||
/// every frame; only the direct-NVENC path consumes it.
|
||||
fn set_hdr_meta(&mut self, _meta: Option<punktfunk_core::quic::HdrMeta>) {}
|
||||
/// Invalidate a contiguous range of previously-encoded reference frames (client frame numbers
|
||||
/// — WIRE frame indexes, the domain [`submit_indexed`](Self::submit_indexed) pins the encoder's
|
||||
/// bookkeeping to) so the encoder re-references an older still-valid frame instead of emitting
|
||||
/// a full IDR. Returns `true` if a real reference invalidation was performed; `false` means the
|
||||
/// encoder couldn't (range older than the DPB/LTR history, or the backend has no RFI) and the
|
||||
/// caller should fall back to [`request_keyframe`](Self::request_keyframe). Default: `false` —
|
||||
/// the Windows direct-NVENC path (`nvEncInvalidateRefFrames`) and native AMF (LTR
|
||||
/// force-reference) implement true RFI; the libavcodec paths can't express it, so they keyframe.
|
||||
fn invalidate_ref_frames(&mut self, _first_frame: i64, _last_frame: i64) -> bool {
|
||||
false
|
||||
}
|
||||
/// Pull the next encoded AU if one is ready.
|
||||
fn poll(&mut self) -> Result<Option<EncodedFrame>>;
|
||||
/// Tear the underlying hardware encoder down and rebuild it in place, keeping the session's
|
||||
/// negotiated parameters — the encode-stall watchdog's recovery lever (a wedged AMF/QSV
|
||||
/// driver stops emitting AUs or accepting frames without ever returning an error). Returns
|
||||
/// `true` when the encoder was rebuilt: every submitted-but-unpolled frame is forfeited and
|
||||
/// the next submitted frame starts a fresh stream (IDR). Default `false`: the backend has no
|
||||
/// in-place rebuild and the caller must treat the stall as fatal instead.
|
||||
fn reset(&mut self) -> bool {
|
||||
false
|
||||
}
|
||||
/// Retarget the encoder's rate control to `bps` (average == max, CBR) **in place** — same
|
||||
/// codec/resolution/fps, only the bitrate and its derived VBV move. Returns `true` when the
|
||||
/// live encoder accepted the change: the reference chain, the in-flight frames and the
|
||||
/// caller's wire-index prediction all survive, so an adaptive-bitrate step costs *nothing* on
|
||||
/// the wire (no IDR, no in-flight forfeit — the whole point vs. a rebuild). `false` = the
|
||||
/// backend can't (or the driver rejected the new rate, e.g. above the codec-level ceiling) —
|
||||
/// the caller falls back to its full rebuild path, which also owns the bitrate clamping.
|
||||
/// Default: no in-place retarget (the libavcodec/software paths).
|
||||
fn reconfigure_bitrate(&mut self, _bps: u64) -> bool {
|
||||
false
|
||||
}
|
||||
/// Wire-chunk the encoder's AUs at the session's shard payload size (the PyroWave
|
||||
/// datagram-aligned mode, plan §4.4): every `shard_payload` window of the emitted AU
|
||||
/// starts a fresh self-delimiting codec packet, zero-padded to the window — so a lost
|
||||
/// datagram costs a few coefficient blocks, not the frame. AUs produced this way are
|
||||
/// flagged [`EncodedFrame::chunk_aligned`] and the session marks them on the wire.
|
||||
/// Default: no-op (the H.26x backends' bitstreams cannot be cut losslessly).
|
||||
fn set_wire_chunking(&mut self, _shard_payload: usize) {}
|
||||
/// Signal end-of-stream. After this, drain the remaining AUs with [`poll`](Self::poll)
|
||||
/// until it returns `None` — NVENC buffers frames internally even at `delay=0`.
|
||||
fn flush(&mut self) -> Result<()>;
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
/// Maximum encodable dimension (px) per side for this codec on NVENC. H.264 tops out at
|
||||
/// 4096 (level constraint); HEVC and AV1 allow 8192. Used to reject out-of-range client
|
||||
/// modes up front (see [`validate_dimensions`]).
|
||||
pub fn max_dimension(self) -> u32 {
|
||||
match self {
|
||||
Codec::H264 => 4096,
|
||||
// PyroWave has no codec-level dimension cap (arbitrary even sizes); 8192 matches the
|
||||
// buffer-math guard the other codecs get.
|
||||
Codec::H265 | Codec::Av1 | Codec::PyroWave => 8192,
|
||||
}
|
||||
}
|
||||
|
||||
/// The codec's *spec* top level/tier bitrate (bits/s) — the usual boundary at which NVENC
|
||||
/// starts rejecting `avcodec_open2` with EINVAL. NOT a hard cap: [`open_video`](crate::
|
||||
/// open_video) probes the actual GPU ceiling by stepping DOWN from the requested bitrate only on
|
||||
/// EINVAL, and uses this purely as the first step-down candidate (so a card that accepts more —
|
||||
/// an RTX 5070 Ti does >1 Gbps HEVC where a 4090 caps at ~800 Mbps — is never clamped to it).
|
||||
/// HEVC Level 6.2 High tier = 800 Mbps; H.264 High level 6.2 ≈ 480 Mbps; AV1's levels allow more.
|
||||
pub fn max_bitrate_bps(self) -> u64 {
|
||||
match self {
|
||||
Codec::H264 => 480_000_000,
|
||||
Codec::H265 => 800_000_000,
|
||||
Codec::Av1 => 1_200_000_000,
|
||||
// No spec level/tier: the rate is a plain per-frame byte budget. Use the protocol's
|
||||
// own bitrate clamp so the step-down probe logic never binds below it.
|
||||
Codec::PyroWave => 8_000_000_000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `PUNKTFUNK_VBV_FRAMES` — HRD/VBV size in frame intervals (default 1.0, the strict low-latency
|
||||
/// shape every backend ships: each frame must fit its rate share, keeping frame sizes uniform for
|
||||
/// the pacer). The AMF/VAAPI/QSV paths parse the same variable locally; this helper brings the
|
||||
/// direct-NVENC paths (which used to hardwire 1 frame) to parity. Larger values let complex
|
||||
/// frames borrow bits — better rate utilization at the cost of per-frame size variance.
|
||||
pub(crate) fn vbv_frames_env() -> f64 {
|
||||
std::env::var("PUNKTFUNK_VBV_FRAMES")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<f64>().ok())
|
||||
.filter(|v| v.is_finite() && *v > 0.0)
|
||||
.unwrap_or(1.0)
|
||||
}
|
||||
|
||||
/// Validate a requested encode resolution before we allocate buffers or open NVENC. Rejects
|
||||
/// zero/odd-sized and out-of-range modes with a clear error instead of letting buffer math
|
||||
/// overflow or the encoder open fail with an opaque NVENC code. A client can request any
|
||||
/// `mode=WxHxFPS`, so this is the gate on attacker/typo-controlled dimensions.
|
||||
pub fn validate_dimensions(codec: Codec, width: u32, height: u32) -> Result<()> {
|
||||
if width == 0 || height == 0 {
|
||||
anyhow::bail!("invalid encode resolution {width}x{height}: dimensions must be non-zero");
|
||||
}
|
||||
// NVENC requires even dimensions for the chroma subsampling it does internally.
|
||||
if width % 2 != 0 || height % 2 != 0 {
|
||||
anyhow::bail!("invalid encode resolution {width}x{height}: dimensions must be even");
|
||||
}
|
||||
// PyroWave's 5-level wavelet decomposition needs ≥ 4·2⁵ px per axis (upstream
|
||||
// `MinimumImageSize` — the band mirroring breaks below it); reject a tiny mode here
|
||||
// (e.g. a match-window resize dragged to a sliver) instead of failing the encoder
|
||||
// rebuild after the switch was acked.
|
||||
if codec == Codec::PyroWave && (width < 128 || height < 128) {
|
||||
anyhow::bail!(
|
||||
"invalid PyroWave resolution {width}x{height}: the wavelet needs at least 128px per axis"
|
||||
);
|
||||
}
|
||||
let max = codec.max_dimension();
|
||||
if width > max || height > max {
|
||||
anyhow::bail!(
|
||||
"{codec:?} max dimension is {max}px; requested {width}x{height} \
|
||||
(use HEVC/AV1 above 4096, or lower the client resolution)"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rejects_zero_and_odd_dimensions() {
|
||||
assert!(validate_dimensions(Codec::H265, 0, 1080).is_err());
|
||||
assert!(validate_dimensions(Codec::H265, 1920, 0).is_err());
|
||||
assert!(validate_dimensions(Codec::H265, 1921, 1080).is_err()); // odd width
|
||||
assert!(validate_dimensions(Codec::H265, 1920, 1081).is_err()); // odd height
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn h264_capped_at_4096() {
|
||||
assert!(validate_dimensions(Codec::H264, 3840, 2160).is_ok()); // 4K fits (width < 4096)
|
||||
assert!(validate_dimensions(Codec::H264, 4096, 4096).is_ok()); // exactly at the limit
|
||||
assert!(validate_dimensions(Codec::H264, 4098, 2160).is_err());
|
||||
assert!(validate_dimensions(Codec::H264, 3840, 4098).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hevc_and_av1_allow_up_to_8192() {
|
||||
for c in [Codec::H265, Codec::Av1] {
|
||||
assert!(validate_dimensions(c, 3840, 2160).is_ok());
|
||||
assert!(validate_dimensions(c, 7680, 4320).is_ok()); // 8K fits
|
||||
assert!(validate_dimensions(c, 8192, 8192).is_ok());
|
||||
assert!(validate_dimensions(c, 8194, 4320).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn common_modes_accepted() {
|
||||
for c in [Codec::H264, Codec::H265, Codec::Av1] {
|
||||
for (w, h) in [(1280, 720), (1920, 1080), (2560, 1440)] {
|
||||
assert!(validate_dimensions(c, w, h).is_ok(), "{c:?} {w}x{h}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wire round-trip and the stats label stay in lockstep with the `quic::CODEC_*` bits.
|
||||
#[test]
|
||||
fn codec_wire_roundtrip_and_label() {
|
||||
for c in [Codec::H264, Codec::H265, Codec::Av1] {
|
||||
assert_eq!(Codec::from_wire(c.to_wire()), c);
|
||||
}
|
||||
assert_eq!(Codec::H264.label(), "h264");
|
||||
assert_eq!(Codec::H265.label(), "hevc");
|
||||
assert_eq!(Codec::Av1.label(), "av1");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
//! Shared libavcodec (FFmpeg) glue for the three libav encode backends — Linux NVENC
|
||||
//! (`encode/linux/mod.rs`), VAAPI (`encode/linux/vaapi.rs`), and Windows AMF/QSV
|
||||
//! (`encode/windows/ffmpeg_win.rs`) — so the byte-identical pieces live once (plan §2.2, the Tier-2
|
||||
//! gap). Free functions and consts over borrowed handles; nothing here is per-frame `dyn`,
|
||||
//! allocating, or on the zero-copy ingest path.
|
||||
use crate::EncodedFrame;
|
||||
use anyhow::{Context, Result};
|
||||
use ffmpeg_next as ffmpeg;
|
||||
use ffmpeg_next::ffi; // = ffmpeg_sys_next
|
||||
use ffmpeg_next::format::Pixel;
|
||||
use ffmpeg_next::{encoder, Packet, Rational};
|
||||
use std::os::raw::c_int;
|
||||
|
||||
/// swscale: nearest-neighbour scaler flag (`SWS_POINT`). We never rescale (src dims == dst dims), so
|
||||
/// the resampler choice only governs the colour-conversion path; POINT is the cheapest.
|
||||
pub(crate) const SWS_POINT: c_int = 0x10;
|
||||
/// swscale colorspace id for ITU-R BT.709 (`SWS_CS_ITU709`) — the CSC coefficients for our RGB→YUV.
|
||||
pub(crate) const SWS_CS_ITU709: c_int = 1;
|
||||
/// swscale colorspace id for ITU-R BT.2020 non-constant-luminance (`SWS_CS_BT2020`) — the CSC
|
||||
/// coefficients for the HDR X2BGR10→P010 path (Windows only today).
|
||||
pub(crate) const SWS_CS_BT2020: c_int = 9;
|
||||
|
||||
/// `Pixel` → `AVPixelFormat`. `Pixel` is `#[repr(i32)]`-compatible with `AVPixelFormat` (the bindgen
|
||||
/// enum) via this documented conversion in ffmpeg-next.
|
||||
pub(crate) fn pixel_to_av(p: Pixel) -> ffi::AVPixelFormat {
|
||||
ffi::AVPixelFormat::from(p)
|
||||
}
|
||||
|
||||
/// One `receive_packet` attempt, with the not-ready states kept distinct so a blocking drain can
|
||||
/// tell "still encoding" (retry) from "stream over" (stop). The Linux NVENC/VAAPI polls collapse
|
||||
/// `Again`/`Eof` to `None`; the Windows AMF/QSV path keeps them apart for its deadline-driven loop.
|
||||
pub(crate) enum PollOutcome {
|
||||
Packet(EncodedFrame),
|
||||
Again,
|
||||
Eof,
|
||||
}
|
||||
|
||||
/// Apply the shared low-latency rate-control contract to a **not-yet-opened** encoder context: a
|
||||
/// fixed frame rate, CBR (target == max bitrate), B-frames off, and a tight ~1-frame VBV/HRD buffer.
|
||||
///
|
||||
/// The VBV size bounds any single frame. Under CBR with no buffer set, libav's encoders use a loose
|
||||
/// default VBV, so a high-motion P-frame can balloon to many times the average; those extra packets
|
||||
/// overflow the bounded send queue + kernel socket buffer and get dropped, which the client sees as
|
||||
/// framedrops/jitter (and, on the infinite-GOP path, as old/stale frames flashing until the next
|
||||
/// RFI). A tight ~1-frame buffer makes the encoder hold frame size roughly constant and absorb motion
|
||||
/// as a momentary QP (quality) dip instead — the trade we want. Default = 1 frame of bits
|
||||
/// (bitrate/fps); `PUNKTFUNK_VBV_FRAMES` tunes it (larger = better motion quality, bigger bursts).
|
||||
///
|
||||
/// The caller still owns `set_format` (pixel format) and `gop_size` (GOP policy differs: NVENC's
|
||||
/// infinite/intra-refresh wave vs the VAAPI/AMF `i32::MAX`), since those are backend-specific.
|
||||
pub(crate) fn apply_low_latency_rc(video: &mut encoder::video::Video, fps: u32, bitrate_bps: u64) {
|
||||
video.set_time_base(Rational(1, fps as i32));
|
||||
video.set_frame_rate(Some(Rational(fps as i32, 1)));
|
||||
video.set_bit_rate(bitrate_bps as usize);
|
||||
video.set_max_bit_rate(bitrate_bps as usize);
|
||||
video.set_max_b_frames(0);
|
||||
let vbv_bits = ((bitrate_bps as f64 / fps.max(1) as f64) * crate::vbv_frames_env())
|
||||
.clamp(1.0, i32::MAX as f64);
|
||||
// SAFETY: `video` wraps a freshly-allocated `AVCodecContext` we hold by value and have not opened
|
||||
// yet; `as_mut_ptr()` returns that non-null, aligned, exclusively-owned context. Writing the plain
|
||||
// `rc_buffer_size` int before `open_with` is the supported way to set a field ffmpeg-next exposes
|
||||
// no setter for. Sole owner → no aliasing; synchronous in-bounds scalar write.
|
||||
unsafe {
|
||||
(*video.as_mut_ptr()).rc_buffer_size = vbv_bits as i32;
|
||||
}
|
||||
}
|
||||
|
||||
/// Drain the encoder for one packet (shared across the NVENC/VAAPI/AMF/QSV libav backends). The
|
||||
/// `EncodedFrame`'s only allocation is the `to_vec()` of the bitstream — the same copy each backend
|
||||
/// already made — so this stays off any per-frame `dyn`/`Box`/channel path.
|
||||
pub(crate) fn poll_encoder(enc: &mut encoder::video::Encoder, fps: u32) -> Result<PollOutcome> {
|
||||
let mut pkt = Packet::empty();
|
||||
match enc.receive_packet(&mut pkt) {
|
||||
Ok(()) => {
|
||||
let data = pkt.data().map(|d| d.to_vec()).unwrap_or_default();
|
||||
let pts = pkt.pts().unwrap_or(0).max(0) as u64;
|
||||
Ok(PollOutcome::Packet(EncodedFrame {
|
||||
data,
|
||||
pts_ns: pts * 1_000_000_000 / fps as u64,
|
||||
keyframe: pkt.is_key(),
|
||||
recovery_anchor: false,
|
||||
chunk_aligned: false,
|
||||
}))
|
||||
}
|
||||
// No packet ready yet (need another input frame).
|
||||
Err(ffmpeg::Error::Other { errno })
|
||||
if errno == ffmpeg::util::error::EAGAIN
|
||||
|| errno == ffmpeg::util::error::EWOULDBLOCK =>
|
||||
{
|
||||
Ok(PollOutcome::Again)
|
||||
}
|
||||
// Fully drained after flush().
|
||||
Err(ffmpeg::Error::Eof) => Ok(PollOutcome::Eof),
|
||||
Err(e) => Err(e).context("receive_packet"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
// Cursor-overlay blend kernels for the CUDA/NVENC path (cursor-as-metadata). The cursor bitmap is
|
||||
// straight-alpha RGBA, row-packed (stride = curW*4). Blended into the encoder-OWNED NVENC input
|
||||
// surface — never the compositor's dmabuf. One thread per cursor pixel (ARGB / YUV444) or per 2x2
|
||||
// chroma block (NV12). Coefficients are BT.709 limited, matching rgb2yuv.comp so the cursor colour
|
||||
// matches the rest of the frame regardless of which zero-copy backend encodes it.
|
||||
//
|
||||
// Build (regenerate cursor_blend.ptx after editing):
|
||||
// nvcc -ptx -arch=compute_75 cursor_blend.cu -o cursor_blend.ptx
|
||||
// PTX is JIT'd by the driver forward to the actual GPU, so a compute_75 (Turing) baseline runs on
|
||||
// every Turing-or-newer NVENC GPU. (CUDA 13's nvcc no longer targets pre-Turing archs.)
|
||||
|
||||
typedef unsigned char u8;
|
||||
|
||||
__device__ __forceinline__ u8 blend8(int dst, int src, int a) {
|
||||
return (u8)((src * a + dst * (255 - a)) / 255);
|
||||
}
|
||||
|
||||
// Packed 4-byte surface. NVENC's ARGB format stores bytes B,G,R,A in memory; the cursor is R,G,B,A.
|
||||
extern "C" __global__ void blend_argb(
|
||||
u8* surf, int pitch, int surfW, int surfH,
|
||||
const u8* cur, int curW, int curH, int ox, int oy)
|
||||
{
|
||||
int cx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int cy = blockIdx.y * blockDim.y + threadIdx.y;
|
||||
if (cx >= curW || cy >= curH) return;
|
||||
int px = ox + cx, py = oy + cy;
|
||||
if (px < 0 || py < 0 || px >= surfW || py >= surfH) return;
|
||||
const u8* s = cur + (size_t)(cy * curW + cx) * 4;
|
||||
int a = s[3];
|
||||
if (a == 0) return;
|
||||
u8* d = surf + (size_t)py * pitch + (size_t)px * 4;
|
||||
d[0] = blend8(d[0], s[2], a); // B <- cursor B
|
||||
d[1] = blend8(d[1], s[1], a); // G <- cursor G
|
||||
d[2] = blend8(d[2], s[0], a); // R <- cursor R
|
||||
}
|
||||
|
||||
// Planar YUV444: three full-res planes stacked at base, base+plane, base+2*plane (plane=pitch*surfH).
|
||||
extern "C" __global__ void blend_yuv444(
|
||||
u8* base, int pitch, int surfW, int surfH,
|
||||
const u8* cur, int curW, int curH, int ox, int oy)
|
||||
{
|
||||
int cx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int cy = blockIdx.y * blockDim.y + threadIdx.y;
|
||||
if (cx >= curW || cy >= curH) return;
|
||||
int px = ox + cx, py = oy + cy;
|
||||
if (px < 0 || py < 0 || px >= surfW || py >= surfH) return;
|
||||
const u8* s = cur + (size_t)(cy * curW + cx) * 4;
|
||||
int a = s[3];
|
||||
if (a == 0) return;
|
||||
float R = s[0], G = s[1], B = s[2];
|
||||
int Y = (int)(16.0f + 0.1826f * R + 0.6142f * G + 0.0620f * B + 0.5f);
|
||||
int U = (int)(128.0f - 0.1006f * R - 0.3386f * G + 0.4392f * B + 0.5f);
|
||||
int V = (int)(128.0f + 0.4392f * R - 0.3989f * G - 0.0403f * B + 0.5f);
|
||||
size_t plane = (size_t)pitch * surfH;
|
||||
u8* yp = base + (size_t)py * pitch + px;
|
||||
u8* up = base + plane + (size_t)py * pitch + px;
|
||||
u8* vp = base + 2 * plane + (size_t)py * pitch + px;
|
||||
*yp = blend8(*yp, Y, a);
|
||||
*up = blend8(*up, U, a);
|
||||
*vp = blend8(*vp, V, a);
|
||||
}
|
||||
|
||||
// NV12: full-res Y plane + interleaved half-res UV plane. One thread per 2x2 luma block; each blends
|
||||
// up to four Y samples and one (alpha-weighted) UV sample.
|
||||
extern "C" __global__ void blend_nv12(
|
||||
u8* yb, int yPitch, u8* uvb, int uvPitch, int surfW, int surfH,
|
||||
const u8* cur, int curW, int curH, int ox, int oy)
|
||||
{
|
||||
int bx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int by = blockIdx.y * blockDim.y + threadIdx.y;
|
||||
int base_cx = bx * 2, base_cy = by * 2;
|
||||
if (base_cx >= curW || base_cy >= curH) return;
|
||||
float ua = 0.0f, va = 0.0f, wa = 0.0f;
|
||||
int cnt = 0;
|
||||
for (int j = 0; j < 2; j++) {
|
||||
for (int i = 0; i < 2; i++) {
|
||||
int cx = base_cx + i, cy = base_cy + j;
|
||||
if (cx >= curW || cy >= curH) continue;
|
||||
int px = ox + cx, py = oy + cy;
|
||||
if (px < 0 || py < 0 || px >= surfW || py >= surfH) continue;
|
||||
const u8* s = cur + (size_t)(cy * curW + cx) * 4;
|
||||
int a = s[3];
|
||||
if (a == 0) continue;
|
||||
float R = s[0], G = s[1], B = s[2];
|
||||
int Y = (int)(16.0f + 0.1826f * R + 0.6142f * G + 0.0620f * B + 0.5f);
|
||||
u8* yp = yb + (size_t)py * yPitch + px;
|
||||
*yp = blend8(*yp, Y, a);
|
||||
ua += (128.0f - 0.1006f * R - 0.3386f * G + 0.4392f * B) * a;
|
||||
va += (128.0f + 0.4392f * R - 0.3989f * G - 0.0403f * B) * a;
|
||||
wa += a;
|
||||
cnt++;
|
||||
}
|
||||
}
|
||||
if (wa <= 0.0f || cnt == 0) return;
|
||||
// The chroma sample covering this block's top-left surface pixel.
|
||||
int uvx = (ox + base_cx) / 2;
|
||||
int uvy = (oy + base_cy) / 2;
|
||||
if (uvx < 0 || uvy < 0 || uvx * 2 >= surfW || uvy * 2 >= surfH) return;
|
||||
int U = (int)(ua / wa + 0.5f);
|
||||
int V = (int)(va / wa + 0.5f);
|
||||
int amean = (int)(wa / cnt + 0.5f);
|
||||
u8* uv = uvb + (size_t)uvy * uvPitch + (size_t)uvx * 2;
|
||||
uv[0] = blend8(uv[0], U, amean);
|
||||
uv[1] = blend8(uv[1], V, amean);
|
||||
}
|
||||
@@ -0,0 +1,576 @@
|
||||
//
|
||||
// Generated by NVIDIA NVVM Compiler
|
||||
//
|
||||
// Compiler Build ID: CL-38244171
|
||||
// Cuda compilation tools, release 13.3, V13.3.73
|
||||
// Based on NVVM 7.0.1
|
||||
//
|
||||
|
||||
.version 9.3
|
||||
.target sm_75
|
||||
.address_size 64
|
||||
|
||||
// .globl blend_argb
|
||||
|
||||
.visible .entry blend_argb(
|
||||
.param .u64 blend_argb_param_0,
|
||||
.param .u32 blend_argb_param_1,
|
||||
.param .u32 blend_argb_param_2,
|
||||
.param .u32 blend_argb_param_3,
|
||||
.param .u64 blend_argb_param_4,
|
||||
.param .u32 blend_argb_param_5,
|
||||
.param .u32 blend_argb_param_6,
|
||||
.param .u32 blend_argb_param_7,
|
||||
.param .u32 blend_argb_param_8
|
||||
)
|
||||
{
|
||||
.reg .pred %p<10>;
|
||||
.reg .b16 %rs<2>;
|
||||
.reg .b32 %r<34>;
|
||||
.reg .b64 %rd<17>;
|
||||
|
||||
|
||||
ld.param.u64 %rd2, [blend_argb_param_0];
|
||||
ld.param.u32 %r5, [blend_argb_param_1];
|
||||
ld.param.u32 %r6, [blend_argb_param_2];
|
||||
ld.param.u32 %r7, [blend_argb_param_3];
|
||||
ld.param.u64 %rd3, [blend_argb_param_4];
|
||||
ld.param.u32 %r8, [blend_argb_param_5];
|
||||
ld.param.u32 %r11, [blend_argb_param_6];
|
||||
ld.param.u32 %r9, [blend_argb_param_7];
|
||||
ld.param.u32 %r10, [blend_argb_param_8];
|
||||
mov.u32 %r12, %ntid.x;
|
||||
mov.u32 %r13, %ctaid.x;
|
||||
mov.u32 %r14, %tid.x;
|
||||
mad.lo.s32 %r1, %r13, %r12, %r14;
|
||||
mov.u32 %r15, %ntid.y;
|
||||
mov.u32 %r16, %ctaid.y;
|
||||
mov.u32 %r17, %tid.y;
|
||||
mad.lo.s32 %r2, %r16, %r15, %r17;
|
||||
setp.ge.s32 %p1, %r1, %r8;
|
||||
setp.ge.s32 %p2, %r2, %r11;
|
||||
or.pred %p3, %p1, %p2;
|
||||
@%p3 bra $L__BB0_4;
|
||||
|
||||
add.s32 %r3, %r1, %r9;
|
||||
add.s32 %r4, %r2, %r10;
|
||||
or.b32 %r18, %r4, %r3;
|
||||
setp.lt.s32 %p4, %r18, 0;
|
||||
setp.ge.s32 %p5, %r3, %r6;
|
||||
or.pred %p6, %p5, %p4;
|
||||
setp.ge.s32 %p7, %r4, %r7;
|
||||
or.pred %p8, %p7, %p6;
|
||||
@%p8 bra $L__BB0_4;
|
||||
|
||||
mad.lo.s32 %r19, %r2, %r8, %r1;
|
||||
mul.wide.s32 %rd4, %r19, 4;
|
||||
cvta.to.global.u64 %rd5, %rd3;
|
||||
add.s64 %rd1, %rd5, %rd4;
|
||||
ld.global.u8 %rs1, [%rd1+3];
|
||||
setp.eq.s16 %p9, %rs1, 0;
|
||||
@%p9 bra $L__BB0_4;
|
||||
|
||||
cvt.u32.u16 %r20, %rs1;
|
||||
mul.wide.s32 %rd6, %r4, %r5;
|
||||
mul.wide.s32 %rd7, %r3, 4;
|
||||
add.s64 %rd8, %rd6, %rd7;
|
||||
cvta.to.global.u64 %rd9, %rd2;
|
||||
add.s64 %rd10, %rd9, %rd8;
|
||||
ld.global.u8 %r21, [%rd10];
|
||||
ld.global.u8 %r22, [%rd1+2];
|
||||
xor.b32 %r23, %r20, 255;
|
||||
mul.lo.s32 %r24, %r23, %r21;
|
||||
mad.lo.s32 %r25, %r22, %r20, %r24;
|
||||
mul.wide.u32 %rd11, %r25, -2139062143;
|
||||
shr.u64 %rd12, %rd11, 39;
|
||||
st.global.u8 [%rd10], %rd12;
|
||||
ld.global.u8 %r26, [%rd10+1];
|
||||
ld.global.u8 %r27, [%rd1+1];
|
||||
mul.lo.s32 %r28, %r23, %r26;
|
||||
mad.lo.s32 %r29, %r27, %r20, %r28;
|
||||
mul.wide.u32 %rd13, %r29, -2139062143;
|
||||
shr.u64 %rd14, %rd13, 39;
|
||||
st.global.u8 [%rd10+1], %rd14;
|
||||
ld.global.u8 %r30, [%rd10+2];
|
||||
ld.global.u8 %r31, [%rd1];
|
||||
mul.lo.s32 %r32, %r23, %r30;
|
||||
mad.lo.s32 %r33, %r31, %r20, %r32;
|
||||
mul.wide.u32 %rd15, %r33, -2139062143;
|
||||
shr.u64 %rd16, %rd15, 39;
|
||||
st.global.u8 [%rd10+2], %rd16;
|
||||
|
||||
$L__BB0_4:
|
||||
ret;
|
||||
|
||||
}
|
||||
// .globl blend_yuv444
|
||||
.visible .entry blend_yuv444(
|
||||
.param .u64 blend_yuv444_param_0,
|
||||
.param .u32 blend_yuv444_param_1,
|
||||
.param .u32 blend_yuv444_param_2,
|
||||
.param .u32 blend_yuv444_param_3,
|
||||
.param .u64 blend_yuv444_param_4,
|
||||
.param .u32 blend_yuv444_param_5,
|
||||
.param .u32 blend_yuv444_param_6,
|
||||
.param .u32 blend_yuv444_param_7,
|
||||
.param .u32 blend_yuv444_param_8
|
||||
)
|
||||
{
|
||||
.reg .pred %p<10>;
|
||||
.reg .b16 %rs<5>;
|
||||
.reg .f32 %f<16>;
|
||||
.reg .b32 %r<49>;
|
||||
.reg .b64 %rd<14>;
|
||||
|
||||
|
||||
ld.param.u64 %rd2, [blend_yuv444_param_0];
|
||||
ld.param.u32 %r5, [blend_yuv444_param_1];
|
||||
ld.param.u32 %r6, [blend_yuv444_param_2];
|
||||
ld.param.u32 %r7, [blend_yuv444_param_3];
|
||||
ld.param.u64 %rd3, [blend_yuv444_param_4];
|
||||
ld.param.u32 %r8, [blend_yuv444_param_5];
|
||||
ld.param.u32 %r11, [blend_yuv444_param_6];
|
||||
ld.param.u32 %r9, [blend_yuv444_param_7];
|
||||
ld.param.u32 %r10, [blend_yuv444_param_8];
|
||||
mov.u32 %r12, %ntid.x;
|
||||
mov.u32 %r13, %ctaid.x;
|
||||
mov.u32 %r14, %tid.x;
|
||||
mad.lo.s32 %r1, %r13, %r12, %r14;
|
||||
mov.u32 %r15, %ntid.y;
|
||||
mov.u32 %r16, %ctaid.y;
|
||||
mov.u32 %r17, %tid.y;
|
||||
mad.lo.s32 %r2, %r16, %r15, %r17;
|
||||
setp.ge.s32 %p1, %r1, %r8;
|
||||
setp.ge.s32 %p2, %r2, %r11;
|
||||
or.pred %p3, %p1, %p2;
|
||||
@%p3 bra $L__BB1_4;
|
||||
|
||||
add.s32 %r3, %r1, %r9;
|
||||
add.s32 %r4, %r2, %r10;
|
||||
or.b32 %r18, %r4, %r3;
|
||||
setp.lt.s32 %p4, %r18, 0;
|
||||
setp.ge.s32 %p5, %r3, %r6;
|
||||
or.pred %p6, %p5, %p4;
|
||||
setp.ge.s32 %p7, %r4, %r7;
|
||||
or.pred %p8, %p7, %p6;
|
||||
@%p8 bra $L__BB1_4;
|
||||
|
||||
mad.lo.s32 %r19, %r2, %r8, %r1;
|
||||
mul.wide.s32 %rd4, %r19, 4;
|
||||
cvta.to.global.u64 %rd5, %rd3;
|
||||
add.s64 %rd1, %rd5, %rd4;
|
||||
ld.global.u8 %rs1, [%rd1+3];
|
||||
setp.eq.s16 %p9, %rs1, 0;
|
||||
@%p9 bra $L__BB1_4;
|
||||
|
||||
cvt.u32.u16 %r20, %rs1;
|
||||
ld.global.u8 %rs2, [%rd1];
|
||||
cvt.rn.f32.u16 %f1, %rs2;
|
||||
ld.global.u8 %rs3, [%rd1+1];
|
||||
cvt.rn.f32.u16 %f2, %rs3;
|
||||
ld.global.u8 %rs4, [%rd1+2];
|
||||
cvt.rn.f32.u16 %f3, %rs4;
|
||||
fma.rn.f32 %f4, %f1, 0f3E3AFB7F, 0f41800000;
|
||||
fma.rn.f32 %f5, %f2, 0f3F1D3C36, %f4;
|
||||
fma.rn.f32 %f6, %f3, 0f3D7DF3B6, %f5;
|
||||
add.f32 %f7, %f6, 0f3F000000;
|
||||
cvt.rzi.s32.f32 %r21, %f7;
|
||||
fma.rn.f32 %f8, %f1, 0fBDCE075F, 0f43000000;
|
||||
fma.rn.f32 %f9, %f2, 0fBEAD5CFB, %f8;
|
||||
fma.rn.f32 %f10, %f3, 0f3EE0DED3, %f9;
|
||||
add.f32 %f11, %f10, 0f3F000000;
|
||||
cvt.rzi.s32.f32 %r22, %f11;
|
||||
fma.rn.f32 %f12, %f1, 0f3EE0DED3, 0f43000000;
|
||||
fma.rn.f32 %f13, %f2, 0fBECC3C9F, %f12;
|
||||
fma.rn.f32 %f14, %f3, 0fBD25119D, %f13;
|
||||
add.f32 %f15, %f14, 0f3F000000;
|
||||
cvt.rzi.s32.f32 %r23, %f15;
|
||||
mul.wide.s32 %rd6, %r4, %r5;
|
||||
cvt.s64.s32 %rd7, %r3;
|
||||
add.s64 %rd8, %rd6, %rd7;
|
||||
cvta.to.global.u64 %rd9, %rd2;
|
||||
add.s64 %rd10, %rd9, %rd8;
|
||||
ld.global.u8 %r24, [%rd10];
|
||||
mul.lo.s32 %r25, %r21, %r20;
|
||||
xor.b32 %r26, %r20, 255;
|
||||
mad.lo.s32 %r27, %r26, %r24, %r25;
|
||||
mul.hi.s32 %r28, %r27, -2139062143;
|
||||
add.s32 %r29, %r28, %r27;
|
||||
shr.u32 %r30, %r29, 31;
|
||||
shr.u32 %r31, %r29, 7;
|
||||
add.s32 %r32, %r31, %r30;
|
||||
st.global.u8 [%rd10], %r32;
|
||||
mul.wide.s32 %rd11, %r7, %r5;
|
||||
add.s64 %rd12, %rd10, %rd11;
|
||||
ld.global.u8 %r33, [%rd12];
|
||||
mul.lo.s32 %r34, %r22, %r20;
|
||||
mad.lo.s32 %r35, %r26, %r33, %r34;
|
||||
mul.hi.s32 %r36, %r35, -2139062143;
|
||||
add.s32 %r37, %r36, %r35;
|
||||
shr.u32 %r38, %r37, 31;
|
||||
shr.u32 %r39, %r37, 7;
|
||||
add.s32 %r40, %r39, %r38;
|
||||
st.global.u8 [%rd12], %r40;
|
||||
add.s64 %rd13, %rd12, %rd11;
|
||||
ld.global.u8 %r41, [%rd13];
|
||||
mul.lo.s32 %r42, %r23, %r20;
|
||||
mad.lo.s32 %r43, %r26, %r41, %r42;
|
||||
mul.hi.s32 %r44, %r43, -2139062143;
|
||||
add.s32 %r45, %r44, %r43;
|
||||
shr.u32 %r46, %r45, 31;
|
||||
shr.u32 %r47, %r45, 7;
|
||||
add.s32 %r48, %r47, %r46;
|
||||
st.global.u8 [%rd13], %r48;
|
||||
|
||||
$L__BB1_4:
|
||||
ret;
|
||||
|
||||
}
|
||||
// .globl blend_nv12
|
||||
.visible .entry blend_nv12(
|
||||
.param .u64 blend_nv12_param_0,
|
||||
.param .u32 blend_nv12_param_1,
|
||||
.param .u64 blend_nv12_param_2,
|
||||
.param .u32 blend_nv12_param_3,
|
||||
.param .u32 blend_nv12_param_4,
|
||||
.param .u32 blend_nv12_param_5,
|
||||
.param .u64 blend_nv12_param_6,
|
||||
.param .u32 blend_nv12_param_7,
|
||||
.param .u32 blend_nv12_param_8,
|
||||
.param .u32 blend_nv12_param_9,
|
||||
.param .u32 blend_nv12_param_10
|
||||
)
|
||||
{
|
||||
.reg .pred %p<43>;
|
||||
.reg .b16 %rs<17>;
|
||||
.reg .f32 %f<108>;
|
||||
.reg .b32 %r<123>;
|
||||
.reg .b64 %rd<35>;
|
||||
|
||||
|
||||
ld.param.u64 %rd11, [blend_nv12_param_0];
|
||||
ld.param.u32 %r21, [blend_nv12_param_1];
|
||||
ld.param.u64 %rd10, [blend_nv12_param_2];
|
||||
ld.param.u32 %r22, [blend_nv12_param_3];
|
||||
ld.param.u32 %r23, [blend_nv12_param_4];
|
||||
ld.param.u32 %r24, [blend_nv12_param_5];
|
||||
ld.param.u64 %rd12, [blend_nv12_param_6];
|
||||
ld.param.u32 %r25, [blend_nv12_param_7];
|
||||
ld.param.u32 %r26, [blend_nv12_param_8];
|
||||
ld.param.u32 %r27, [blend_nv12_param_9];
|
||||
ld.param.u32 %r28, [blend_nv12_param_10];
|
||||
cvta.to.global.u64 %rd1, %rd11;
|
||||
cvta.to.global.u64 %rd2, %rd12;
|
||||
mov.u32 %r29, %ntid.x;
|
||||
mov.u32 %r30, %ctaid.x;
|
||||
mov.u32 %r31, %tid.x;
|
||||
mad.lo.s32 %r32, %r30, %r29, %r31;
|
||||
mov.u32 %r33, %ntid.y;
|
||||
mov.u32 %r34, %ctaid.y;
|
||||
mov.u32 %r35, %tid.y;
|
||||
mad.lo.s32 %r36, %r34, %r33, %r35;
|
||||
shl.b32 %r1, %r32, 1;
|
||||
shl.b32 %r2, %r36, 1;
|
||||
setp.ge.s32 %p1, %r1, %r25;
|
||||
setp.ge.s32 %p2, %r2, %r26;
|
||||
or.pred %p3, %p1, %p2;
|
||||
mov.f32 %f102, 0f00000000;
|
||||
mov.f32 %f103, 0f00000000;
|
||||
mov.f32 %f104, 0f00000000;
|
||||
@%p3 bra $L__BB2_19;
|
||||
|
||||
cvt.s64.s32 %rd3, %r21;
|
||||
add.s32 %r3, %r2, %r28;
|
||||
setp.ge.s32 %p4, %r3, %r24;
|
||||
mul.lo.s32 %r4, %r2, %r25;
|
||||
mul.wide.s32 %rd4, %r3, %r21;
|
||||
add.s32 %r5, %r1, %r27;
|
||||
or.b32 %r38, %r5, %r3;
|
||||
setp.lt.s32 %p5, %r38, 0;
|
||||
mov.u32 %r121, 0;
|
||||
setp.ge.s32 %p6, %r5, %r23;
|
||||
or.pred %p7, %p6, %p5;
|
||||
or.pred %p8, %p4, %p7;
|
||||
@%p8 bra $L__BB2_4;
|
||||
|
||||
add.s32 %r40, %r1, %r4;
|
||||
mul.wide.s32 %rd13, %r40, 4;
|
||||
add.s64 %rd5, %rd2, %rd13;
|
||||
ld.global.u8 %rs1, [%rd5+3];
|
||||
setp.eq.s16 %p9, %rs1, 0;
|
||||
@%p9 bra $L__BB2_4;
|
||||
|
||||
cvt.u32.u16 %r42, %rs1;
|
||||
ld.global.u8 %rs5, [%rd5];
|
||||
cvt.rn.f32.u16 %f31, %rs5;
|
||||
ld.global.u8 %rs6, [%rd5+1];
|
||||
cvt.rn.f32.u16 %f32, %rs6;
|
||||
ld.global.u8 %rs7, [%rd5+2];
|
||||
cvt.rn.f32.u16 %f33, %rs7;
|
||||
fma.rn.f32 %f34, %f31, 0f3E3AFB7F, 0f41800000;
|
||||
fma.rn.f32 %f35, %f32, 0f3F1D3C36, %f34;
|
||||
fma.rn.f32 %f36, %f33, 0f3D7DF3B6, %f35;
|
||||
add.f32 %f37, %f36, 0f3F000000;
|
||||
cvt.rzi.s32.f32 %r43, %f37;
|
||||
cvt.s64.s32 %rd14, %r5;
|
||||
add.s64 %rd15, %rd4, %rd14;
|
||||
add.s64 %rd16, %rd1, %rd15;
|
||||
ld.global.u8 %r44, [%rd16];
|
||||
mul.lo.s32 %r45, %r43, %r42;
|
||||
xor.b32 %r46, %r42, 255;
|
||||
mad.lo.s32 %r47, %r46, %r44, %r45;
|
||||
mul.hi.s32 %r48, %r47, -2139062143;
|
||||
add.s32 %r49, %r48, %r47;
|
||||
shr.u32 %r50, %r49, 31;
|
||||
shr.u32 %r51, %r49, 7;
|
||||
add.s32 %r52, %r51, %r50;
|
||||
st.global.u8 [%rd16], %r52;
|
||||
fma.rn.f32 %f38, %f31, 0fBDCE075F, 0f43000000;
|
||||
fma.rn.f32 %f39, %f32, 0fBEAD5CFB, %f38;
|
||||
fma.rn.f32 %f40, %f33, 0f3EE0DED3, %f39;
|
||||
cvt.rn.f32.u16 %f104, %rs1;
|
||||
fma.rn.f32 %f102, %f40, %f104, 0f00000000;
|
||||
fma.rn.f32 %f41, %f31, 0f3EE0DED3, 0f43000000;
|
||||
fma.rn.f32 %f42, %f32, 0fBECC3C9F, %f41;
|
||||
fma.rn.f32 %f43, %f33, 0fBD25119D, %f42;
|
||||
fma.rn.f32 %f103, %f43, %f104, 0f00000000;
|
||||
mov.u32 %r121, 1;
|
||||
|
||||
$L__BB2_4:
|
||||
add.s32 %r7, %r1, 1;
|
||||
setp.ge.s32 %p10, %r7, %r25;
|
||||
@%p10 bra $L__BB2_8;
|
||||
|
||||
add.s32 %r8, %r7, %r27;
|
||||
or.b32 %r53, %r8, %r3;
|
||||
setp.lt.s32 %p12, %r53, 0;
|
||||
setp.ge.s32 %p13, %r8, %r23;
|
||||
or.pred %p14, %p13, %p12;
|
||||
or.pred %p15, %p4, %p14;
|
||||
@%p15 bra $L__BB2_8;
|
||||
|
||||
add.s32 %r54, %r7, %r4;
|
||||
mul.wide.s32 %rd17, %r54, 4;
|
||||
add.s64 %rd6, %rd2, %rd17;
|
||||
ld.global.u8 %rs2, [%rd6+3];
|
||||
setp.eq.s16 %p16, %rs2, 0;
|
||||
@%p16 bra $L__BB2_8;
|
||||
|
||||
cvt.u32.u16 %r55, %rs2;
|
||||
ld.global.u8 %rs8, [%rd6];
|
||||
cvt.rn.f32.u16 %f44, %rs8;
|
||||
ld.global.u8 %rs9, [%rd6+1];
|
||||
cvt.rn.f32.u16 %f45, %rs9;
|
||||
ld.global.u8 %rs10, [%rd6+2];
|
||||
cvt.rn.f32.u16 %f46, %rs10;
|
||||
fma.rn.f32 %f47, %f44, 0f3E3AFB7F, 0f41800000;
|
||||
fma.rn.f32 %f48, %f45, 0f3F1D3C36, %f47;
|
||||
fma.rn.f32 %f49, %f46, 0f3D7DF3B6, %f48;
|
||||
add.f32 %f50, %f49, 0f3F000000;
|
||||
cvt.rzi.s32.f32 %r56, %f50;
|
||||
cvt.s64.s32 %rd18, %r8;
|
||||
add.s64 %rd19, %rd4, %rd18;
|
||||
add.s64 %rd20, %rd1, %rd19;
|
||||
ld.global.u8 %r57, [%rd20];
|
||||
mul.lo.s32 %r58, %r56, %r55;
|
||||
xor.b32 %r59, %r55, 255;
|
||||
mad.lo.s32 %r60, %r59, %r57, %r58;
|
||||
mul.hi.s32 %r61, %r60, -2139062143;
|
||||
add.s32 %r62, %r61, %r60;
|
||||
shr.u32 %r63, %r62, 31;
|
||||
shr.u32 %r64, %r62, 7;
|
||||
add.s32 %r65, %r64, %r63;
|
||||
st.global.u8 [%rd20], %r65;
|
||||
fma.rn.f32 %f51, %f44, 0fBDCE075F, 0f43000000;
|
||||
fma.rn.f32 %f52, %f45, 0fBEAD5CFB, %f51;
|
||||
fma.rn.f32 %f53, %f46, 0f3EE0DED3, %f52;
|
||||
cvt.rn.f32.u16 %f54, %rs2;
|
||||
fma.rn.f32 %f102, %f53, %f54, %f102;
|
||||
fma.rn.f32 %f55, %f44, 0f3EE0DED3, 0f43000000;
|
||||
fma.rn.f32 %f56, %f45, 0fBECC3C9F, %f55;
|
||||
fma.rn.f32 %f57, %f46, 0fBD25119D, %f56;
|
||||
fma.rn.f32 %f103, %f57, %f54, %f103;
|
||||
add.f32 %f104, %f104, %f54;
|
||||
add.s32 %r121, %r121, 1;
|
||||
|
||||
$L__BB2_8:
|
||||
add.s32 %r11, %r2, 1;
|
||||
setp.ge.s32 %p17, %r11, %r26;
|
||||
add.s32 %r12, %r11, %r28;
|
||||
add.s32 %r13, %r4, %r25;
|
||||
cvt.s64.s32 %rd21, %r12;
|
||||
mul.lo.s64 %rd7, %rd21, %rd3;
|
||||
@%p17 bra $L__BB2_12;
|
||||
|
||||
setp.ge.s32 %p18, %r12, %r24;
|
||||
or.b32 %r66, %r5, %r12;
|
||||
setp.lt.s32 %p19, %r66, 0;
|
||||
or.pred %p21, %p6, %p19;
|
||||
or.pred %p22, %p18, %p21;
|
||||
@%p22 bra $L__BB2_12;
|
||||
|
||||
add.s32 %r67, %r1, %r13;
|
||||
mul.wide.s32 %rd22, %r67, 4;
|
||||
add.s64 %rd8, %rd2, %rd22;
|
||||
ld.global.u8 %rs3, [%rd8+3];
|
||||
setp.eq.s16 %p23, %rs3, 0;
|
||||
@%p23 bra $L__BB2_12;
|
||||
|
||||
cvt.u32.u16 %r68, %rs3;
|
||||
ld.global.u8 %rs11, [%rd8];
|
||||
cvt.rn.f32.u16 %f58, %rs11;
|
||||
ld.global.u8 %rs12, [%rd8+1];
|
||||
cvt.rn.f32.u16 %f59, %rs12;
|
||||
ld.global.u8 %rs13, [%rd8+2];
|
||||
cvt.rn.f32.u16 %f60, %rs13;
|
||||
fma.rn.f32 %f61, %f58, 0f3E3AFB7F, 0f41800000;
|
||||
fma.rn.f32 %f62, %f59, 0f3F1D3C36, %f61;
|
||||
fma.rn.f32 %f63, %f60, 0f3D7DF3B6, %f62;
|
||||
add.f32 %f64, %f63, 0f3F000000;
|
||||
cvt.rzi.s32.f32 %r69, %f64;
|
||||
cvt.s64.s32 %rd23, %r5;
|
||||
add.s64 %rd24, %rd7, %rd23;
|
||||
add.s64 %rd25, %rd1, %rd24;
|
||||
ld.global.u8 %r70, [%rd25];
|
||||
mul.lo.s32 %r71, %r69, %r68;
|
||||
xor.b32 %r72, %r68, 255;
|
||||
mad.lo.s32 %r73, %r72, %r70, %r71;
|
||||
mul.hi.s32 %r74, %r73, -2139062143;
|
||||
add.s32 %r75, %r74, %r73;
|
||||
shr.u32 %r76, %r75, 31;
|
||||
shr.u32 %r77, %r75, 7;
|
||||
add.s32 %r78, %r77, %r76;
|
||||
st.global.u8 [%rd25], %r78;
|
||||
fma.rn.f32 %f65, %f58, 0fBDCE075F, 0f43000000;
|
||||
fma.rn.f32 %f66, %f59, 0fBEAD5CFB, %f65;
|
||||
fma.rn.f32 %f67, %f60, 0f3EE0DED3, %f66;
|
||||
cvt.rn.f32.u16 %f68, %rs3;
|
||||
fma.rn.f32 %f102, %f67, %f68, %f102;
|
||||
fma.rn.f32 %f69, %f58, 0f3EE0DED3, 0f43000000;
|
||||
fma.rn.f32 %f70, %f59, 0fBECC3C9F, %f69;
|
||||
fma.rn.f32 %f71, %f60, 0fBD25119D, %f70;
|
||||
fma.rn.f32 %f103, %f71, %f68, %f103;
|
||||
add.f32 %f104, %f104, %f68;
|
||||
add.s32 %r121, %r121, 1;
|
||||
|
||||
$L__BB2_12:
|
||||
or.pred %p26, %p17, %p10;
|
||||
@%p26 bra $L__BB2_16;
|
||||
|
||||
setp.ge.s32 %p27, %r12, %r24;
|
||||
add.s32 %r16, %r7, %r27;
|
||||
or.b32 %r79, %r16, %r12;
|
||||
setp.lt.s32 %p28, %r79, 0;
|
||||
setp.ge.s32 %p29, %r16, %r23;
|
||||
or.pred %p30, %p29, %p28;
|
||||
or.pred %p31, %p27, %p30;
|
||||
@%p31 bra $L__BB2_16;
|
||||
|
||||
add.s32 %r80, %r7, %r13;
|
||||
mul.wide.s32 %rd26, %r80, 4;
|
||||
add.s64 %rd9, %rd2, %rd26;
|
||||
ld.global.u8 %rs4, [%rd9+3];
|
||||
setp.eq.s16 %p32, %rs4, 0;
|
||||
@%p32 bra $L__BB2_16;
|
||||
|
||||
cvt.u32.u16 %r81, %rs4;
|
||||
ld.global.u8 %rs14, [%rd9];
|
||||
cvt.rn.f32.u16 %f72, %rs14;
|
||||
ld.global.u8 %rs15, [%rd9+1];
|
||||
cvt.rn.f32.u16 %f73, %rs15;
|
||||
ld.global.u8 %rs16, [%rd9+2];
|
||||
cvt.rn.f32.u16 %f74, %rs16;
|
||||
fma.rn.f32 %f75, %f72, 0f3E3AFB7F, 0f41800000;
|
||||
fma.rn.f32 %f76, %f73, 0f3F1D3C36, %f75;
|
||||
fma.rn.f32 %f77, %f74, 0f3D7DF3B6, %f76;
|
||||
add.f32 %f78, %f77, 0f3F000000;
|
||||
cvt.rzi.s32.f32 %r82, %f78;
|
||||
cvt.s64.s32 %rd27, %r16;
|
||||
add.s64 %rd28, %rd7, %rd27;
|
||||
add.s64 %rd29, %rd1, %rd28;
|
||||
ld.global.u8 %r83, [%rd29];
|
||||
mul.lo.s32 %r84, %r82, %r81;
|
||||
xor.b32 %r85, %r81, 255;
|
||||
mad.lo.s32 %r86, %r85, %r83, %r84;
|
||||
mul.hi.s32 %r87, %r86, -2139062143;
|
||||
add.s32 %r88, %r87, %r86;
|
||||
shr.u32 %r89, %r88, 31;
|
||||
shr.u32 %r90, %r88, 7;
|
||||
add.s32 %r91, %r90, %r89;
|
||||
st.global.u8 [%rd29], %r91;
|
||||
fma.rn.f32 %f79, %f72, 0fBDCE075F, 0f43000000;
|
||||
fma.rn.f32 %f80, %f73, 0fBEAD5CFB, %f79;
|
||||
fma.rn.f32 %f81, %f74, 0f3EE0DED3, %f80;
|
||||
cvt.rn.f32.u16 %f82, %rs4;
|
||||
fma.rn.f32 %f102, %f81, %f82, %f102;
|
||||
fma.rn.f32 %f83, %f72, 0f3EE0DED3, 0f43000000;
|
||||
fma.rn.f32 %f84, %f73, 0fBECC3C9F, %f83;
|
||||
fma.rn.f32 %f85, %f74, 0fBD25119D, %f84;
|
||||
fma.rn.f32 %f103, %f85, %f82, %f103;
|
||||
add.f32 %f104, %f104, %f82;
|
||||
add.s32 %r121, %r121, 1;
|
||||
|
||||
$L__BB2_16:
|
||||
setp.eq.s32 %p33, %r121, 0;
|
||||
setp.le.f32 %p34, %f104, 0f00000000;
|
||||
or.pred %p35, %p34, %p33;
|
||||
@%p35 bra $L__BB2_19;
|
||||
|
||||
shr.u32 %r92, %r5, 31;
|
||||
add.s32 %r93, %r5, %r92;
|
||||
shr.s32 %r19, %r93, 1;
|
||||
setp.lt.s32 %p36, %r3, -1;
|
||||
setp.lt.s32 %p37, %r5, -1;
|
||||
or.pred %p38, %p37, %p36;
|
||||
and.b32 %r94, %r93, -2;
|
||||
setp.ge.s32 %p39, %r94, %r23;
|
||||
or.pred %p40, %p38, %p39;
|
||||
shr.u32 %r95, %r3, 31;
|
||||
add.s32 %r96, %r3, %r95;
|
||||
shr.s32 %r20, %r96, 1;
|
||||
and.b32 %r97, %r96, -2;
|
||||
setp.ge.s32 %p41, %r97, %r24;
|
||||
or.pred %p42, %p40, %p41;
|
||||
@%p42 bra $L__BB2_19;
|
||||
|
||||
div.rn.f32 %f86, %f102, %f104;
|
||||
add.f32 %f87, %f86, 0f3F000000;
|
||||
cvt.rzi.s32.f32 %r98, %f87;
|
||||
div.rn.f32 %f88, %f103, %f104;
|
||||
add.f32 %f89, %f88, 0f3F000000;
|
||||
cvt.rzi.s32.f32 %r99, %f89;
|
||||
cvt.rn.f32.s32 %f90, %r121;
|
||||
div.rn.f32 %f91, %f104, %f90;
|
||||
add.f32 %f92, %f91, 0f3F000000;
|
||||
cvt.rzi.s32.f32 %r100, %f92;
|
||||
mul.wide.s32 %rd30, %r20, %r22;
|
||||
mul.wide.s32 %rd31, %r19, 2;
|
||||
add.s64 %rd32, %rd30, %rd31;
|
||||
cvta.to.global.u64 %rd33, %rd10;
|
||||
add.s64 %rd34, %rd33, %rd32;
|
||||
ld.global.u8 %r101, [%rd34];
|
||||
mul.lo.s32 %r102, %r100, %r98;
|
||||
mov.u32 %r103, 255;
|
||||
sub.s32 %r104, %r103, %r100;
|
||||
mad.lo.s32 %r105, %r104, %r101, %r102;
|
||||
mul.hi.s32 %r106, %r105, -2139062143;
|
||||
add.s32 %r107, %r106, %r105;
|
||||
shr.u32 %r108, %r107, 31;
|
||||
shr.u32 %r109, %r107, 7;
|
||||
add.s32 %r110, %r109, %r108;
|
||||
st.global.u8 [%rd34], %r110;
|
||||
ld.global.u8 %r111, [%rd34+1];
|
||||
mul.lo.s32 %r112, %r100, %r99;
|
||||
mad.lo.s32 %r113, %r104, %r111, %r112;
|
||||
mul.hi.s32 %r114, %r113, -2139062143;
|
||||
add.s32 %r115, %r114, %r113;
|
||||
shr.u32 %r116, %r115, 31;
|
||||
shr.u32 %r117, %r115, 7;
|
||||
add.s32 %r118, %r117, %r116;
|
||||
st.global.u8 [%rd34+1], %r118;
|
||||
|
||||
$L__BB2_19:
|
||||
ret;
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,857 @@
|
||||
//! NVENC encoder via `ffmpeg-next` (binds the system FFmpeg — `ffmpeg-sys-next` auto-detects the
|
||||
//! installed version, so this builds against FFmpeg 7.x/libavcodec 61 *or* 8.x/libavcodec 62;
|
||||
//! validated live on Ubuntu 26.04 (FFmpeg 8) and Bazzite F43 (FFmpeg 7.1)).
|
||||
//!
|
||||
//! Input is a packed RGB/BGR CPU frame; `*_nvenc` accepts `rgb0`/`bgr0`/`rgba`/`bgra`
|
||||
//! directly and does the RGB→YUV conversion on the GPU, so the host stays off the
|
||||
//! colour-conversion path. The portal commonly negotiates packed 24-bit `RGB`, which NVENC
|
||||
//! does *not* accept — we expand it to `rgb0` (one padding byte/pixel, no colour math).
|
||||
//! The encoder is opened *without* a global header so VPS/SPS/PPS are emitted in-band on
|
||||
//! every IDR — the output is both a playable raw Annex-B stream and self-contained AUs.
|
||||
// Every `unsafe` block in this file carries a `// SAFETY:` proof; enforce it (unsafe-proof program).
|
||||
#![deny(clippy::undocumented_unsafe_blocks)]
|
||||
|
||||
use super::{ChromaFormat, Codec, EncodedFrame, Encoder};
|
||||
use anyhow::{anyhow, bail, Context, Result};
|
||||
use ffmpeg::format::Pixel;
|
||||
use ffmpeg::util::frame::Video as VideoFrame;
|
||||
use ffmpeg::{codec, encoder, Dictionary};
|
||||
use ffmpeg_next as ffmpeg;
|
||||
use pf_frame::{CapturedFrame, FramePayload, PixelFormat};
|
||||
use std::os::raw::c_int;
|
||||
use std::ptr;
|
||||
|
||||
use super::libav::{
|
||||
apply_low_latency_rc, pixel_to_av, poll_encoder, PollOutcome, SWS_CS_ITU709, SWS_POINT,
|
||||
};
|
||||
use ffmpeg::ffi; // = ffmpeg_sys_next
|
||||
|
||||
/// The swscale *source* pixel format for a captured packed RGB/BGR layout (the real byte order, not
|
||||
/// the NVENC-padded `*0` form). Used by the 4:4:4 RGB→YUV444P conversion path. Mirrors the VAAPI
|
||||
/// CPU-input mapping; YUV/10-bit inputs can't feed this path (the 4:4:4 session forces packed RGB).
|
||||
fn sws_src_pixel(format: PixelFormat) -> Result<Pixel> {
|
||||
Ok(match format {
|
||||
PixelFormat::Bgrx => Pixel::BGRZ, // bgr0
|
||||
PixelFormat::Rgbx => Pixel::RGBZ, // rgb0
|
||||
PixelFormat::Bgra => Pixel::BGRA,
|
||||
PixelFormat::Rgba => Pixel::RGBA,
|
||||
PixelFormat::Rgb => Pixel::RGB24,
|
||||
PixelFormat::Bgr => Pixel::BGR24,
|
||||
PixelFormat::Nv12 | PixelFormat::P010 | PixelFormat::Rgb10a2 | PixelFormat::Yuv444 => {
|
||||
bail!("NVENC 4:4:4 CPU-input path supports packed RGB/BGR only; got {format:?}")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// `AVCUDADeviceContext` (libavutil/hwcontext_cuda.h) — not in the ffmpeg-sys bindings (the
|
||||
/// crate doesn't allowlist that header), so mirror its stable 3-pointer layout. We set the
|
||||
/// first field to *our* `CUcontext` so NVENC shares the context the EGL importer maps into.
|
||||
#[repr(C)]
|
||||
struct AVCUDADeviceContext {
|
||||
cuda_ctx: *mut std::ffi::c_void, // CUcontext
|
||||
stream: *mut std::ffi::c_void, // CUstream (null = default)
|
||||
internal: *mut std::ffi::c_void, // filled by ctx_init
|
||||
}
|
||||
|
||||
/// CUDA hardware-frame contexts that wrap our shared `CUcontext`, so `hevc_nvenc` reads the
|
||||
/// imported device buffer directly. Owns two `AVBufferRef`s, unref'd on drop.
|
||||
struct CudaHw {
|
||||
device_ref: *mut ffi::AVBufferRef,
|
||||
frames_ref: *mut ffi::AVBufferRef,
|
||||
}
|
||||
|
||||
impl CudaHw {
|
||||
/// Build a CUDA hwdevice wrapping `cu_ctx` and a frames pool (`sw_format` = `pixel`).
|
||||
unsafe fn new(cu_ctx: *mut std::ffi::c_void, sw_format: Pixel, w: u32, h: u32) -> Result<Self> {
|
||||
let mut device_ref = ffi::av_hwdevice_ctx_alloc(ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA);
|
||||
if device_ref.is_null() {
|
||||
bail!("av_hwdevice_ctx_alloc(CUDA) failed");
|
||||
}
|
||||
let dev_ctx = (*device_ref).data as *mut ffi::AVHWDeviceContext;
|
||||
let cu = (*dev_ctx).hwctx as *mut AVCUDADeviceContext;
|
||||
(*cu).cuda_ctx = cu_ctx; // share the importer's context
|
||||
let r = ffi::av_hwdevice_ctx_init(device_ref);
|
||||
if r < 0 {
|
||||
ffi::av_buffer_unref(&mut device_ref);
|
||||
bail!("av_hwdevice_ctx_init failed ({r})");
|
||||
}
|
||||
|
||||
let mut frames_ref = ffi::av_hwframe_ctx_alloc(device_ref);
|
||||
if frames_ref.is_null() {
|
||||
ffi::av_buffer_unref(&mut device_ref);
|
||||
bail!("av_hwframe_ctx_alloc failed");
|
||||
}
|
||||
let fc = (*frames_ref).data as *mut ffi::AVHWFramesContext;
|
||||
(*fc).format = ffi::AVPixelFormat::AV_PIX_FMT_CUDA;
|
||||
(*fc).sw_format = pixel_to_av(sw_format);
|
||||
(*fc).width = w as c_int;
|
||||
(*fc).height = h as c_int;
|
||||
(*fc).initial_pool_size = 0; // we supply the device pointers
|
||||
let r = ffi::av_hwframe_ctx_init(frames_ref);
|
||||
if r < 0 {
|
||||
ffi::av_buffer_unref(&mut frames_ref);
|
||||
ffi::av_buffer_unref(&mut device_ref);
|
||||
bail!("av_hwframe_ctx_init failed ({r})");
|
||||
}
|
||||
Ok(CudaHw {
|
||||
device_ref,
|
||||
frames_ref,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CudaHw {
|
||||
fn drop(&mut self) {
|
||||
// SAFETY: `frames_ref`/`device_ref` are the two non-null `AVBufferRef`s `CudaHw::new` created
|
||||
// (it bails before returning `Self` if either alloc fails, so a live `CudaHw` always holds
|
||||
// both). `av_buffer_unref` drops one reference and nulls the pointer through the `&mut`. This
|
||||
// `Drop` runs exactly once and `CudaHw` owns these refs exclusively → no double-free /
|
||||
// use-after-free. Frames are unref'd before the device (the frames ctx internally refs the
|
||||
// device; refcounted, so the order is sound regardless).
|
||||
unsafe {
|
||||
ffi::av_buffer_unref(&mut self.frames_ref);
|
||||
ffi::av_buffer_unref(&mut self.device_ref);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a captured layout to the NVENC input pixel format, and whether a 3→4 byte expand is
|
||||
/// needed (packed RGB/BGR have no padding byte; the NVENC `*0` formats do).
|
||||
fn nvenc_input(format: PixelFormat) -> (Pixel, bool) {
|
||||
match format {
|
||||
PixelFormat::Bgrx => (Pixel::BGRZ, false), // bgr0
|
||||
PixelFormat::Rgbx => (Pixel::RGBZ, false), // rgb0
|
||||
PixelFormat::Bgra => (Pixel::BGRA, false),
|
||||
PixelFormat::Rgba => (Pixel::RGBA, false),
|
||||
PixelFormat::Rgb => (Pixel::RGBZ, true), // RGB -> rgb0
|
||||
PixelFormat::Bgr => (Pixel::BGRZ, true), // BGR -> bgr0
|
||||
// NV12 is native YUV: NVENC encodes it with NO internal RGB→YUV CSC (the Tier 2A win). On
|
||||
// Linux it's produced by the GPU convert on the zero-copy tiled path (`PUNKTFUNK_NV12`); on
|
||||
// Windows by the D3D11 video processor.
|
||||
PixelFormat::Nv12 => (Pixel::NV12, false),
|
||||
// Planar YUV444 from the zero-copy worker's GPU convert (a 4:4:4 session) — native
|
||||
// full-chroma YUV in, `hevc_nvenc` emits Range-Extensions 4:4:4.
|
||||
PixelFormat::Yuv444 => (Pixel::YUV444P, false),
|
||||
// Rgb10a2 (HDR) and P010 (the Windows 10-bit video-processor output) are produced only by
|
||||
// the Windows paths; the Linux capturer never emits them. Map to BGRA so the match is
|
||||
// exhaustive — unreachable here.
|
||||
PixelFormat::Rgb10a2 | PixelFormat::P010 => (Pixel::BGRA, false),
|
||||
}
|
||||
}
|
||||
|
||||
/// The [`NvencEncoder::open`] arguments, kept on the encoder so [`Encoder::reset`] can rebuild it
|
||||
/// in place with the session's negotiated parameters — the encode-stall watchdog's recovery lever
|
||||
/// (drop the wedged libavcodec encoder, reopen fresh, forfeit the owed AUs, restart at an IDR).
|
||||
#[derive(Clone, Copy)]
|
||||
struct OpenArgs {
|
||||
codec: Codec,
|
||||
format: PixelFormat,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
bitrate_bps: u64,
|
||||
cuda: bool,
|
||||
bit_depth: u8,
|
||||
chroma: ChromaFormat,
|
||||
}
|
||||
|
||||
pub struct NvencEncoder {
|
||||
enc: encoder::video::Encoder,
|
||||
/// Reusable 4-bpp CPU input frame (CPU path only; `None` for the zero-copy/CUDA path).
|
||||
/// Mutating it in place across frames is sound only because the encoder is opened with
|
||||
/// `delay=0`/`bf=0`/`max_b_frames=0` and the caller drains `poll()` after each `submit`,
|
||||
/// so libavcodec holds no reference to the previous frame's buffer when we overwrite it.
|
||||
frame: Option<VideoFrame>,
|
||||
/// Zero-copy path: CUDA hwdevice/hwframes contexts (the encoder takes `AV_PIX_FMT_CUDA`).
|
||||
cuda: Option<CudaHw>,
|
||||
/// 4:4:4 CPU path only: swscale context converting the captured packed RGB/BGR → planar
|
||||
/// YUV444P into [`Self::frame`], because `hevc_nvenc` only emits 4:4:4 from a YUV444 *input*
|
||||
/// (RGB-in is always 4:2:0). `None` on the 4:2:0 paths AND on the zero-copy 4:4:4 path (the
|
||||
/// worker's GPU convert delivers YUV444 CUDA frames). Freed in `Drop`.
|
||||
sws_444: Option<*mut ffi::SwsContext>,
|
||||
/// This session opened as full-chroma 4:4:4 (FREXT) — via either input path.
|
||||
want_444: bool,
|
||||
src_format: PixelFormat,
|
||||
expand: bool,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
/// Monotonic presentation index, in `1/fps` time-base units.
|
||||
frame_idx: i64,
|
||||
/// Force the next submitted frame to be an IDR (set by [`request_keyframe`]).
|
||||
force_kf: bool,
|
||||
/// Opened in intra-refresh mode (surfaced via [`caps`](Encoder::caps) so the session glue
|
||||
/// rate-limits forced IDRs — the wave heals loss without them).
|
||||
intra_refresh: bool,
|
||||
/// Resolved wave length in frames when [`intra_refresh`](Self::intra_refresh), else 0. Cached at
|
||||
/// open so the pump's per-AU `caps()` doesn't re-read `PUNKTFUNK_IR_PERIOD_FRAMES`; the pump marks
|
||||
/// every Nth AU with `USER_FLAG_RECOVERY_POINT` for the client's clean re-anchor.
|
||||
intra_refresh_period: u32,
|
||||
/// The open arguments, for the in-place [`reset`](Encoder::reset) rebuild.
|
||||
args: OpenArgs,
|
||||
}
|
||||
|
||||
// `CudaHw` holds raw `AVBufferRef`s and `sws_444` a raw `SwsContext`; the encoder lives on a single
|
||||
// thread. The CPU encoder is already `Send` via ffmpeg-next; assert it for the raw fields too.
|
||||
// SAFETY: `NvencEncoder` owns an ffmpeg-next `Encoder`/`VideoFrame` (already `Send`) plus a `CudaHw`
|
||||
// holding raw `AVBufferRef`s and an optional raw `SwsContext`, none of which are `Send` by default.
|
||||
// The `SwsContext` is a self-contained swscale state object with no thread affinity, touched only
|
||||
// through `&mut self` on the one encode thread. The encoder is owned and driven by
|
||||
// exactly ONE thread — the per-session encode thread it is moved to — and is only touched through
|
||||
// `&mut self` methods, so it is never aliased or accessed concurrently. The wrapped libav contexts
|
||||
// (and the shared `CUcontext` the `CudaHw` references) have no thread affinity, so transferring
|
||||
// ownership across threads is sound. This asserts `Send` (transfer) only, extending ffmpeg-next's
|
||||
// existing `Send` to the raw CUDA fields; `Sync` (shared `&`) is deliberately NOT implemented.
|
||||
unsafe impl Send for NvencEncoder {}
|
||||
|
||||
/// Latched true once an intra-refresh open failed with the device-capability error (ENOSYS from
|
||||
/// `NV_ENC_CAPS_SUPPORT_INTRA_REFRESH`), so later sessions skip the doomed attempt. Never set by
|
||||
/// other open failures (a bitrate EINVAL must not permanently disable the feature).
|
||||
static IR_UNSUPPORTED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||
|
||||
/// Whether this open should run the NVENC **intra-refresh** loss-recovery mode
|
||||
/// (`PUNKTFUNK_INTRA_REFRESH` truthy, opt-in until on-glass validated): a moving intra band +
|
||||
/// recovery-point SEI refreshes the whole picture every [`intra_refresh_period`] frames, so
|
||||
/// FEC-unrecoverable loss heals without the 20-40× full-IDR spike (which under loss causes more
|
||||
/// loss — the cascade). The session glue then rate-limits client keyframe requests
|
||||
/// ([`EncoderCaps::intra_refresh`](super::EncoderCaps)).
|
||||
fn intra_refresh_requested() -> bool {
|
||||
std::env::var("PUNKTFUNK_INTRA_REFRESH")
|
||||
.map(|v| matches!(v.trim(), "1" | "true" | "yes" | "on"))
|
||||
.unwrap_or(false)
|
||||
&& !IR_UNSUPPORTED.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// The intra-refresh wave length in frames — ffmpeg derives `intraRefreshPeriod`/`Cnt` from
|
||||
/// `gop_size` before forcing the real GOP infinite, so this is what `gop_size` is set to in IR
|
||||
/// mode. Default = half a second of frames (heals fast, spreads the intra cost to ~2-3% per
|
||||
/// frame); `PUNKTFUNK_IR_PERIOD_FRAMES` overrides.
|
||||
fn intra_refresh_period(fps: u32) -> i32 {
|
||||
std::env::var("PUNKTFUNK_IR_PERIOD_FRAMES")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<i32>().ok())
|
||||
.filter(|v| *v >= 2)
|
||||
.unwrap_or_else(|| (fps.max(16) / 2) as i32)
|
||||
}
|
||||
|
||||
impl NvencEncoder {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn open(
|
||||
codec: Codec,
|
||||
format: PixelFormat,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
bitrate_bps: u64,
|
||||
cuda: bool,
|
||||
bit_depth: u8,
|
||||
chroma: ChromaFormat,
|
||||
) -> Result<Self> {
|
||||
// TODO(hdr): Linux 10-bit parity. Unlike the Windows raw-SDK path (which upconverts 8-bit
|
||||
// ARGB → Main10 via pixelBitDepthMinus8), libavcodec hevc_nvenc needs a 10-bit input pixel
|
||||
// format (p010) for Main10, so it's a bigger change; deferred until a Linux GPU box is
|
||||
// available to validate. The Linux host stays 8-bit for now.
|
||||
if bit_depth != 8 {
|
||||
tracing::warn!(
|
||||
bit_depth,
|
||||
"Linux NVENC 10-bit not yet wired — encoding 8-bit"
|
||||
);
|
||||
}
|
||||
// Full-chroma 4:4:4 (HEVC Range Extensions). `hevc_nvenc` only emits 4:4:4 from a YUV444
|
||||
// *input* frame — feeding RGB always subsamples to 4:2:0 regardless of profile (verified on
|
||||
// the RTX 5070 Ti). Two ways to produce that input: the zero-copy worker's GPU convert
|
||||
// (planar-YUV444 CUDA frames — `cuda` true), or the CPU path's swscale RGB→YUV444P. Both
|
||||
// feed `profile=rext`; the range follows `PUNKTFUNK_444_FULLRANGE` in both.
|
||||
let want_444 = chroma.is_444() && codec == Codec::H265;
|
||||
ffmpeg::init().context("ffmpeg init")?;
|
||||
if std::env::var_os("PUNKTFUNK_FFMPEG_DEBUG").is_some() {
|
||||
// SAFETY: `av_log_set_level` sets libav's global integer log level; `48` (= AV_LOG_DEBUG)
|
||||
// is a valid level with no pointer args, and libav was just initialized by `ffmpeg::init()`
|
||||
// above — always sound.
|
||||
unsafe { ffi::av_log_set_level(48) }; // AV_LOG_DEBUG — surface NVENC hw-frame rejects
|
||||
}
|
||||
let name = codec.nvenc_name();
|
||||
let av_codec = encoder::find_by_name(name)
|
||||
.ok_or_else(|| anyhow!("{name} not built into libavcodec"))?;
|
||||
let (rgb_pixel, rgb_expand) = nvenc_input(format);
|
||||
// 4:4:4 feeds NVENC a planar YUV444P frame we produce by swscale; the ordinary path feeds the
|
||||
// captured RGB straight in and lets NVENC's internal CSC subsample to 4:2:0.
|
||||
let (nvenc_pixel, expand) = if want_444 {
|
||||
(Pixel::YUV444P, false)
|
||||
} else {
|
||||
(rgb_pixel, rgb_expand)
|
||||
};
|
||||
|
||||
let mut video = codec::context::Context::new_with_codec(av_codec)
|
||||
.encoder()
|
||||
.video()
|
||||
.context("alloc video encoder")?;
|
||||
video.set_width(width);
|
||||
video.set_height(height);
|
||||
video.set_format(nvenc_pixel); // NVENC converts RGB→YUV internally
|
||||
// Fixed rate, CBR, no B-frames, ~1-frame VBV — the shared low-latency RC contract.
|
||||
apply_low_latency_rc(&mut video, fps, bitrate_bps);
|
||||
// Infinite GOP — NO periodic IDR. A keyframe at 5120x1440 is ~20-40x a P-frame, so a
|
||||
// periodic IDR is a recurring multi-millisecond encode+packetize+send spike — the ~2s
|
||||
// "freeze". NVENC emits one IDR at stream start, then P-frames only; `forced-idr` (below)
|
||||
// turns a client recovery request (RFI, via `request_keyframe`) into an IDR on demand.
|
||||
// This is the Moonlight/Sunshine low-latency model.
|
||||
// In intra-refresh mode the GOP is still infinite — ffmpeg reads `gop_size` as the refresh
|
||||
// WAVE length (`intraRefreshPeriod`/`Cnt`) and then forces `gopLength` infinite itself, so
|
||||
// a positive `gop_size` here does NOT reintroduce periodic IDRs.
|
||||
let intra_refresh = intra_refresh_requested();
|
||||
// SAFETY: same `video` builder as above — a non-null, properly-aligned, sole-owned, not-yet-
|
||||
// opened `AVCodecContext`. We write the plain `gop_size` int field (-1 = infinite GOP, or the
|
||||
// intra-refresh wave length) before `open_with`, which ffmpeg-next has no setter for. No
|
||||
// aliasing; synchronous scalar write.
|
||||
unsafe {
|
||||
(*video.as_mut_ptr()).gop_size = if intra_refresh {
|
||||
intra_refresh_period(fps)
|
||||
} else {
|
||||
-1
|
||||
};
|
||||
}
|
||||
|
||||
// NV12 / 4:4:4 paths: we do the RGB→YUV conversion ourselves as BT.709 (swscale), so
|
||||
// signal that in the bitstream VUI (colorspace/range/primaries/transfer) — otherwise the
|
||||
// client decoder assumes a default and the picture comes out washed-out / wrong-contrast.
|
||||
// The RGB-input 4:2:0 path leaves these unset (NVENC's internal CSC writes its own VUI).
|
||||
// Matches the Windows NV12 path's BT.709 limited-range signalling.
|
||||
//
|
||||
// PUNKTFUNK_444_FULLRANGE=1 (experimental, 4:4:4-only): convert AND signal FULL range —
|
||||
// recovers the ~12% of code space limited-range quantization gives up, for the exact
|
||||
// text/UI chroma 4:4:4 exists for. Every punktfunk client honors the signaled range
|
||||
// (csc_rows / the Apple rows port); ship as default only if the on-glass A/B shows a
|
||||
// visible win. Linux-only: the Windows path's NVENC-internal CSC range is unmeasured.
|
||||
let full_range_444 =
|
||||
want_444 && std::env::var("PUNKTFUNK_444_FULLRANGE").is_ok_and(|v| v.trim() == "1");
|
||||
if matches!(format, PixelFormat::Nv12) || want_444 {
|
||||
// SAFETY: same `video` builder — `raw = video.as_mut_ptr()` is the non-null, properly-
|
||||
// aligned, sole-owned, not-yet-opened `AVCodecContext`. We set its four VUI colour enum
|
||||
// fields to valid `AVColorSpace`/`AVColorRange`/`AVColorPrimaries`/`AVColorTransfer-
|
||||
// Characteristic` variants before `open_with`. Sole owner → no aliasing; synchronous writes.
|
||||
unsafe {
|
||||
let raw = video.as_mut_ptr();
|
||||
(*raw).colorspace = ffi::AVColorSpace::AVCOL_SPC_BT709;
|
||||
(*raw).color_range = if full_range_444 {
|
||||
ffi::AVColorRange::AVCOL_RANGE_JPEG // full
|
||||
} else {
|
||||
ffi::AVColorRange::AVCOL_RANGE_MPEG // limited/studio
|
||||
};
|
||||
(*raw).color_primaries = ffi::AVColorPrimaries::AVCOL_PRI_BT709;
|
||||
(*raw).color_trc = ffi::AVColorTransferCharacteristic::AVCOL_TRC_BT709;
|
||||
}
|
||||
}
|
||||
|
||||
// For the zero-copy path, take CUDA surfaces: wrap the shared CUcontext in CUDA
|
||||
// hwdevice/hwframes contexts and set `pix_fmt = CUDA` on the raw encoder context
|
||||
// *before* open (NVENC derives the device from `hw_frames_ctx`).
|
||||
let cuda_hw = if cuda {
|
||||
let cu_ctx = pf_zerocopy::cuda::context().context("shared CUDA context")?;
|
||||
// SAFETY: `CudaHw::new` (an `unsafe fn`) requires libav initialized (the `ffmpeg::init()`
|
||||
// above ran) and a valid `CUcontext`; `cu_ctx` is the shared importer context from
|
||||
// `zerocopy::cuda::context()?`, non-null on the `Ok` path. `nvenc_pixel` is a valid `Pixel`
|
||||
// and `width`/`height` are the validated positive dims. It returns a RAII `CudaHw` wrapping
|
||||
// (not owning) `cu_ctx` and owning two `AVBufferRef`s freed on drop.
|
||||
let hw = unsafe { CudaHw::new(cu_ctx, nvenc_pixel, width, height)? };
|
||||
// SAFETY: `raw = video.as_mut_ptr()` is the non-null, sole-owned, not-yet-opened
|
||||
// `AVCodecContext`. We set `pix_fmt = CUDA` and attach NEW refs (`av_buffer_ref`) of
|
||||
// `hw.device_ref`/`hw.frames_ref` — both non-null (`CudaHw::new` guarantees) and from the
|
||||
// live `hw`, which is moved into `NvencEncoder.cuda` next to `enc` and so outlives the
|
||||
// encoder. The context owns its own refs (freed when the context closes). No aliasing.
|
||||
unsafe {
|
||||
let raw = video.as_mut_ptr();
|
||||
(*raw).pix_fmt = ffi::AVPixelFormat::AV_PIX_FMT_CUDA;
|
||||
(*raw).hw_device_ctx = ffi::av_buffer_ref(hw.device_ref);
|
||||
(*raw).hw_frames_ctx = ffi::av_buffer_ref(hw.frames_ref);
|
||||
}
|
||||
Some(hw)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// 4:4:4 CPU path: build the RGB→YUV444P swscale (BT.709, range per the flag; no rescale).
|
||||
// Mirrors the VAAPI CPU path's RGB→NV12 scaler, but the dst is full-chroma planar 4:4:4.
|
||||
// Skipped on the zero-copy path (`cuda`): the worker's GPU convert already delivers
|
||||
// planar YUV444 CUDA frames — no CPU pixels exist to scale.
|
||||
let sws_444 = if want_444 && !cuda {
|
||||
let src_av = pixel_to_av(sws_src_pixel(format)?);
|
||||
// SAFETY: `sws_getContext` allocates a swscale context for the given src/dst dims + pixel
|
||||
// formats. Both dims are the encoder's positive `width`/`height` as `c_int`; `src_av` is a
|
||||
// valid `AVPixelFormat` (from the `sws_src_pixel`-validated, packed-RGB-only source), the
|
||||
// dst is YUV444P. The trailing filter/param pointers are null = "use defaults" (documented
|
||||
// as accepted). No Rust memory is borrowed; the returned pointer is null-checked below.
|
||||
let sws = unsafe {
|
||||
ffi::sws_getContext(
|
||||
width as c_int,
|
||||
height as c_int,
|
||||
src_av,
|
||||
width as c_int,
|
||||
height as c_int,
|
||||
ffi::AVPixelFormat::AV_PIX_FMT_YUV444P,
|
||||
SWS_POINT,
|
||||
ptr::null_mut(),
|
||||
ptr::null_mut(),
|
||||
ptr::null(),
|
||||
)
|
||||
};
|
||||
if sws.is_null() {
|
||||
bail!("sws_getContext(RGB→YUV444P) failed");
|
||||
}
|
||||
// SAFETY: `sws` is the non-null context from the call above (null-checked). The ITU-709
|
||||
// coefficient table from `sws_getCoefficients` is a process-lifetime libswscale static,
|
||||
// reused for src+dst matrices; `sws_setColorspaceDetails` only reads it and writes scalar
|
||||
// CSC settings into `sws` (dstRange matches the VUI: 0 = limited, 1 = the
|
||||
// PUNKTFUNK_444_FULLRANGE experiment). No Rust memory is passed.
|
||||
unsafe {
|
||||
let cs709 = ffi::sws_getCoefficients(SWS_CS_ITU709);
|
||||
let dst_range = i32::from(full_range_444);
|
||||
ffi::sws_setColorspaceDetails(sws, cs709, 1, cs709, dst_range, 0, 1 << 16, 1 << 16);
|
||||
}
|
||||
Some(sws)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Low-latency NVENC tuning (plan §7 / linux-setup doc).
|
||||
let mut opts = Dictionary::new();
|
||||
opts.set("preset", "p1"); // fastest
|
||||
opts.set("tune", "ull"); // ultra-low-latency
|
||||
opts.set("rc", "cbr");
|
||||
opts.set("bf", "0");
|
||||
opts.set("delay", "0");
|
||||
opts.set("forced-idr", "1"); // RFI/request_keyframe → real IDR under the infinite GOP
|
||||
if intra_refresh {
|
||||
// Moving intra band + recovery-point SEI (period set via gop_size above). Loss now
|
||||
// self-heals within the wave; forced IDRs remain available (rate-limited by the glue).
|
||||
opts.set("intra-refresh", "1");
|
||||
}
|
||||
if want_444 {
|
||||
// HEVC Range Extensions — the profile that carries chroma_format_idc=3. With a YUV444P
|
||||
// input `hevc_nvenc` auto-selects it, but pin it explicitly so the chroma is never silently
|
||||
// dropped on a future libavcodec.
|
||||
opts.set("profile", "rext");
|
||||
}
|
||||
|
||||
// Split-frame encode across both NVENC engines (GB203 has 2) when the pixel rate exceeds
|
||||
// a single engine's HEVC capacity (~1 Gpix/s); e.g. 5120x1440@240 = 1.77 Gpix/s needs it,
|
||||
// @120 = 0.88 Gpix/s does not. HEVC/AV1 only (not H.264). AUTO won't engage below ~2112px
|
||||
// height, so we force `2`; below the threshold we leave it AUTO (split costs ~2% BD-rate).
|
||||
// Output is standard HEVC — transparent to the client. Override with PUNKTFUNK_SPLIT_ENCODE.
|
||||
let pix_rate = width as u64 * height as u64 * fps as u64;
|
||||
let split = std::env::var("PUNKTFUNK_SPLIT_ENCODE").ok();
|
||||
match split.as_deref() {
|
||||
Some(mode) => opts.set("split_encode_mode", mode),
|
||||
None if matches!(codec, Codec::H265 | Codec::Av1) && pix_rate > 1_000_000_000 => {
|
||||
opts.set("split_encode_mode", "2");
|
||||
tracing::info!(
|
||||
pix_rate,
|
||||
"NVENC: forcing 2-way split encode (high pixel rate)"
|
||||
);
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
|
||||
let enc = match video.open_with(opts) {
|
||||
Ok(enc) => enc,
|
||||
// The GPU lacks NV_ENC_CAPS_SUPPORT_INTRA_REFRESH — ffmpeg fails the open with
|
||||
// ENOSYS ("Function not implemented"). Latch it (skip the doomed attempt on later
|
||||
// sessions) and reopen this session without intra-refresh; any other failure — and
|
||||
// any failure when IR wasn't requested — propagates untouched (the bitrate probe
|
||||
// keys on EINVAL, which must not trip the latch).
|
||||
Err(e) if intra_refresh && format!("{e:#}").contains("Function not implemented") => {
|
||||
tracing::warn!(
|
||||
encoder = name,
|
||||
"NVENC intra-refresh not supported by this GPU — falling back to IDR-only \
|
||||
recovery"
|
||||
);
|
||||
IR_UNSUPPORTED.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
return Self::open(
|
||||
codec,
|
||||
format,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
bitrate_bps,
|
||||
cuda,
|
||||
bit_depth,
|
||||
chroma,
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(e).with_context(|| {
|
||||
format!("open {name} ({width}x{height}@{fps}, {bitrate_bps} bps)")
|
||||
})
|
||||
}
|
||||
};
|
||||
if intra_refresh {
|
||||
tracing::info!(
|
||||
encoder = name,
|
||||
period_frames = intra_refresh_period(fps),
|
||||
"NVENC intra-refresh recovery active (no periodic IDR; wave heals loss)"
|
||||
);
|
||||
}
|
||||
|
||||
let frame = if cuda {
|
||||
None
|
||||
} else {
|
||||
Some(VideoFrame::new(nvenc_pixel, width, height))
|
||||
};
|
||||
Ok(NvencEncoder {
|
||||
enc,
|
||||
frame,
|
||||
cuda: cuda_hw,
|
||||
sws_444,
|
||||
want_444,
|
||||
src_format: format,
|
||||
expand,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
frame_idx: 0,
|
||||
force_kf: false,
|
||||
intra_refresh,
|
||||
intra_refresh_period: if intra_refresh {
|
||||
intra_refresh_period(fps).max(1) as u32
|
||||
} else {
|
||||
0
|
||||
},
|
||||
args: OpenArgs {
|
||||
codec,
|
||||
format,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
bitrate_bps,
|
||||
cuda,
|
||||
bit_depth,
|
||||
chroma,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Encoder for NvencEncoder {
|
||||
fn caps(&self) -> super::EncoderCaps {
|
||||
super::EncoderCaps {
|
||||
// 4:4:4 iff this session opened FREXT — the CPU swscale path or the zero-copy GPU
|
||||
// convert. RFI/HDR-SEI stay unsupported on libavcodec NVENC (the trait defaults).
|
||||
chroma_444: self.want_444,
|
||||
intra_refresh: self.intra_refresh,
|
||||
// NVENC intra-refresh is purpose-built GDR loss recovery (moving band + recovery-point
|
||||
// SEI): the wave heals a lost picture within one period, so mark the boundary AUs and let
|
||||
// the client re-anchor on them instead of forcing a full IDR. Tied to `intra_refresh`
|
||||
// (already the `PUNKTFUNK_INTRA_REFRESH` opt-in), unlike AMF/QSV which stay unvalidated.
|
||||
intra_refresh_recovery: self.intra_refresh,
|
||||
intra_refresh_period: self.intra_refresh_period,
|
||||
..super::EncoderCaps::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn submit(&mut self, captured: &CapturedFrame) -> Result<()> {
|
||||
anyhow::ensure!(
|
||||
captured.width == self.width && captured.height == self.height,
|
||||
"captured frame {}x{} != encoder {}x{}",
|
||||
captured.width,
|
||||
captured.height,
|
||||
self.width,
|
||||
self.height
|
||||
);
|
||||
let pts = self.frame_idx;
|
||||
self.frame_idx += 1;
|
||||
// Force an IDR when requested (client RFI); otherwise let NVENC pick (GOP/P-frame).
|
||||
let idr = self.force_kf;
|
||||
self.force_kf = false;
|
||||
match &captured.payload {
|
||||
FramePayload::Cuda(buf) => self.submit_cuda(buf, pts, idr),
|
||||
FramePayload::Cpu(bytes) => self.submit_cpu(bytes, captured.format, pts, idr),
|
||||
FramePayload::Dmabuf(_) => {
|
||||
bail!("NVENC got a VAAPI dmabuf frame — capture/encoder backend mismatch")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn request_keyframe(&mut self) {
|
||||
self.force_kf = true;
|
||||
}
|
||||
|
||||
/// Encode-stall recovery: drop the wedged libavcodec encoder and reopen it fresh with the
|
||||
/// session's negotiated parameters (the stored [`OpenArgs`]) — the drop-and-reopen lever the
|
||||
/// QSV/VAAPI paths use, so the encode-stall watchdog can heal a wedged NVENC/driver instead of
|
||||
/// ending the session. Owed AUs are forfeited; the fresh encoder opens on an IDR.
|
||||
fn reset(&mut self) -> bool {
|
||||
let a = self.args;
|
||||
match Self::open(
|
||||
a.codec,
|
||||
a.format,
|
||||
a.width,
|
||||
a.height,
|
||||
a.fps,
|
||||
a.bitrate_bps,
|
||||
a.cuda,
|
||||
a.bit_depth,
|
||||
a.chroma,
|
||||
) {
|
||||
Ok(mut fresh) => {
|
||||
fresh.force_kf = true;
|
||||
*self = fresh; // drops the wedged encoder (frees its contexts) in the same step
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(error = %format!("{e:#}"), "NVENC in-place reopen failed");
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn poll(&mut self) -> Result<Option<EncodedFrame>> {
|
||||
// Non-blocking single drain: a packet ships, EAGAIN (need another input frame) and EOF
|
||||
// (drained after flush) both mean "nothing this tick".
|
||||
match poll_encoder(&mut self.enc, self.fps)? {
|
||||
PollOutcome::Packet(au) => Ok(Some(au)),
|
||||
PollOutcome::Again | PollOutcome::Eof => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Result<()> {
|
||||
self.enc.send_eof().context("send_eof")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl NvencEncoder {
|
||||
/// CPU path: expand/copy the packed RGB/BGR bytes into the reusable 4-bpp frame, then send.
|
||||
fn submit_cpu(&mut self, bytes: &[u8], format: PixelFormat, pts: i64, idr: bool) -> Result<()> {
|
||||
anyhow::ensure!(
|
||||
format == self.src_format,
|
||||
"captured format {:?} != encoder source {:?}",
|
||||
format,
|
||||
self.src_format
|
||||
);
|
||||
let w = self.width as usize;
|
||||
let h = self.height as usize;
|
||||
let src_bpp = self.src_format.bytes_per_pixel();
|
||||
let src_row = w * src_bpp;
|
||||
anyhow::ensure!(
|
||||
bytes.len() >= src_row * h,
|
||||
"captured buffer {} bytes < required {}",
|
||||
bytes.len(),
|
||||
src_row * h
|
||||
);
|
||||
// 4:4:4: swscale the packed RGB straight into the planar YUV444P input frame (BT.709 limited),
|
||||
// then send it — no byte-expand. The 4:2:0 RGB path (below) feeds NVENC packed RGB directly.
|
||||
if let Some(sws) = self.sws_444 {
|
||||
let frame = self
|
||||
.frame
|
||||
.as_mut()
|
||||
.context("CPU frame missing (encoder opened in CUDA mode)")?;
|
||||
// SAFETY: `format == self.src_format` and `bytes.len() >= src_row * h` (the `ensure!`s
|
||||
// above), so `sws_scale` reads `h` rows of `src_row` bytes from `src_data[0] = bytes`
|
||||
// (packed RGB is single-plane; the other src planes are null/0) — all in bounds. `sws` is
|
||||
// the non-null context built in `open`. The dst is `frame`'s underlying `AVFrame`: its
|
||||
// `data`/`linesize` in-struct arrays were sized for YUV444P by `VideoFrame::new`, and the
|
||||
// 3 planes are each `width`×`height`. All pointers are live locals for this synchronous
|
||||
// call; the encoder runs only on this thread (`unsafe impl Send`), so no aliasing/race.
|
||||
unsafe {
|
||||
let dst_av = frame.as_mut_ptr();
|
||||
let src_data: [*const u8; 4] =
|
||||
[bytes.as_ptr(), ptr::null(), ptr::null(), ptr::null()];
|
||||
let src_stride: [c_int; 4] = [src_row as c_int, 0, 0, 0];
|
||||
let r = ffi::sws_scale(
|
||||
sws,
|
||||
src_data.as_ptr(),
|
||||
src_stride.as_ptr(),
|
||||
0,
|
||||
h as c_int,
|
||||
(*dst_av).data.as_ptr(),
|
||||
(*dst_av).linesize.as_ptr(),
|
||||
);
|
||||
if r < 0 {
|
||||
bail!("sws_scale(RGB→YUV444P) failed ({r})");
|
||||
}
|
||||
}
|
||||
frame.set_pts(Some(pts));
|
||||
frame.set_kind(if idr {
|
||||
ffmpeg::picture::Type::I
|
||||
} else {
|
||||
ffmpeg::picture::Type::None
|
||||
});
|
||||
self.enc.send_frame(frame).context("send_frame(444)")?;
|
||||
return Ok(());
|
||||
}
|
||||
let frame = self
|
||||
.frame
|
||||
.as_mut()
|
||||
.context("CPU frame missing (encoder opened in CUDA mode)")?;
|
||||
let stride = frame.stride(0); // dst is 4-bpp, aligned
|
||||
let dst = frame.data_mut(0);
|
||||
if self.expand {
|
||||
// packed 3-bpp RGB/BGR → 4-bpp *0 (copy 3 bytes, zero the pad byte)
|
||||
for y in 0..h {
|
||||
let s = &bytes[y * src_row..y * src_row + src_row];
|
||||
let drow = &mut dst[y * stride..y * stride + w * 4];
|
||||
for x in 0..w {
|
||||
drow[x * 4..x * 4 + 3].copy_from_slice(&s[x * 3..x * 3 + 3]);
|
||||
drow[x * 4 + 3] = 0;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 4-bpp → 4-bpp, honoring the (possibly larger) dst stride
|
||||
for y in 0..h {
|
||||
dst[y * stride..y * stride + src_row]
|
||||
.copy_from_slice(&bytes[y * src_row..y * src_row + src_row]);
|
||||
}
|
||||
}
|
||||
frame.set_pts(Some(pts));
|
||||
frame.set_kind(if idr {
|
||||
ffmpeg::picture::Type::I
|
||||
} else {
|
||||
ffmpeg::picture::Type::None
|
||||
});
|
||||
self.enc.send_frame(frame).context("send_frame")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Zero-copy path: hand the imported CUDA device buffer to NVENC with no CPU touch.
|
||||
///
|
||||
/// We take a *pooled* surface from the CUDA hwframes context (`av_hwframe_get_buffer`) and
|
||||
/// device→device-copy our imported buffer into it, rather than wrapping our own pointer in a
|
||||
/// bare frame. Two reasons: (1) NVENC's `nvenc_send_frame` ignores frames whose `buf[0]` is
|
||||
/// null and the generic encode path's `av_frame_ref` needs a refcounted buffer — a bare
|
||||
/// frame is rejected with `EINVAL`; (2) NVENC caches CUDA-resource *registrations* keyed by
|
||||
/// device pointer with a bounded table, so a fresh pointer every frame would thrash/overflow
|
||||
/// it — the pool recycles a small set of pointers. The extra copy is device-local (~8 MB at
|
||||
/// 1080p, sub-millisecond on the GPU) and keeps the host fully off the pixel path.
|
||||
fn submit_cuda(&mut self, buf: &pf_zerocopy::DeviceBuffer, pts: i64, idr: bool) -> Result<()> {
|
||||
let frames_ref = self
|
||||
.cuda
|
||||
.as_ref()
|
||||
.context("CUDA hw context missing (encoder opened in CPU mode)")?
|
||||
.frames_ref;
|
||||
// The device→device copy below uses our shared context directly; make it current on the
|
||||
// encode thread (ffmpeg pushes its own around the pool alloc, so order is fine).
|
||||
pf_zerocopy::cuda::make_current().context("CUDA context current (encode thread)")?;
|
||||
// SAFETY: `frames_ref` is the non-null CUDA frames ctx from `self.cuda` (unwrapped via
|
||||
// `.context(..)?` above), and the shared CUDA context was just made current on THIS thread
|
||||
// (`make_current()?`), the precondition for the device-pointer copies below.
|
||||
// * `av_frame_alloc` → `f` (null-checked). `av_hwframe_get_buffer(frames_ref, f, 0)` fills `f`
|
||||
// with a pooled CUDA surface (sets `data[]`/`linesize[]`/`buf[0]`/`hw_frames_ctx`); on
|
||||
// failure we free `f` and bail.
|
||||
// * For NV12 we read `(*f).data[0..2]` / `linesize[0..2]` (Y + interleaved UV), else
|
||||
// `data[0]`/`linesize[0]` — in-struct fields of the non-null `f`, valid for the surface dims
|
||||
// ffmpeg allocated — and pass them to the cuda copy helpers, which device→device copy `buf`
|
||||
// (the imported `DeviceBuffer`, owned by the caller and live for this call) into the surface.
|
||||
// * On copy error we free `f` and return. Otherwise we write `pts`/`pict_type` through `f` and
|
||||
// `avcodec_send_frame` it into the live owned `self.enc` context (which takes its own ref of
|
||||
// the pooled surface), then free our `f` ref exactly once. Single-threaded encoder → no race.
|
||||
unsafe {
|
||||
let mut f = ffi::av_frame_alloc();
|
||||
if f.is_null() {
|
||||
bail!("av_frame_alloc failed");
|
||||
}
|
||||
// Pooled CUDA surface: sets format, width/height, data[0]/linesize[0], buf[0] and
|
||||
// hw_frames_ctx. Reused across frames (the pool recycles), keeping NVENC's
|
||||
// registration cache warm.
|
||||
let r = ffi::av_hwframe_get_buffer(frames_ref, f, 0);
|
||||
if r < 0 {
|
||||
ffi::av_frame_free(&mut f);
|
||||
bail!("av_hwframe_get_buffer(CUDA) failed ({r})");
|
||||
}
|
||||
// NV12 surfaces are two-plane (Y in data[0], interleaved UV in data[1]); YUV444
|
||||
// surfaces are three-plane (`yuv444p` frames ctx — data[0..3]); the RGB surfaces are
|
||||
// single-plane. Copy the matching layout into NVENC's pooled surface. A 4:4:4 session
|
||||
// whose buffer ISN'T YUV444 (a LINEAR/gamescope capture the worker can't convert)
|
||||
// fails loudly here rather than letting `hevc_nvenc` silently subsample RGB to 4:2:0.
|
||||
let copy_res = if buf.yuv444 {
|
||||
let dsts = core::array::from_fn(|i| {
|
||||
(
|
||||
(*f).data[i] as pf_zerocopy::cuda::CUdeviceptr,
|
||||
(*f).linesize[i] as usize,
|
||||
)
|
||||
});
|
||||
pf_zerocopy::cuda::copy_yuv444_to_device(buf, dsts)
|
||||
} else if self.want_444 {
|
||||
ffi::av_frame_free(&mut f);
|
||||
bail!(
|
||||
"4:4:4 session but the zero-copy frame is not YUV444 (LINEAR/gamescope \
|
||||
capture has no GPU 4:4:4 convert) — unset PUNKTFUNK_ZEROCOPY to use the \
|
||||
CPU 4:4:4 path on this compositor"
|
||||
);
|
||||
} else if buf.is_nv12() {
|
||||
let y_ptr = (*f).data[0] as pf_zerocopy::cuda::CUdeviceptr;
|
||||
let y_pitch = (*f).linesize[0] as usize;
|
||||
let uv_ptr = (*f).data[1] as pf_zerocopy::cuda::CUdeviceptr;
|
||||
let uv_pitch = (*f).linesize[1] as usize;
|
||||
pf_zerocopy::cuda::copy_nv12_to_device(buf, y_ptr, y_pitch, uv_ptr, uv_pitch)
|
||||
} else {
|
||||
let dst_ptr = (*f).data[0] as pf_zerocopy::cuda::CUdeviceptr;
|
||||
let dst_pitch = (*f).linesize[0] as usize;
|
||||
pf_zerocopy::cuda::copy_device_to_device(buf, dst_ptr, dst_pitch)
|
||||
};
|
||||
if let Err(e) = copy_res {
|
||||
ffi::av_frame_free(&mut f);
|
||||
return Err(e).context("copy imported buffer into NVENC surface");
|
||||
}
|
||||
(*f).pts = pts;
|
||||
(*f).pict_type = if idr {
|
||||
ffi::AVPictureType::AV_PICTURE_TYPE_I
|
||||
} else {
|
||||
ffi::AVPictureType::AV_PICTURE_TYPE_NONE
|
||||
};
|
||||
let r = ffi::avcodec_send_frame(self.enc.as_mut_ptr(), f);
|
||||
ffi::av_frame_free(&mut f);
|
||||
if r < 0 {
|
||||
bail!("avcodec_send_frame(CUDA) failed ({r})");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for NvencEncoder {
|
||||
fn drop(&mut self) {
|
||||
if let Some(sws) = self.sws_444.take() {
|
||||
// SAFETY: `sws` is the non-null `SwsContext` allocated by `sws_getContext` in `open` and
|
||||
// owned exclusively by this encoder (taken out of the field so it can't be freed twice).
|
||||
// `sws_freeContext` frees it; nothing else references it after this single-threaded drop.
|
||||
unsafe { ffi::sws_freeContext(sws) };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe whether this NVIDIA GPU + driver + libavcodec can actually encode HEVC **4:4:4** (Range
|
||||
/// Extensions). Opens a tiny real `hevc_nvenc` 4:4:4 session — the exact path [`NvencEncoder::open`]
|
||||
/// takes for a live 4:4:4 stream — and reports whether it succeeded. HEVC-only; the result is cached
|
||||
/// by the caller ([`crate::can_encode_444`]). A GPU/driver/ffmpeg without RExt 4:4:4 fails
|
||||
/// the open here, so the host resolves the session to 4:2:0 before the Welcome (honest downgrade).
|
||||
pub fn probe_can_encode_444(codec: Codec) -> bool {
|
||||
if codec != Codec::H265 {
|
||||
return false;
|
||||
}
|
||||
if ffmpeg::init().is_err() {
|
||||
return false;
|
||||
}
|
||||
// Quiet ffmpeg's open error on a GPU that lacks 4:4:4 — the probe failing is an expected outcome.
|
||||
// SAFETY: libav initialized above; `av_log_{get,set}_level` only read/write the global int level
|
||||
// (no pointer args) and are always sound post-init.
|
||||
let prev = unsafe {
|
||||
let p = ffi::av_log_get_level();
|
||||
ffi::av_log_set_level(ffi::AV_LOG_FATAL);
|
||||
p
|
||||
};
|
||||
let ok = NvencEncoder::open(
|
||||
codec,
|
||||
PixelFormat::Bgra,
|
||||
640,
|
||||
480,
|
||||
30,
|
||||
2_000_000,
|
||||
false, // CPU input (the 4:4:4 path never uses CUDA)
|
||||
8,
|
||||
ChromaFormat::Yuv444,
|
||||
)
|
||||
.is_ok();
|
||||
// SAFETY: restore the saved global log level (scalar arg, no pointers).
|
||||
unsafe { ffi::av_log_set_level(prev) };
|
||||
ok
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,51 @@
|
||||
#version 450
|
||||
// RGB(A) -> NV12 (BT.709 limited range). One invocation per chroma sample = 2x2 luma block.
|
||||
// Optionally blends a straight-alpha RGBA cursor bitmap over the RGB *before* the YUV conversion
|
||||
// (so the chroma stays correct) — cursor-as-metadata for the GPU zero-copy paths. The blend is
|
||||
// gated by the push constant and touches only the cursor's small rectangle, so a disabled or
|
||||
// off-screen cursor costs one compare per invocation.
|
||||
layout(local_size_x = 8, local_size_y = 8) in;
|
||||
layout(binding = 0) uniform sampler2D rgb; // packed RGB input (sampled; BGRA import ok)
|
||||
layout(binding = 1, r8) uniform writeonly image2D yImg; // full-res Y
|
||||
layout(binding = 2, rg8) uniform writeonly image2D uvImg; // half-res UV (interleaved)
|
||||
layout(binding = 3) uniform sampler2D cursorTex; // straight-alpha RGBA cursor (top-left)
|
||||
|
||||
layout(push_constant) uniform Push {
|
||||
ivec2 curOrigin; // top-left of the cursor in frame pixels (position - hotspot)
|
||||
ivec2 curSize; // cursor w,h in pixels; x <= 0 => disabled
|
||||
} pc;
|
||||
|
||||
float lumaY(vec3 c) { return 16.0/255.0 + 0.1826*c.r + 0.6142*c.g + 0.0620*c.b; }
|
||||
|
||||
// Blend the cursor over `col` at frame pixel `p`, when `p` falls inside the cursor rectangle.
|
||||
vec3 withCursor(ivec2 p, vec3 col) {
|
||||
if (pc.curSize.x <= 0) return col;
|
||||
ivec2 cp = p - pc.curOrigin;
|
||||
if (cp.x < 0 || cp.y < 0 || cp.x >= pc.curSize.x || cp.y >= pc.curSize.y) return col;
|
||||
vec4 c = texelFetch(cursorTex, cp, 0);
|
||||
return mix(col, c.rgb, c.a);
|
||||
}
|
||||
|
||||
// Source may be SMALLER than the coded (16-aligned) Y plane — e.g. 1080 source vs 1088 coded. Clamp
|
||||
// every fetch to the source edge so the alignment-padding rows duplicate the last real row instead
|
||||
// of reading out of bounds (undefined → green garbage that shows if a client ignores the SPS
|
||||
// conformance-window crop). `textureSize` gives the bound source's real extent.
|
||||
void main() {
|
||||
ivec2 sz = imageSize(yImg);
|
||||
ivec2 rmax = textureSize(rgb, 0) - 1;
|
||||
ivec2 uvc = ivec2(gl_GlobalInvocationID.xy);
|
||||
ivec2 p = uvc * 2;
|
||||
if (p.x >= sz.x || p.y >= sz.y) return;
|
||||
vec3 c00 = withCursor(p, texelFetch(rgb, min(p, rmax), 0).rgb);
|
||||
vec3 c10 = withCursor(p + ivec2(1, 0), texelFetch(rgb, min(p + ivec2(1, 0), rmax), 0).rgb);
|
||||
vec3 c01 = withCursor(p + ivec2(0, 1), texelFetch(rgb, min(p + ivec2(0, 1), rmax), 0).rgb);
|
||||
vec3 c11 = withCursor(p + ivec2(1, 1), texelFetch(rgb, min(p + ivec2(1, 1), rmax), 0).rgb);
|
||||
imageStore(yImg, p, vec4(lumaY(c00), 0, 0, 1));
|
||||
imageStore(yImg, p + ivec2(1, 0), vec4(lumaY(c10), 0, 0, 1));
|
||||
imageStore(yImg, p + ivec2(0, 1), vec4(lumaY(c01), 0, 0, 1));
|
||||
imageStore(yImg, p + ivec2(1, 1), vec4(lumaY(c11), 0, 0, 1));
|
||||
vec3 a = (c00 + c10 + c01 + c11) * 0.25;
|
||||
float U = 128.0/255.0 - 0.1006*a.r - 0.3386*a.g + 0.4392*a.b;
|
||||
float V = 128.0/255.0 + 0.4392*a.r - 0.3989*a.g - 0.0403*a.b;
|
||||
imageStore(uvImg, uvc, vec4(U, V, 0, 1));
|
||||
}
|
||||
Binary file not shown.
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,500 @@
|
||||
//! Vendored `VK_KHR_video_encode_av1` bindings — the AV1-encode structs, `StdVideoEncodeAV1*`
|
||||
//! types and struct-type constants that our pinned `ash 0.38.0+1.3.281` does not ship (the
|
||||
//! extension was finalized in Vulkan 1.3.290). Bumping `ash` to git-master (`+1.4.352`, which has
|
||||
//! them) breaks the *client*: it drops the lifetime on `vk::AllocationCallbacks`, and `sdl3-sys`'s
|
||||
//! `ash` feature still generates `AllocationCallbacks<'static>`, so the presenter's SDL/Vulkan
|
||||
//! surface path won't compile. Rather than churn the client for a host-only need, we vendor just
|
||||
//! the encode-side definitions here, **copied verbatim from ash-master's generated code** (so the
|
||||
//! layouts are correct-by-construction) and chain them into ash's generic video-encode-queue calls
|
||||
//! via raw `p_next`, exactly as the HEVC path already chains its rate-control struct.
|
||||
//!
|
||||
//! Everything *common* to AV1 (sequence header, tile/quant/loop-filter/CDEF/… sub-structs, the
|
||||
//! `StdVideoAV1*` enums) is already present in 1.3.281's `ash::vk::native` — AV1 **decode** brought
|
||||
//! it in — so we reuse those and vendor only the encode-specific pieces. Delete this module and
|
||||
//! switch to `ash::vk::*` once `ash` publishes a 1.4.x release and `sdl3-sys` regenerates.
|
||||
#![allow(non_snake_case, non_camel_case_types, dead_code)]
|
||||
|
||||
use ash::vk;
|
||||
use ash::vk::native::{
|
||||
StdVideoAV1FrameType, StdVideoAV1InterpolationFilter, StdVideoAV1Level, StdVideoAV1Profile,
|
||||
StdVideoAV1SequenceHeader, StdVideoAV1TxMode,
|
||||
};
|
||||
use std::ffi::{c_void, CStr};
|
||||
|
||||
/// `VK_KHR_video_encode_av1` extension name — ash 0.38's `ash::khr::video_encode_av1` doesn't exist,
|
||||
/// so we pass this raw to `enabled_extension_names`.
|
||||
pub const EXTENSION_NAME: &CStr = c"VK_KHR_video_encode_av1";
|
||||
|
||||
// ---------- struct-type (VkStructureType) values — construct via `vk::StructureType::from_raw` ----------
|
||||
pub const ST_CAPABILITIES: i32 = 1_000_513_000;
|
||||
pub const ST_SESSION_PARAMETERS_CREATE_INFO: i32 = 1_000_513_001;
|
||||
pub const ST_PICTURE_INFO: i32 = 1_000_513_002;
|
||||
pub const ST_DPB_SLOT_INFO: i32 = 1_000_513_003;
|
||||
pub const ST_PHYSICAL_DEVICE_FEATURES: i32 = 1_000_513_004;
|
||||
pub const ST_PROFILE_INFO: i32 = 1_000_513_005;
|
||||
pub const ST_RATE_CONTROL_INFO: i32 = 1_000_513_006;
|
||||
pub const ST_RATE_CONTROL_LAYER_INFO: i32 = 1_000_513_007;
|
||||
pub const ST_SESSION_CREATE_INFO: i32 = 1_000_513_009;
|
||||
pub const ST_GOP_REMAINING_FRAME_INFO: i32 = 1_000_513_010;
|
||||
|
||||
/// `VK_VIDEO_CODEC_OPERATION_ENCODE_AV1_BIT_KHR` (bit 18).
|
||||
pub const VIDEO_CODEC_OPERATION_ENCODE_AV1: u32 = 0x0004_0000;
|
||||
/// `VK_MAX_VIDEO_AV1_REFERENCES_PER_FRAME_KHR` — LAST..ALTREF (the 7 inter reference names).
|
||||
pub const MAX_VIDEO_AV1_REFERENCES_PER_FRAME: usize = 7;
|
||||
/// `STD_VIDEO_AV1_PRIMARY_REF_NONE` — a frame that inherits no CDF/context from any reference
|
||||
/// (the recovery-anchor lever: a clean P-frame independent of prior probability state).
|
||||
pub const PRIMARY_REF_NONE: u8 = 7;
|
||||
/// `VK_VIDEO_ENCODE_AV1_SUPERBLOCK_SIZE_128_BIT_KHR` (bit 1 of the superblock-size flags).
|
||||
pub const SUPERBLOCK_SIZE_128: u32 = 0x2;
|
||||
|
||||
// `VkVideoEncodeAV1PredictionModeKHR`
|
||||
pub const PREDICTION_MODE_INTRA_ONLY: i32 = 0;
|
||||
pub const PREDICTION_MODE_SINGLE_REFERENCE: i32 = 1;
|
||||
// `VkVideoEncodeAV1RateControlGroupKHR`
|
||||
pub const RC_GROUP_INTRA: i32 = 0;
|
||||
pub const RC_GROUP_PREDICTIVE: i32 = 1;
|
||||
pub const RC_GROUP_BIPREDICTIVE: i32 = 2;
|
||||
|
||||
// AV1 reference names (index into `reference_name_slot_indices`, which is 0-based over LAST..ALTREF).
|
||||
pub const REFERENCE_NAME_LAST_FRAME_IDX: usize = 0; // STD_VIDEO_AV1_REFERENCE_NAME_LAST_FRAME - 1
|
||||
|
||||
// ---------- bindgen bitfield helper (copied verbatim from ash-master native.rs) ----------
|
||||
#[repr(C)]
|
||||
#[derive(Debug, Default, Copy, Clone)]
|
||||
pub struct __BindgenBitfieldUnit<Storage> {
|
||||
storage: Storage,
|
||||
}
|
||||
impl<Storage> __BindgenBitfieldUnit<Storage> {
|
||||
#[inline]
|
||||
pub const fn new(storage: Storage) -> Self {
|
||||
Self { storage }
|
||||
}
|
||||
}
|
||||
impl<Storage> __BindgenBitfieldUnit<Storage>
|
||||
where
|
||||
Storage: AsRef<[u8]> + AsMut<[u8]>,
|
||||
{
|
||||
#[inline]
|
||||
pub fn get_bit(&self, index: usize) -> bool {
|
||||
let byte_index = index / 8;
|
||||
let byte = self.storage.as_ref()[byte_index];
|
||||
let bit_index = if cfg!(target_endian = "big") {
|
||||
7 - (index % 8)
|
||||
} else {
|
||||
index % 8
|
||||
};
|
||||
byte & (1 << bit_index) == (1 << bit_index)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_bit(&mut self, index: usize, val: bool) {
|
||||
let byte_index = index / 8;
|
||||
let byte = &mut self.storage.as_mut()[byte_index];
|
||||
let bit_index = if cfg!(target_endian = "big") {
|
||||
7 - (index % 8)
|
||||
} else {
|
||||
index % 8
|
||||
};
|
||||
let mask = 1 << bit_index;
|
||||
if val {
|
||||
*byte |= mask;
|
||||
} else {
|
||||
*byte &= !mask;
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
pub fn get(&self, bit_offset: usize, bit_width: u8) -> u64 {
|
||||
let mut val = 0;
|
||||
for i in 0..(bit_width as usize) {
|
||||
if self.get_bit(i + bit_offset) {
|
||||
let index = if cfg!(target_endian = "big") {
|
||||
bit_width as usize - 1 - i
|
||||
} else {
|
||||
i
|
||||
};
|
||||
val |= 1 << index;
|
||||
}
|
||||
}
|
||||
val
|
||||
}
|
||||
#[inline]
|
||||
pub fn set(&mut self, bit_offset: usize, bit_width: u8, val: u64) {
|
||||
for i in 0..(bit_width as usize) {
|
||||
let mask = 1 << i;
|
||||
let val_bit_is_set = val & mask == mask;
|
||||
let index = if cfg!(target_endian = "big") {
|
||||
bit_width as usize - 1 - i
|
||||
} else {
|
||||
i
|
||||
};
|
||||
self.set_bit(index + bit_offset, val_bit_is_set);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- Std encode structs (copied from ash-master native.rs; common Std types reused from ash) ----------
|
||||
#[repr(C, align(4))]
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct StdVideoEncodeAV1PictureInfoFlags {
|
||||
pub _bitfield_align_1: [u8; 0],
|
||||
pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>,
|
||||
}
|
||||
impl StdVideoEncodeAV1PictureInfoFlags {
|
||||
#[inline]
|
||||
pub fn set_error_resilient_mode(&mut self, val: u32) {
|
||||
self._bitfield_1.set(0, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_disable_cdf_update(&mut self, val: u32) {
|
||||
self._bitfield_1.set(1, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_use_superres(&mut self, val: u32) {
|
||||
self._bitfield_1.set(2, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_render_and_frame_size_different(&mut self, val: u32) {
|
||||
self._bitfield_1.set(3, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_allow_screen_content_tools(&mut self, val: u32) {
|
||||
self._bitfield_1.set(4, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_is_filter_switchable(&mut self, val: u32) {
|
||||
self._bitfield_1.set(5, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_force_integer_mv(&mut self, val: u32) {
|
||||
self._bitfield_1.set(6, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_frame_size_override_flag(&mut self, val: u32) {
|
||||
self._bitfield_1.set(7, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_buffer_removal_time_present_flag(&mut self, val: u32) {
|
||||
self._bitfield_1.set(8, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_allow_intrabc(&mut self, val: u32) {
|
||||
self._bitfield_1.set(9, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_frame_refs_short_signaling(&mut self, val: u32) {
|
||||
self._bitfield_1.set(10, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_allow_high_precision_mv(&mut self, val: u32) {
|
||||
self._bitfield_1.set(11, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_is_motion_mode_switchable(&mut self, val: u32) {
|
||||
self._bitfield_1.set(12, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_use_ref_frame_mvs(&mut self, val: u32) {
|
||||
self._bitfield_1.set(13, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_disable_frame_end_update_cdf(&mut self, val: u32) {
|
||||
self._bitfield_1.set(14, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_allow_warped_motion(&mut self, val: u32) {
|
||||
self._bitfield_1.set(15, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_reduced_tx_set(&mut self, val: u32) {
|
||||
self._bitfield_1.set(16, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_skip_mode_present(&mut self, val: u32) {
|
||||
self._bitfield_1.set(17, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_delta_q_present(&mut self, val: u32) {
|
||||
self._bitfield_1.set(18, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_delta_lf_present(&mut self, val: u32) {
|
||||
self._bitfield_1.set(19, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_delta_lf_multi(&mut self, val: u32) {
|
||||
self._bitfield_1.set(20, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_segmentation_enabled(&mut self, val: u32) {
|
||||
self._bitfield_1.set(21, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_segmentation_update_map(&mut self, val: u32) {
|
||||
self._bitfield_1.set(22, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_segmentation_temporal_update(&mut self, val: u32) {
|
||||
self._bitfield_1.set(23, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_segmentation_update_data(&mut self, val: u32) {
|
||||
self._bitfield_1.set(24, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_UsesLr(&mut self, val: u32) {
|
||||
self._bitfield_1.set(25, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_usesChromaLr(&mut self, val: u32) {
|
||||
self._bitfield_1.set(26, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_show_frame(&mut self, val: u32) {
|
||||
self._bitfield_1.set(27, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_showable_frame(&mut self, val: u32) {
|
||||
self._bitfield_1.set(28, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_reserved(&mut self, val: u32) {
|
||||
self._bitfield_1.set(29, 3, val as u64)
|
||||
}
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct StdVideoEncodeAV1PictureInfo {
|
||||
pub flags: StdVideoEncodeAV1PictureInfoFlags,
|
||||
pub frame_type: StdVideoAV1FrameType,
|
||||
pub frame_presentation_time: u32,
|
||||
pub current_frame_id: u32,
|
||||
pub order_hint: u8,
|
||||
pub primary_ref_frame: u8,
|
||||
pub refresh_frame_flags: u8,
|
||||
pub coded_denom: u8,
|
||||
pub render_width_minus_1: u16,
|
||||
pub render_height_minus_1: u16,
|
||||
pub interpolation_filter: StdVideoAV1InterpolationFilter,
|
||||
pub TxMode: StdVideoAV1TxMode,
|
||||
pub delta_q_res: u8,
|
||||
pub delta_lf_res: u8,
|
||||
pub ref_order_hint: [u8; 8usize],
|
||||
pub ref_frame_idx: [i8; 7usize],
|
||||
pub reserved1: [u8; 3usize],
|
||||
pub delta_frame_id_minus_1: [u32; 7usize],
|
||||
pub pTileInfo: *const ash::vk::native::StdVideoAV1TileInfo,
|
||||
pub pQuantization: *const ash::vk::native::StdVideoAV1Quantization,
|
||||
pub pSegmentation: *const ash::vk::native::StdVideoAV1Segmentation,
|
||||
pub pLoopFilter: *const ash::vk::native::StdVideoAV1LoopFilter,
|
||||
pub pCDEF: *const ash::vk::native::StdVideoAV1CDEF,
|
||||
pub pLoopRestoration: *const ash::vk::native::StdVideoAV1LoopRestoration,
|
||||
pub pGlobalMotion: *const ash::vk::native::StdVideoAV1GlobalMotion,
|
||||
pub pExtensionHeader: *const StdVideoEncodeAV1ExtensionHeader,
|
||||
pub pBufferRemovalTimes: *const u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct StdVideoEncodeAV1ReferenceInfoFlags {
|
||||
pub _bitfield_align_1: [u32; 0],
|
||||
pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>,
|
||||
}
|
||||
impl StdVideoEncodeAV1ReferenceInfoFlags {
|
||||
#[inline]
|
||||
pub fn set_disable_frame_end_update_cdf(&mut self, val: u32) {
|
||||
self._bitfield_1.set(0, 1, val as u64)
|
||||
}
|
||||
#[inline]
|
||||
pub fn set_segmentation_enabled(&mut self, val: u32) {
|
||||
self._bitfield_1.set(1, 1, val as u64)
|
||||
}
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct StdVideoEncodeAV1ReferenceInfo {
|
||||
pub flags: StdVideoEncodeAV1ReferenceInfoFlags,
|
||||
pub RefFrameId: u32,
|
||||
pub frame_type: StdVideoAV1FrameType,
|
||||
pub OrderHint: u8,
|
||||
pub reserved1: [u8; 3usize],
|
||||
pub pExtensionHeader: *const StdVideoEncodeAV1ExtensionHeader,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct StdVideoEncodeAV1ExtensionHeader {
|
||||
pub temporal_id: u8,
|
||||
pub spatial_id: u8,
|
||||
}
|
||||
|
||||
// ---------- KHR extension structs (repr(C); lifetimes/PhantomData dropped — layout-identical,
|
||||
// chained by raw p_next). Flag/enum newtypes flattened to their u32/i32 repr. ----------
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct VideoEncodeAV1ProfileInfoKHR {
|
||||
pub s_type: vk::StructureType,
|
||||
pub p_next: *const c_void,
|
||||
pub std_profile: StdVideoAV1Profile,
|
||||
}
|
||||
|
||||
/// `VkPhysicalDeviceVideoEncodeAV1FeaturesKHR` — the `videoEncodeAV1` feature MUST be enabled at
|
||||
/// device creation for any `VK_VIDEO_CODEC_OPERATION_ENCODE_AV1` use (a spec requirement RADV may
|
||||
/// tolerate omitting but validation layers and stricter drivers do not).
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct PhysicalDeviceVideoEncodeAV1FeaturesKHR {
|
||||
pub s_type: vk::StructureType,
|
||||
pub p_next: *mut c_void,
|
||||
pub video_encode_av1: vk::Bool32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct VideoEncodeAV1CapabilitiesKHR {
|
||||
pub s_type: vk::StructureType,
|
||||
pub p_next: *mut c_void,
|
||||
pub flags: u32,
|
||||
pub max_level: StdVideoAV1Level,
|
||||
pub coded_picture_alignment: vk::Extent2D,
|
||||
pub max_tiles: vk::Extent2D,
|
||||
pub min_tile_size: vk::Extent2D,
|
||||
pub max_tile_size: vk::Extent2D,
|
||||
pub superblock_sizes: u32,
|
||||
pub max_single_reference_count: u32,
|
||||
pub single_reference_name_mask: u32,
|
||||
pub max_unidirectional_compound_reference_count: u32,
|
||||
pub max_unidirectional_compound_group1_reference_count: u32,
|
||||
pub unidirectional_compound_reference_name_mask: u32,
|
||||
pub max_bidirectional_compound_reference_count: u32,
|
||||
pub max_bidirectional_compound_group1_reference_count: u32,
|
||||
pub max_bidirectional_compound_group2_reference_count: u32,
|
||||
pub bidirectional_compound_reference_name_mask: u32,
|
||||
pub max_temporal_layer_count: u32,
|
||||
pub max_spatial_layer_count: u32,
|
||||
pub max_operating_points: u32,
|
||||
pub min_q_index: u32,
|
||||
pub max_q_index: u32,
|
||||
pub prefers_gop_remaining_frames: vk::Bool32,
|
||||
pub requires_gop_remaining_frames: vk::Bool32,
|
||||
pub std_syntax_flags: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct VideoEncodeAV1SessionParametersCreateInfoKHR {
|
||||
pub s_type: vk::StructureType,
|
||||
pub p_next: *const c_void,
|
||||
pub p_std_sequence_header: *const StdVideoAV1SequenceHeader,
|
||||
pub p_std_decoder_model_info: *const c_void,
|
||||
pub std_operating_point_count: u32,
|
||||
pub p_std_operating_points: *const c_void,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct VideoEncodeAV1PictureInfoKHR {
|
||||
pub s_type: vk::StructureType,
|
||||
pub p_next: *const c_void,
|
||||
pub prediction_mode: i32,
|
||||
pub rate_control_group: i32,
|
||||
pub constant_q_index: u32,
|
||||
pub p_std_picture_info: *const StdVideoEncodeAV1PictureInfo,
|
||||
pub reference_name_slot_indices: [i32; MAX_VIDEO_AV1_REFERENCES_PER_FRAME],
|
||||
pub primary_reference_cdf_only: vk::Bool32,
|
||||
pub generate_obu_extension_header: vk::Bool32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct VideoEncodeAV1DpbSlotInfoKHR {
|
||||
pub s_type: vk::StructureType,
|
||||
pub p_next: *const c_void,
|
||||
pub p_std_reference_info: *const StdVideoEncodeAV1ReferenceInfo,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct VideoEncodeAV1RateControlInfoKHR {
|
||||
pub s_type: vk::StructureType,
|
||||
pub p_next: *const c_void,
|
||||
pub flags: u32,
|
||||
pub gop_frame_count: u32,
|
||||
pub key_frame_period: u32,
|
||||
pub consecutive_bipredictive_frame_count: u32,
|
||||
pub temporal_layer_count: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct VideoEncodeAV1QIndexKHR {
|
||||
pub intra_q_index: u32,
|
||||
pub predictive_q_index: u32,
|
||||
pub bipredictive_q_index: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct VideoEncodeAV1FrameSizeKHR {
|
||||
pub intra_frame_size: u32,
|
||||
pub predictive_frame_size: u32,
|
||||
pub bipredictive_frame_size: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct VideoEncodeAV1RateControlLayerInfoKHR {
|
||||
pub s_type: vk::StructureType,
|
||||
pub p_next: *const c_void,
|
||||
pub use_min_q_index: vk::Bool32,
|
||||
pub min_q_index: VideoEncodeAV1QIndexKHR,
|
||||
pub use_max_q_index: vk::Bool32,
|
||||
pub max_q_index: VideoEncodeAV1QIndexKHR,
|
||||
pub use_max_frame_size: vk::Bool32,
|
||||
pub max_frame_size: VideoEncodeAV1FrameSizeKHR,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct VideoEncodeAV1GopRemainingFrameInfoKHR {
|
||||
pub s_type: vk::StructureType,
|
||||
pub p_next: *const c_void,
|
||||
pub use_gop_remaining_frames: vk::Bool32,
|
||||
pub gop_remaining_intra: u32,
|
||||
pub gop_remaining_predictive: u32,
|
||||
pub gop_remaining_bipredictive: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct VideoEncodeAV1SessionCreateInfoKHR {
|
||||
pub s_type: vk::StructureType,
|
||||
pub p_next: *const c_void,
|
||||
pub use_max_level: vk::Bool32,
|
||||
pub max_level: StdVideoAV1Level,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct StdVideoEncodeAV1OperatingPointInfoFlags {
|
||||
pub _bitfield_align_1: [u32; 0],
|
||||
pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct StdVideoEncodeAV1OperatingPointInfo {
|
||||
pub flags: StdVideoEncodeAV1OperatingPointInfoFlags,
|
||||
pub operating_point_idc: u16,
|
||||
pub seq_level_idx: u8,
|
||||
pub seq_tier: u8,
|
||||
pub decoder_buffer_delay: u32,
|
||||
pub encoder_buffer_delay: u32,
|
||||
pub initial_display_delay_minus_1: u8,
|
||||
}
|
||||
|
||||
/// `vk::StructureType` for a raw `ST_*` constant above.
|
||||
#[inline]
|
||||
pub fn stype(raw: i32) -> vk::StructureType {
|
||||
vk::StructureType::from_raw(raw)
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
//! Small ash/Vulkan leaf helpers shared by the Linux Vulkan encode backends
|
||||
//! (`vulkan_video.rs`, `pyrowave.rs`) — extracted verbatim from `vulkan_video.rs`
|
||||
//! when the PyroWave backend arrived so the two don't fork copies.
|
||||
// Every unsafe block carries a `// SAFETY:` proof (parent module enforces it).
|
||||
|
||||
use anyhow::Result;
|
||||
use ash::vk;
|
||||
use pf_frame::PixelFormat;
|
||||
|
||||
pub(crate) fn color_range(layer: u32) -> vk::ImageSubresourceRange {
|
||||
vk::ImageSubresourceRange {
|
||||
aspect_mask: vk::ImageAspectFlags::COLOR,
|
||||
base_mip_level: 0,
|
||||
level_count: 1,
|
||||
base_array_layer: layer,
|
||||
layer_count: 1,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) unsafe fn find_mem(
|
||||
mp: &vk::PhysicalDeviceMemoryProperties,
|
||||
bits: u32,
|
||||
want: vk::MemoryPropertyFlags,
|
||||
) -> u32 {
|
||||
for i in 0..mp.memory_type_count {
|
||||
if (bits & (1 << i)) != 0 && mp.memory_types[i as usize].property_flags.contains(want) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
0
|
||||
}
|
||||
|
||||
/// DRM fourcc -> the VkFormat whose *color* components match (Vulkan handles the byte swizzle).
|
||||
pub(crate) fn fourcc_to_vk(fourcc: u32) -> Option<vk::Format> {
|
||||
// fourcc_code(a,b,c,d) = a | b<<8 | c<<16 | d<<24
|
||||
const XR24: u32 = 0x3432_5258; // XRGB8888
|
||||
const AR24: u32 = 0x3432_5241; // ARGB8888
|
||||
const XB24: u32 = 0x3432_4258; // XBGR8888
|
||||
const AB24: u32 = 0x3432_4241; // ABGR8888
|
||||
match fourcc {
|
||||
XR24 | AR24 => Some(vk::Format::B8G8R8A8_UNORM),
|
||||
XB24 | AB24 => Some(vk::Format::R8G8B8A8_UNORM),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn pixel_to_vk(fmt: PixelFormat) -> Option<vk::Format> {
|
||||
match fmt {
|
||||
PixelFormat::Bgrx | PixelFormat::Bgra => Some(vk::Format::B8G8R8A8_UNORM),
|
||||
PixelFormat::Rgbx | PixelFormat::Rgba => Some(vk::Format::R8G8B8A8_UNORM),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) unsafe fn make_view(
|
||||
device: &ash::Device,
|
||||
image: vk::Image,
|
||||
fmt: vk::Format,
|
||||
layer: u32,
|
||||
) -> Result<vk::ImageView> {
|
||||
Ok(device.create_image_view(
|
||||
&vk::ImageViewCreateInfo::default()
|
||||
.image(image)
|
||||
.view_type(vk::ImageViewType::TYPE_2D)
|
||||
.format(fmt)
|
||||
.subresource_range(color_range(layer)),
|
||||
None,
|
||||
)?)
|
||||
}
|
||||
|
||||
/// Import a packed-RGB dmabuf as a SAMPLED VkImage (explicit DRM modifier). Caller destroys all
|
||||
/// three returned handles. Extracted verbatim from `vulkan_video.rs`'s import path.
|
||||
pub(crate) unsafe fn import_rgb_dmabuf(
|
||||
device: &ash::Device,
|
||||
ext_fd: &ash::khr::external_memory_fd::Device,
|
||||
mem_props: &vk::PhysicalDeviceMemoryProperties,
|
||||
d: &pf_frame::DmabufFrame,
|
||||
cw: u32,
|
||||
ch: u32,
|
||||
) -> Result<(vk::Image, vk::DeviceMemory, vk::ImageView)> {
|
||||
use anyhow::Context;
|
||||
use std::os::fd::IntoRawFd;
|
||||
let fmt = fourcc_to_vk(d.fourcc)
|
||||
.with_context(|| format!("unsupported dmabuf fourcc {:#x}", d.fourcc))?;
|
||||
let plane = [vk::SubresourceLayout::default()
|
||||
.offset(d.offset as u64)
|
||||
.row_pitch(d.stride as u64)];
|
||||
let mut drm = vk::ImageDrmFormatModifierExplicitCreateInfoEXT::default()
|
||||
.drm_format_modifier(d.modifier)
|
||||
.plane_layouts(&plane);
|
||||
let mut ext = vk::ExternalMemoryImageCreateInfo::default()
|
||||
.handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT);
|
||||
let img = device.create_image(
|
||||
&vk::ImageCreateInfo::default()
|
||||
.image_type(vk::ImageType::TYPE_2D)
|
||||
.format(fmt)
|
||||
.extent(vk::Extent3D {
|
||||
width: cw,
|
||||
height: ch,
|
||||
depth: 1,
|
||||
})
|
||||
.mip_levels(1)
|
||||
.array_layers(1)
|
||||
.samples(vk::SampleCountFlags::TYPE_1)
|
||||
.tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT)
|
||||
.usage(vk::ImageUsageFlags::SAMPLED)
|
||||
.sharing_mode(vk::SharingMode::EXCLUSIVE)
|
||||
.initial_layout(vk::ImageLayout::UNDEFINED)
|
||||
.push_next(&mut ext)
|
||||
.push_next(&mut drm),
|
||||
None,
|
||||
)?;
|
||||
// dup the fd; Vulkan takes ownership of the dup on a successful import.
|
||||
let dup = d.fd.try_clone().context("dup dmabuf fd")?.into_raw_fd();
|
||||
let fd_props = {
|
||||
let mut p = vk::MemoryFdPropertiesKHR::default();
|
||||
let _ = (ext_fd.fp().get_memory_fd_properties_khr)(
|
||||
device.handle(),
|
||||
vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT,
|
||||
dup,
|
||||
&mut p,
|
||||
);
|
||||
p.memory_type_bits
|
||||
};
|
||||
let req = device.get_image_memory_requirements(img);
|
||||
let bits = req.memory_type_bits & fd_props;
|
||||
let ti = find_mem(
|
||||
mem_props,
|
||||
if bits != 0 {
|
||||
bits
|
||||
} else {
|
||||
req.memory_type_bits
|
||||
},
|
||||
vk::MemoryPropertyFlags::empty(),
|
||||
);
|
||||
let mut ded = vk::MemoryDedicatedAllocateInfo::default().image(img);
|
||||
let mut import = vk::ImportMemoryFdInfoKHR::default()
|
||||
.handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT)
|
||||
.fd(dup);
|
||||
let mem = device.allocate_memory(
|
||||
&vk::MemoryAllocateInfo::default()
|
||||
.allocation_size(req.size)
|
||||
.memory_type_index(ti)
|
||||
.push_next(&mut ded)
|
||||
.push_next(&mut import),
|
||||
None,
|
||||
)?;
|
||||
device.bind_image_memory(img, mem, 0)?;
|
||||
let view = device.create_image_view(
|
||||
&vk::ImageViewCreateInfo::default()
|
||||
.image(img)
|
||||
.view_type(vk::ImageViewType::TYPE_2D)
|
||||
.format(fmt)
|
||||
.subresource_range(color_range(0)),
|
||||
None,
|
||||
)?;
|
||||
Ok((img, mem, view))
|
||||
}
|
||||
|
||||
pub(crate) unsafe fn make_plain_image(
|
||||
device: &ash::Device,
|
||||
mp: &vk::PhysicalDeviceMemoryProperties,
|
||||
fmt: vk::Format,
|
||||
w: u32,
|
||||
h: u32,
|
||||
usage: vk::ImageUsageFlags,
|
||||
) -> Result<(vk::Image, vk::DeviceMemory, vk::ImageView)> {
|
||||
let img = device.create_image(
|
||||
&vk::ImageCreateInfo::default()
|
||||
.image_type(vk::ImageType::TYPE_2D)
|
||||
.format(fmt)
|
||||
.extent(vk::Extent3D {
|
||||
width: w,
|
||||
height: h,
|
||||
depth: 1,
|
||||
})
|
||||
.mip_levels(1)
|
||||
.array_layers(1)
|
||||
.samples(vk::SampleCountFlags::TYPE_1)
|
||||
.tiling(vk::ImageTiling::OPTIMAL)
|
||||
.usage(usage)
|
||||
.initial_layout(vk::ImageLayout::UNDEFINED),
|
||||
None,
|
||||
)?;
|
||||
let req = device.get_image_memory_requirements(img);
|
||||
let mem = device.allocate_memory(
|
||||
&vk::MemoryAllocateInfo::default()
|
||||
.allocation_size(req.size)
|
||||
.memory_type_index(find_mem(
|
||||
mp,
|
||||
req.memory_type_bits,
|
||||
vk::MemoryPropertyFlags::DEVICE_LOCAL,
|
||||
)),
|
||||
None,
|
||||
)?;
|
||||
device.bind_image_memory(img, mem, 0)?;
|
||||
let view = make_view(device, img, fmt, 0)?;
|
||||
Ok((img, mem, view))
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,245 @@
|
||||
//! Shared direct-SDK NVENC core — the platform-agnostic pieces of the two `nvEncodeAPI` backends,
|
||||
//! Windows D3D11 (`encode/windows/nvenc.rs`) and Linux CUDA (`encode/linux/nvenc_cuda.rs`), so the
|
||||
//! byte-identical glue lives once (plan §2.2, the direct-NVENC Tier-2). The per-platform parts —
|
||||
//! the entry-table load (`nvEncodeAPI64.dll` via `LoadLibrary` vs `libnvidia-encode.so` via
|
||||
//! `libloading`), the device binding (D3D11 vs CUDA), input-surface registration, and the
|
||||
//! Windows-only async retrieve — stay in their backends. Sibling of [`super::nvenc_status`].
|
||||
|
||||
use super::Codec;
|
||||
use nvidia_video_codec_sdk::sys::nvEncodeAPI as nv;
|
||||
|
||||
/// Local `NVENCSTATUS` → `Result` (replaces the sdk's `result_without_string`, which lives in the
|
||||
/// crate's `safe` module — code these backends must not pull in). The raw status's Debug repr
|
||||
/// (`NV_ENC_ERR_INVALID_PARAM`, …) is the error payload; callers fold it through
|
||||
/// [`super::nvenc_status`] for an operator-actionable cause.
|
||||
pub(super) trait NvStatusExt {
|
||||
fn nv_ok(self) -> std::result::Result<(), nv::NVENCSTATUS>;
|
||||
}
|
||||
impl NvStatusExt for nv::NVENCSTATUS {
|
||||
fn nv_ok(self) -> std::result::Result<(), nv::NVENCSTATUS> {
|
||||
match self {
|
||||
nv::NVENCSTATUS::NV_ENC_SUCCESS => Ok(()),
|
||||
err => Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The NVENC codec GUID for a session [`Codec`]. PyroWave never opens the direct-NVENC backend
|
||||
/// (guarded by the `open_video` dispatch), so it is unreachable here.
|
||||
pub(super) fn codec_guid(codec: Codec) -> nv::GUID {
|
||||
match codec {
|
||||
Codec::H264 => nv::NV_ENC_CODEC_H264_GUID,
|
||||
Codec::H265 => nv::NV_ENC_CODEC_HEVC_GUID,
|
||||
Codec::Av1 => nv::NV_ENC_CODEC_AV1_GUID,
|
||||
Codec::PyroWave => unreachable!("PyroWave never opens the direct-NVENC backend"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reference-frame DPB depth when RFI is supported (Apollo uses 5). A deeper DPB lets an invalidated
|
||||
/// reference fall back to an older still-valid frame instead of a full IDR; `numRefL0 = 1` keeps each
|
||||
/// P-frame single-reference for low latency. Also the window the backends' `invalidate_ref_frames`
|
||||
/// paths check against (`frame_idx - RFI_DPB` = the oldest frame still in the DPB).
|
||||
pub(super) const RFI_DPB: u32 = 5;
|
||||
|
||||
/// The per-session knobs both direct-NVENC backends feed [`apply_low_latency_config`]. `Copy` so the
|
||||
/// backend fills it from `self` at the call. The two input-format fields bridge the only real
|
||||
/// divergence between the CUDA and D3D11 paths (which surface formats can carry full chroma / 10-bit
|
||||
/// input); everything else in the config is identical across platforms.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct LowLatencyConfig {
|
||||
pub codec: Codec,
|
||||
/// Target bitrate (bps); CBR average == max.
|
||||
pub bitrate: u64,
|
||||
pub fps: u32,
|
||||
/// This GPU advertises custom VBV — else leave the preset default (per the caps probe).
|
||||
pub custom_vbv: bool,
|
||||
/// A 4:4:4 session was negotiated (HEVC Range Extensions).
|
||||
pub chroma_444: bool,
|
||||
/// The input surface can carry full chroma — Linux feeds a YUV444 surface, Windows a packed-RGB
|
||||
/// surface NVENC CSCs internally. 4:4:4 engages only when this AND [`chroma_444`](Self::chroma_444).
|
||||
pub full_chroma_input: bool,
|
||||
/// Output bit depth (8 or 10).
|
||||
pub bit_depth: u8,
|
||||
/// AV1 `inputPixelBitDepthMinus8` — the encoder's view of the INPUT depth (Linux is 8-bit in
|
||||
/// today, so 0; Windows derives it from the surface format). `u32` to match the SDK setter.
|
||||
pub av1_input_depth_minus8: u32,
|
||||
pub hdr: bool,
|
||||
/// This GPU supports reference-frame invalidation (a deeper DPB for graceful loss recovery).
|
||||
pub rfi_supported: bool,
|
||||
}
|
||||
|
||||
/// Author the shared `NV_ENC_INITIALIZE_PARAMS` (P1/ULL preset, PTD, the session dimensions/rate)
|
||||
/// pointing at `cfg`. `enable_async` drives the Windows two-thread async retrieve — Linux is
|
||||
/// sync-only and passes `false`, leaving `enableEncodeAsync` at 0 as before. The returned struct
|
||||
/// borrows `cfg` as a raw pointer; the caller must keep `cfg` alive across the NVENC call it feeds
|
||||
/// this into. Used at open and at in-place reconfigure, which must present the SAME init params.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) fn build_init_params(
|
||||
codec_guid: nv::GUID,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
cfg: &mut nv::NV_ENC_CONFIG,
|
||||
split_mode: u32,
|
||||
enable_async: bool,
|
||||
) -> nv::NV_ENC_INITIALIZE_PARAMS {
|
||||
let mut init = nv::NV_ENC_INITIALIZE_PARAMS {
|
||||
version: nv::NV_ENC_INITIALIZE_PARAMS_VER,
|
||||
encodeGUID: codec_guid,
|
||||
presetGUID: nv::NV_ENC_PRESET_P1_GUID,
|
||||
tuningInfo: nv::NV_ENC_TUNING_INFO::NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY,
|
||||
encodeWidth: width,
|
||||
encodeHeight: height,
|
||||
darWidth: width,
|
||||
darHeight: height,
|
||||
frameRateNum: fps,
|
||||
frameRateDen: 1,
|
||||
enablePTD: 1,
|
||||
enableEncodeAsync: enable_async as u32,
|
||||
encodeConfig: cfg,
|
||||
..Default::default()
|
||||
};
|
||||
// splitEncodeMode is a C bitfield — set via the generated accessor, not a struct field.
|
||||
init.set_splitEncodeMode(split_mode);
|
||||
init
|
||||
}
|
||||
|
||||
/// Author the shared low-latency NVENC config onto a **preset-seeded** `cfg`: CBR + infinite GOP +
|
||||
/// P-only + ~1-frame VBV, per-codec tier/level, chroma + bit depth, unconditional colour signaling,
|
||||
/// and the RFI DPB. The caller seeds `cfg` from the P1/ULL preset first (that call needs the
|
||||
/// per-platform entry table) and passes the input-format specifics via [`LowLatencyConfig`]; the
|
||||
/// per-platform surface registration, device binding, and async retrieve stay in the backends.
|
||||
///
|
||||
/// # Safety
|
||||
/// Writes NVENC codec-config union fields on `cfg`, which must be a valid, preset-seeded
|
||||
/// `NV_ENC_CONFIG` whose active union arm matches [`LowLatencyConfig::codec`].
|
||||
pub(super) unsafe fn apply_low_latency_config(cfg: &mut nv::NV_ENC_CONFIG, c: LowLatencyConfig) {
|
||||
// CBR, infinite GOP, P-only, ~1-frame VBV — mirrors the AMF/VAAPI/QSV libav RC config.
|
||||
cfg.gopLength = nv::NVENC_INFINITE_GOPLENGTH;
|
||||
cfg.frameIntervalP = 1;
|
||||
cfg.rcParams.rateControlMode = nv::NV_ENC_PARAMS_RC_MODE::NV_ENC_PARAMS_RC_CBR;
|
||||
let bps = c.bitrate.min(u32::MAX as u64) as u32;
|
||||
cfg.rcParams.averageBitRate = bps;
|
||||
cfg.rcParams.maxBitRate = bps;
|
||||
// Shrink the VBV with the bitrate (NVENC validates it against the same level ceiling), but only
|
||||
// when the GPU advertises custom-VBV support — else keep the preset default.
|
||||
if c.custom_vbv {
|
||||
// ~1-frame VBV by default; PUNKTFUNK_VBV_FRAMES scales it (parity with AMF/VAAPI/QSV).
|
||||
let vbv = ((c.bitrate as f64 / c.fps.max(1) as f64) * crate::vbv_frames_env())
|
||||
.clamp(1.0, u32::MAX as f64) as u32;
|
||||
cfg.rcParams.vbvBufferSize = vbv;
|
||||
cfg.rcParams.vbvInitialDelay = vbv;
|
||||
}
|
||||
|
||||
// Tier + autoselect level, PER CODEC (the union writes must match the negotiated codec). HEVC
|
||||
// keeps HIGH tier for its higher per-level bitrate ceiling (Main at 5K ≈ 240 Mbps, HIGH ≈ 800
|
||||
// Mbps); AV1 supports the Main tier ONLY — tier=1 fails the session open with INVALID_PARAM, and
|
||||
// its level enum's 0 is LEVEL 2.0 (NOT autoselect), so AV1 takes NO writes (the preset defaults
|
||||
// are the only accepted config). H.264 has no tier. Level 0 = autoselect for HEVC.
|
||||
match c.codec {
|
||||
Codec::H265 => {
|
||||
cfg.encodeCodecConfig.hevcConfig.tier = 1;
|
||||
cfg.encodeCodecConfig.hevcConfig.level = 0;
|
||||
}
|
||||
Codec::Av1 => {}
|
||||
Codec::H264 => {}
|
||||
Codec::PyroWave => unreachable!("PyroWave never opens the direct-NVENC backend"),
|
||||
}
|
||||
|
||||
// Chroma + bit depth. Full-chroma 4:4:4 (HEVC Range Extensions, chromaFormatIDC=3 under the FREXT
|
||||
// profile) takes precedence and composes with 10-bit (Main 4:4:4 10); it needs a full-chroma-
|
||||
// capable input. Otherwise 10-bit selects Main10 (HEVC) or the AV1 output depth — stamping the
|
||||
// HEVC Main10 GUID onto an AV1 session is an INVALID_PARAM, so bit depth is set PER CODEC.
|
||||
if c.chroma_444 && c.full_chroma_input {
|
||||
cfg.profileGUID = nv::NV_ENC_HEVC_PROFILE_FREXT_GUID;
|
||||
cfg.encodeCodecConfig.hevcConfig.set_chromaFormatIDC(3);
|
||||
if c.bit_depth == 10 {
|
||||
cfg.encodeCodecConfig.hevcConfig.set_pixelBitDepthMinus8(2); // Main 4:4:4 10
|
||||
}
|
||||
} else if c.bit_depth == 10 {
|
||||
match c.codec {
|
||||
Codec::H265 => {
|
||||
cfg.profileGUID = nv::NV_ENC_HEVC_PROFILE_MAIN10_GUID;
|
||||
cfg.encodeCodecConfig.hevcConfig.set_pixelBitDepthMinus8(2);
|
||||
}
|
||||
Codec::Av1 => {
|
||||
cfg.encodeCodecConfig.av1Config.set_pixelBitDepthMinus8(2);
|
||||
cfg.encodeCodecConfig
|
||||
.av1Config
|
||||
.set_inputPixelBitDepthMinus8(c.av1_input_depth_minus8);
|
||||
}
|
||||
Codec::H264 => {} // no 10-bit H.264 encode on NVENC — negotiation never asks
|
||||
Codec::PyroWave => unreachable!("PyroWave never opens the direct-NVENC backend"),
|
||||
}
|
||||
}
|
||||
|
||||
// Colour signaling, written UNCONDITIONALLY: the input is already CSC'd to a specific matrix
|
||||
// (BT.709 limited SDR, or BT.2020 PQ for HDR), so the stream must say so — a decoder whose
|
||||
// "unspecified" default is 601 (Moonlight/third-party/Android-vendor at sub-HD) otherwise
|
||||
// mis-renders. HEVC/H.264 carry it in the VUI; AV1 has no VUI, so the same CICP code points go in
|
||||
// the sequence-header colour config.
|
||||
{
|
||||
let (prim, trc, mat) = if c.hdr {
|
||||
(
|
||||
nv::NV_ENC_VUI_COLOR_PRIMARIES::NV_ENC_VUI_COLOR_PRIMARIES_BT2020,
|
||||
nv::NV_ENC_VUI_TRANSFER_CHARACTERISTIC::NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE2084,
|
||||
nv::NV_ENC_VUI_MATRIX_COEFFS::NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL,
|
||||
)
|
||||
} else {
|
||||
(
|
||||
nv::NV_ENC_VUI_COLOR_PRIMARIES::NV_ENC_VUI_COLOR_PRIMARIES_BT709,
|
||||
nv::NV_ENC_VUI_TRANSFER_CHARACTERISTIC::NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709,
|
||||
nv::NV_ENC_VUI_MATRIX_COEFFS::NV_ENC_VUI_MATRIX_COEFFS_BT709,
|
||||
)
|
||||
};
|
||||
match c.codec {
|
||||
Codec::H265 => {
|
||||
let vui = &mut cfg.encodeCodecConfig.hevcConfig.hevcVUIParameters;
|
||||
vui.videoSignalTypePresentFlag = 1;
|
||||
vui.videoFullRangeFlag = 0;
|
||||
vui.colourDescriptionPresentFlag = 1;
|
||||
vui.colourPrimaries = prim;
|
||||
vui.transferCharacteristics = trc;
|
||||
vui.colourMatrix = mat;
|
||||
}
|
||||
Codec::H264 => {
|
||||
let vui = &mut cfg.encodeCodecConfig.h264Config.h264VUIParameters;
|
||||
vui.videoSignalTypePresentFlag = 1;
|
||||
vui.videoFullRangeFlag = 0;
|
||||
vui.colourDescriptionPresentFlag = 1;
|
||||
vui.colourPrimaries = prim;
|
||||
vui.transferCharacteristics = trc;
|
||||
vui.colourMatrix = mat;
|
||||
}
|
||||
Codec::Av1 => {
|
||||
let av1 = &mut cfg.encodeCodecConfig.av1Config;
|
||||
av1.colorPrimaries = prim;
|
||||
av1.transferCharacteristics = trc;
|
||||
av1.matrixCoefficients = mat;
|
||||
av1.colorRange = 0; // studio/limited swing
|
||||
}
|
||||
Codec::PyroWave => unreachable!("PyroWave never opens the direct-NVENC backend"),
|
||||
}
|
||||
}
|
||||
|
||||
// Reference-frame invalidation: a deeper DPB so an invalidated reference can fall back to an
|
||||
// older still-valid frame instead of a full IDR; `numRefL0 = 1` keeps each P-frame
|
||||
// single-reference for low latency. Only when this GPU supports RFI.
|
||||
if c.rfi_supported {
|
||||
let one = nv::NV_ENC_NUM_REF_FRAMES::NV_ENC_NUM_REF_FRAMES_1;
|
||||
match c.codec {
|
||||
Codec::H264 => {
|
||||
cfg.encodeCodecConfig.h264Config.maxNumRefFrames = RFI_DPB;
|
||||
cfg.encodeCodecConfig.h264Config.numRefL0 = one;
|
||||
}
|
||||
Codec::H265 => {
|
||||
cfg.encodeCodecConfig.hevcConfig.maxNumRefFramesInDPB = RFI_DPB;
|
||||
cfg.encodeCodecConfig.hevcConfig.numRefL0 = one;
|
||||
}
|
||||
Codec::Av1 => {
|
||||
cfg.encodeCodecConfig.av1Config.maxNumRefFramesInDPB = RFI_DPB;
|
||||
}
|
||||
Codec::PyroWave => unreachable!("PyroWave never opens the direct-NVENC backend"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
//! Actionable explanations for `NVENCSTATUS` failures — shared by the direct-SDK NVENC backends on
|
||||
//! Windows (`encode/windows/nvenc.rs`) and Linux (`encode/linux/nvenc_cuda.rs`).
|
||||
//!
|
||||
//! Every NVENC entry-point failure used to be annotated `(no NVIDIA GPU?)`, which actively misled
|
||||
//! triage: the direct-NVENC path only loads on a machine that HAS an NVIDIA GPU, and the failure a
|
||||
//! user actually hit — `NV_ENC_ERR_INVALID_VERSION` from a userspace/kernel driver version skew,
|
||||
//! fixed by a reboot — has nothing to do with a missing GPU. This maps each status to what it really
|
||||
//! means and what the operator should do, and folds that cause into the `anyhow::Error` at
|
||||
//! construction, so every downstream `{e:#}` log (the encode-recovery loop, session teardown) says
|
||||
//! the useful thing without extra plumbing.
|
||||
|
||||
use nvidia_video_codec_sdk::sys::nvEncodeAPI as nv;
|
||||
|
||||
/// A one-line, operator-actionable cause for an NVENC status. Does not repeat the raw code —
|
||||
/// callers print that alongside (see [`call_err`]). Public for the few sites that build a
|
||||
/// `String`/`format!` error instead of an `anyhow::Error`.
|
||||
pub(super) fn explain(status: nv::NVENCSTATUS) -> String {
|
||||
match status {
|
||||
// The one this whole module exists for: a version word the driver rejects. Either the
|
||||
// driver is genuinely older than our headers, or (the sneaky case) the userspace
|
||||
// `libnvidia-encode` reports a new-enough version to the pre-flight probe but the running
|
||||
// kernel module is older and rejects the session — the classic "updated the driver, didn't
|
||||
// reboot" skew. Both heal the same way.
|
||||
nv::NVENCSTATUS::NV_ENC_ERR_INVALID_VERSION => format!(
|
||||
"the NVIDIA driver is older than this build's NVENC headers (needs NVENC API {}.{} or \
|
||||
newer), or the userspace and kernel-module driver versions are mismatched — common \
|
||||
right after a driver update without a reboot. Update the NVIDIA driver, or reboot if \
|
||||
you just updated it (a host restart is the usual fix).",
|
||||
nv::NVENCAPI_MAJOR_VERSION,
|
||||
nv::NVENCAPI_MINOR_VERSION,
|
||||
),
|
||||
nv::NVENCSTATUS::NV_ENC_ERR_NO_ENCODE_DEVICE => {
|
||||
"this GPU exposes no usable NVENC engine — it has no hardware video encoder, or NVENC is \
|
||||
disabled on this card"
|
||||
.to_string()
|
||||
}
|
||||
nv::NVENCSTATUS::NV_ENC_ERR_UNSUPPORTED_DEVICE => {
|
||||
"this GPU model is not supported by NVENC".to_string()
|
||||
}
|
||||
nv::NVENCSTATUS::NV_ENC_ERR_INVALID_ENCODERDEVICE
|
||||
| nv::NVENCSTATUS::NV_ENC_ERR_INVALID_DEVICE => {
|
||||
"the device/context handed to NVENC is invalid — a GPU reset or driver reload can cause \
|
||||
this"
|
||||
.to_string()
|
||||
}
|
||||
nv::NVENCSTATUS::NV_ENC_ERR_DEVICE_NOT_EXIST => {
|
||||
"the NVENC device no longer exists — the driver reset, or the GPU fell off the bus"
|
||||
.to_string()
|
||||
}
|
||||
nv::NVENCSTATUS::NV_ENC_ERR_OUT_OF_MEMORY => "the GPU is out of memory".to_string(),
|
||||
nv::NVENCSTATUS::NV_ENC_ERR_INCOMPATIBLE_CLIENT_KEY => {
|
||||
"NVENC rejected the client key — the GeForce concurrent-NVENC-session limit was reached, \
|
||||
or the driver is unlicensed for this many encoders"
|
||||
.to_string()
|
||||
}
|
||||
nv::NVENCSTATUS::NV_ENC_ERR_UNIMPLEMENTED
|
||||
| nv::NVENCSTATUS::NV_ENC_ERR_UNSUPPORTED_PARAM => {
|
||||
"this driver/GPU does not implement the requested NVENC encode mode".to_string()
|
||||
}
|
||||
nv::NVENCSTATUS::NV_ENC_ERR_INVALID_PARAM => {
|
||||
"NVENC rejected a parameter — an encode mode this GPU does not support".to_string()
|
||||
}
|
||||
nv::NVENCSTATUS::NV_ENC_ERR_ENCODER_BUSY => {
|
||||
"the NVENC engine is busy — retry, or reduce the number of concurrent encode sessions"
|
||||
.to_string()
|
||||
}
|
||||
nv::NVENCSTATUS::NV_ENC_ERR_GENERIC => {
|
||||
"the NVIDIA driver returned a generic NVENC failure — check dmesg and the driver install"
|
||||
.to_string()
|
||||
}
|
||||
other => format!("unexpected NVENC status ({other:?})"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Build an actionable `anyhow::Error` for a failed NVENC entry-point call. `call` names the API
|
||||
/// (e.g. `"open_encode_session_ex"`); the message carries both the raw status and its real-world
|
||||
/// cause, so triage never again reads a version mismatch as "(no NVIDIA GPU?)".
|
||||
pub(super) fn call_err(call: &str, status: nv::NVENCSTATUS) -> anyhow::Error {
|
||||
anyhow::anyhow!("NVENC {call} failed: {status:?} — {}", explain(status))
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
//! Software H.264 encoder (openh264) — the GPU-less encode path for the Windows host (and a
|
||||
//! fallback when NVENC is unavailable). Low-latency screen-content config: single-reference,
|
||||
//! no B-frames (Baseline), bitrate rate-control, in-band SPS/PPS each IDR.
|
||||
//! Synchronous: `submit` encodes immediately and stashes the AU for `poll` (no internal queue).
|
||||
//!
|
||||
//! The RGB→YUV conversion is OURS, BT.709 limited range: openh264 writes no colour description
|
||||
//! into the VUI (unspecified), so decoders fall back to their default — BT.709 limited on every
|
||||
//! punktfunk client — and the pixels must match that default. The crate's own `YUVBuffer`
|
||||
//! converter is BT.601 (0.2578/0.5039/0.0977 + 16), which decoded-as-709 is a constant hue
|
||||
//! error; that's why it is NOT used here.
|
||||
// Every `unsafe` block in this file carries a `// SAFETY:` proof; enforce it (unsafe-proof program).
|
||||
#![deny(clippy::undocumented_unsafe_blocks)]
|
||||
|
||||
use super::{EncodedFrame, Encoder};
|
||||
use anyhow::{bail, ensure, Context, Result};
|
||||
use openh264::encoder::{
|
||||
BitRate, Complexity, Encoder as Oh264, EncoderConfig, FrameRate, FrameType, IntraFramePeriod,
|
||||
Profile, RateControlMode, SpsPpsStrategy, UsageType,
|
||||
};
|
||||
use openh264::formats::YUVSlices;
|
||||
use openh264::OpenH264API;
|
||||
use pf_frame::{CapturedFrame, FramePayload, PixelFormat};
|
||||
use std::collections::VecDeque;
|
||||
|
||||
pub struct OpenH264Encoder {
|
||||
enc: Oh264,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
src_format: PixelFormat,
|
||||
/// The converted I420 planes (our BT.709-limited CSC — see the module doc), reused across
|
||||
/// frames: full-res luma + quarter-res Cb/Cr, tightly packed (stride = width, width/2).
|
||||
y_plane: Vec<u8>,
|
||||
u_plane: Vec<u8>,
|
||||
v_plane: Vec<u8>,
|
||||
frame_idx: i64,
|
||||
force_kf: bool,
|
||||
/// One AU per submit (no lookahead), handed back FIFO by `poll`. A queue, not an `Option`:
|
||||
/// the session loop pipelines up to `capturer.pipeline_depth()` submits before polling, and a
|
||||
/// single-slot pending would silently overwrite (lose) the older AUs — including the opening
|
||||
/// IDR — and permanently skew the loop's FIFO pts pairing.
|
||||
pending: VecDeque<EncodedFrame>,
|
||||
}
|
||||
|
||||
// openh264's Encoder holds a raw C handle (not auto-Send); it lives on the single encode thread.
|
||||
// SAFETY: `OpenH264Encoder` wraps `Oh264` (openh264's `Encoder`), which holds a raw C handle to the
|
||||
// openh264 `ISVCEncoder` and is not auto-`Send`; the other fields (the plane `Vec`s, scalars,
|
||||
// `Option<EncodedFrame>`) are plain owned data. The session creates the encoder, calls
|
||||
// `submit`/`poll`/`flush`, and drops it all on one dedicated encode thread, never sharing it by
|
||||
// reference across threads, so the C handle is only ever touched from a single thread. Moving the
|
||||
// whole value to that thread is therefore sound — there is no concurrent access to the handle.
|
||||
unsafe impl Send for OpenH264Encoder {}
|
||||
|
||||
impl OpenH264Encoder {
|
||||
pub fn open(
|
||||
format: PixelFormat,
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: u32,
|
||||
bitrate_bps: u64,
|
||||
) -> Result<Self> {
|
||||
// validate_dimensions() ran in open_video: even, non-zero, <= 4096.
|
||||
let bps: u32 = bitrate_bps.try_into().unwrap_or(u32::MAX);
|
||||
let cfg = EncoderConfig::new()
|
||||
.usage_type(UsageType::ScreenContentRealTime)
|
||||
.max_frame_rate(FrameRate::from_hz(fps.max(1) as f32))
|
||||
.rate_control_mode(RateControlMode::Bitrate)
|
||||
.bitrate(BitRate::from_bps(bps))
|
||||
.skip_frames(false)
|
||||
.intra_frame_period(IntraFramePeriod::from_num_frames(intra_period_frames(fps)))
|
||||
.sps_pps_strategy(SpsPpsStrategy::ConstantId) // SPS/PPS in-band on every IDR
|
||||
.num_threads(num_threads())
|
||||
.scene_change_detect(false) // no surprise IDRs (bitrate spikes / freeze)
|
||||
.adaptive_quantization(true)
|
||||
.complexity(Complexity::Low) // latency over BD-rate
|
||||
.profile(Profile::Baseline); // no B-frames; the VUI carries no colour description
|
||||
let api = OpenH264API::from_source(); // statically-bundled build (default `source` feature)
|
||||
let enc = Oh264::with_api_config(api, cfg).context("openh264 Encoder::with_api_config")?;
|
||||
let (w, h) = (width as usize, height as usize);
|
||||
tracing::info!(
|
||||
"openh264 software encoder: {width}x{height}@{fps} {} Mbps (Baseline, screen-content)",
|
||||
bps / 1_000_000
|
||||
);
|
||||
Ok(Self {
|
||||
enc,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
src_format: format,
|
||||
y_plane: vec![0; w * h],
|
||||
u_plane: vec![0; (w / 2) * (h / 2)],
|
||||
v_plane: vec![0; (w / 2) * (h / 2)],
|
||||
frame_idx: 0,
|
||||
force_kf: false,
|
||||
pending: VecDeque::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert one packed full-range RGB frame into the I420 planes, BT.709 limited range.
|
||||
/// `bpp` is the source pixel stride; `ri`/`gi`/`bi` the channel byte offsets within a pixel.
|
||||
/// Luma per pixel; Cb/Cr from the 2×2 block's averaged RGB (the same box filter the crate's
|
||||
/// converter used, so only the matrix changed).
|
||||
fn convert_bt709(&mut self, src: &[u8], bpp: usize, ri: usize, gi: usize, bi: usize) {
|
||||
let w = self.width as usize;
|
||||
let h = self.height as usize;
|
||||
let cw = w / 2;
|
||||
for by in 0..h / 2 {
|
||||
for bx in 0..cw {
|
||||
let mut sum = (0f32, 0f32, 0f32);
|
||||
for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {
|
||||
let (px, py) = (bx * 2 + dx, by * 2 + dy);
|
||||
let s = &src[(py * w + px) * bpp..];
|
||||
let (r, g, b) = (f32::from(s[ri]), f32::from(s[gi]), f32::from(s[bi]));
|
||||
self.y_plane[py * w + px] = luma709(r, g, b);
|
||||
sum = (sum.0 + r, sum.1 + g, sum.2 + b);
|
||||
}
|
||||
let (cb, cr) = chroma709(sum.0 / 4.0, sum.1 / 4.0, sum.2 / 4.0);
|
||||
self.u_plane[by * cw + bx] = cb;
|
||||
self.v_plane[by * cw + bx] = cr;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// BT.709 luma coefficients (Kg = 1 − Kr − Kb).
|
||||
const KR: f32 = 0.2126;
|
||||
const KB: f32 = 0.0722;
|
||||
const KG: f32 = 1.0 - KR - KB;
|
||||
|
||||
/// One full-range RGB pixel (0..=255 channels) → the BT.709 limited-range 8-bit luma code
|
||||
/// (16..=235). Kept in lockstep with the client-side inverse (`pf-client-core::video::csc_rows`).
|
||||
fn luma709(r: f32, g: f32, b: f32) -> u8 {
|
||||
let y = KR * r + KG * g + KB * b; // full-scale luma, 0..=255
|
||||
(16.0 + y * (219.0 / 255.0) + 0.5) as u8 // `as` saturates — no manual clamp needed
|
||||
}
|
||||
|
||||
/// (Averaged) full-range RGB → the BT.709 limited-range Cb/Cr codes (16..=240, neutral 128).
|
||||
fn chroma709(r: f32, g: f32, b: f32) -> (u8, u8) {
|
||||
let y = KR * r + KG * g + KB * b;
|
||||
let cb = 128.0 + (b - y) * (224.0 / 255.0) / (2.0 * (1.0 - KB));
|
||||
let cr = 128.0 + (r - y) * (224.0 / 255.0) / (2.0 * (1.0 - KR));
|
||||
((cb + 0.5) as u8, (cr + 0.5) as u8)
|
||||
}
|
||||
|
||||
impl Encoder for OpenH264Encoder {
|
||||
fn submit(&mut self, captured: &CapturedFrame) -> Result<()> {
|
||||
ensure!(
|
||||
captured.width == self.width && captured.height == self.height,
|
||||
"captured {}x{} != encoder {}x{}",
|
||||
captured.width,
|
||||
captured.height,
|
||||
self.width,
|
||||
self.height
|
||||
);
|
||||
ensure!(
|
||||
captured.format == self.src_format,
|
||||
"captured format {:?} != encoder source {:?}",
|
||||
captured.format,
|
||||
self.src_format
|
||||
);
|
||||
// Refutable once the capture backend adds `FramePayload::D3d11`; today `Cpu` is the only
|
||||
// non-Linux variant, so the pattern is (temporarily) irrefutable.
|
||||
#[allow(irrefutable_let_patterns)]
|
||||
let FramePayload::Cpu(bytes) = &captured.payload
|
||||
else {
|
||||
bail!("openh264 backend requires a CPU frame payload");
|
||||
};
|
||||
let w = self.width as usize;
|
||||
let h = self.height as usize;
|
||||
ensure!(
|
||||
bytes.len() >= w * h * self.src_format.bytes_per_pixel(),
|
||||
"captured buffer {} bytes too small for {w}x{h} {:?}",
|
||||
bytes.len(),
|
||||
self.src_format
|
||||
);
|
||||
|
||||
// Source pixel stride + R/G/B byte offsets within a pixel — one converter for every
|
||||
// packed-RGB layout the capturers emit (no BGRA normalization pass needed).
|
||||
let (bpp, ri, gi, bi) = match self.src_format {
|
||||
PixelFormat::Rgb => (3, 0, 1, 2),
|
||||
PixelFormat::Bgr => (3, 2, 1, 0),
|
||||
PixelFormat::Rgba | PixelFormat::Rgbx => (4, 0, 1, 2),
|
||||
PixelFormat::Bgra | PixelFormat::Bgrx => (4, 2, 1, 0),
|
||||
// 10-bit HDR comes only from the GPU NVENC path; the software 8-bit H.264 encoder
|
||||
// can't represent it (and never receives it — the capturer pairs Rgb10a2 with NVENC).
|
||||
PixelFormat::Rgb10a2 => {
|
||||
anyhow::bail!("software H.264 encoder cannot encode 10-bit HDR (Rgb10a2)")
|
||||
}
|
||||
// NV12/P010 are GPU-resident video-processor outputs for the NVENC path; the software
|
||||
// encoder never receives them (it only gets CPU RGB frames).
|
||||
PixelFormat::Nv12 | PixelFormat::P010 | PixelFormat::Yuv444 => {
|
||||
anyhow::bail!(
|
||||
"software encoder cannot encode YUV GPU frames (NV12/P010/YUV444 → NVENC only)"
|
||||
)
|
||||
}
|
||||
};
|
||||
self.convert_bt709(bytes, bpp, ri, gi, bi);
|
||||
|
||||
if self.force_kf {
|
||||
self.enc.force_intra_frame();
|
||||
self.force_kf = false;
|
||||
}
|
||||
let slices = YUVSlices::new(
|
||||
(&self.y_plane, &self.u_plane, &self.v_plane),
|
||||
(w, h),
|
||||
(w, w / 2, w / 2),
|
||||
);
|
||||
let bs = self.enc.encode(&slices).context("openh264 encode")?;
|
||||
let mut data = Vec::new();
|
||||
bs.write_vec(&mut data); // AnnexB start codes; SPS/PPS prepended on IDR
|
||||
if !data.is_empty() {
|
||||
let keyframe = matches!(bs.frame_type(), FrameType::IDR | FrameType::I);
|
||||
let pts_ns = self.frame_idx as u64 * 1_000_000_000 / self.fps.max(1) as u64;
|
||||
self.pending.push_back(EncodedFrame {
|
||||
data,
|
||||
pts_ns,
|
||||
keyframe,
|
||||
recovery_anchor: false,
|
||||
chunk_aligned: false,
|
||||
});
|
||||
}
|
||||
self.frame_idx += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn request_keyframe(&mut self) {
|
||||
self.force_kf = true;
|
||||
}
|
||||
|
||||
fn poll(&mut self) -> Result<Option<EncodedFrame>> {
|
||||
Ok(self.pending.pop_front())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Result<()> {
|
||||
Ok(()) // synchronous: nothing buffered
|
||||
}
|
||||
}
|
||||
|
||||
/// Approximate infinite-GOP: insert IDRs rarely (recovery is via `request_keyframe`/RFI). Env
|
||||
/// `PUNKTFUNK_OH264_GOP` overrides (0 = encoder-auto).
|
||||
fn intra_period_frames(fps: u32) -> u32 {
|
||||
if let Ok(v) = std::env::var("PUNKTFUNK_OH264_GOP") {
|
||||
if let Ok(n) = v.trim().parse::<u32>() {
|
||||
return n;
|
||||
}
|
||||
}
|
||||
fps.max(1).saturating_mul(600) // ~10 min between automatic IDRs
|
||||
}
|
||||
|
||||
/// Encode threads. Env `PUNKTFUNK_OH264_THREADS` overrides; default 2 (latency over throughput).
|
||||
fn num_threads() -> u16 {
|
||||
std::env::var("PUNKTFUNK_OH264_THREADS")
|
||||
.ok()
|
||||
.and_then(|v| v.trim().parse::<u16>().ok())
|
||||
.unwrap_or(2)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use pf_frame::{CapturedFrame, FramePayload, PixelFormat};
|
||||
|
||||
/// The BT.709 limited-range anchor points: reference white → (235,128,128), black →
|
||||
/// (16,128,128), pure red's Cr must hit the positive extreme 240 (it does exactly:
|
||||
/// 255(1−Kr)·(224/255)/(2(1−Kr)) = 112). ±1 code for float rounding.
|
||||
#[test]
|
||||
fn bt709_conversion_anchor_points() {
|
||||
assert_eq!(luma709(255.0, 255.0, 255.0), 235);
|
||||
assert_eq!(luma709(0.0, 0.0, 0.0), 16);
|
||||
assert_eq!(chroma709(255.0, 255.0, 255.0), (128, 128));
|
||||
assert_eq!(chroma709(0.0, 0.0, 0.0), (128, 128));
|
||||
let (cb, cr) = chroma709(255.0, 0.0, 0.0);
|
||||
assert_eq!(cr, 240, "pure red must reach the Cr extreme");
|
||||
assert!((101..=103).contains(&cb), "red Cb ~102, got {cb}");
|
||||
let (cb, _) = chroma709(0.0, 0.0, 255.0);
|
||||
assert_eq!(cb, 240, "pure blue must reach the Cb extreme");
|
||||
}
|
||||
|
||||
/// The 601-vs-709 luma split on pure green (Kg 0.587 vs 0.7152) — guards against anyone
|
||||
/// "simplifying" the coefficients back to the crate's BT.601 converter (the hue-shift bug
|
||||
/// this module's own conversion exists to prevent).
|
||||
#[test]
|
||||
fn bt709_is_not_bt601() {
|
||||
// BT.601 green luma: 16 + 219·0.587 = 144.5; BT.709: 16 + 219·0.7152 = 172.6.
|
||||
let y = luma709(0.0, 255.0, 0.0);
|
||||
assert!((172..=174).contains(&y), "709 green luma ~173, got {y}");
|
||||
}
|
||||
|
||||
/// A flat gray frame converts to neutral chroma and mid luma across every plane byte
|
||||
/// (exercises the block loop + plane sizing, not just the per-pixel math).
|
||||
#[test]
|
||||
fn converts_flat_gray_to_neutral_planes() {
|
||||
let (w, h) = (16u32, 8u32);
|
||||
let mut enc =
|
||||
OpenH264Encoder::open(PixelFormat::Bgrx, w, h, 60, 1_000_000).expect("open openh264");
|
||||
let bytes = vec![0x80u8; (w * h * 4) as usize];
|
||||
enc.convert_bt709(&bytes, 4, 2, 1, 0);
|
||||
// 16 + 128·(219/255) = 125.9 → 126.
|
||||
assert!(
|
||||
enc.y_plane.iter().all(|&y| y == 126),
|
||||
"{:?}",
|
||||
&enc.y_plane[..4]
|
||||
);
|
||||
assert!(enc.u_plane.iter().all(|&u| u == 128));
|
||||
assert!(enc.v_plane.iter().all(|&v| v == 128));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encodes_synthetic_frame_to_annexb_idr() {
|
||||
let (w, h, fps) = (1280u32, 720u32, 60u32);
|
||||
let mut enc =
|
||||
OpenH264Encoder::open(PixelFormat::Bgrx, w, h, fps, 8_000_000).expect("open openh264");
|
||||
// A flat gray BGRx frame.
|
||||
let frame = CapturedFrame {
|
||||
width: w,
|
||||
height: h,
|
||||
pts_ns: 0,
|
||||
format: PixelFormat::Bgrx,
|
||||
payload: FramePayload::Cpu(vec![0x80u8; (w * h * 4) as usize]),
|
||||
cursor: None,
|
||||
};
|
||||
enc.submit(&frame).expect("submit");
|
||||
let au = enc.poll().expect("poll").expect("an AU");
|
||||
assert!(au.keyframe, "first frame must be an IDR");
|
||||
// AnnexB start code + an SPS NAL (type 7) somewhere in the first frame.
|
||||
assert!(
|
||||
au.data.starts_with(&[0, 0, 0, 1]) || au.data.starts_with(&[0, 0, 1]),
|
||||
"expected AnnexB start code"
|
||||
);
|
||||
let has_sps = au
|
||||
.data
|
||||
.windows(5)
|
||||
.any(|w| w[0] == 0 && w[1] == 0 && w[2] == 0 && w[3] == 1 && (w[4] & 0x1f) == 7);
|
||||
assert!(has_sps, "IDR must carry an SPS NAL (type 7)");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user