feat(windows): AMD (AMF) + Intel (QSV) hardware encode on the Windows host

The Windows host was NVIDIA-only (NVENC) with an openh264 software fallback. Add AMD AMF and Intel QSV via libavcodec — the Windows analogue of the Linux VAAPI backend — so one installer serves all three GPU vendors. - encode/ffmpeg_win.rs: new WinVendor{Amf,Qsv} encoder. System-memory NV12/P010 readback (default, robust) + opt-in zero-copy D3D11 (PUNKTFUNK_ZEROCOPY: shares the capturer's ID3D11Device; AMF takes AV_PIX_FMT_D3D11, QSV derives a QSV frames ctx and maps) with a system fallback for the format-group mismatch the capturer's video-processor fallback can produce. HDR Main10 (P010 + BT.2020/PQ VUI; an Rgb10a2->P010 swscale covers the shader fallback). - encode.rs: Codec::amf_name/qsv_name; open_video + windows_resolved_backend() resolve PUNKTFUNK_ENCODER=auto|nvenc|amf|qsv|sw via a DXGI adapter VendorId probe. - capture/dxgi.rs: gpu_mode mirrors the resolved backend (D3D11 NV12/P010 for AMF/QSV). - gamestream/serverinfo.rs: GPU-aware codec advertisement (windows_codec_support; AV1 gated to RDNA3+/Arc, like the VAAPI path). - Cargo.toml: amf-qsv feature (optional ffmpeg-next in the windows target block). - CI/installer: windows-host.yml sets FFMPEG_DIR + builds --features nvenc,amf-qsv; the Inno installer bundles the FFmpeg DLLs; host.env default nvenc -> auto. CI-green target; AMF/QSV not yet on-glass validated (no AMD/Intel Windows box in the lab) — NVENC stays live-validated. An adversarial-review pass caught + fixed real FFI bugs (AV_PIX_FMT_P010 is a macro -> P010LE; windows-rs 0.62 GetImmediateContext/ GetDesc1 return Result; AV_HWFRAME_MAP_* is a bindgen enum with no BitOr). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-22 10:31:54 +00:00
parent fde438a1ed
commit 72eeedc4da
12 changed files with 1515 additions and 86 deletions
@@ -2157,9 +2157,14 @@ impl DuplCapturer {
                .ok()
                .and_then(|s| s.parse().ok())
                .unwrap_or((2000 / refresh_hz.max(1)).max(100));
-            let gpu_mode = std::env::var("PUNKTFUNK_ENCODER")
-                .map(|v| matches!(v.to_ascii_lowercase().as_str(), "nvenc" | "hw" | "nvidia"))
-                .unwrap_or(false);
+            // Produce GPU-resident D3D11 frames (zero-copy NVENC, or the NV12/P010 the AMF/QSV
+            // backends read back / import) whenever the resolved encode backend is a GPU one — so the
+            // capturer's output format matches the encoder's input. Only the software (GPU-less) path
+            // takes CPU staging. Mirrors `encode::open_video`'s dispatch exactly.
+            let gpu_mode = !matches!(
+                crate::encode::windows_resolved_backend(),
+                crate::encode::WindowsBackend::Software
+            );
            // Read the source display's HDR mastering metadata while we still hold `output` (it is
            // moved into the struct below). Only meaningful for an HDR (FP16) duplication.
            let is_hdr_init = dd.ModeDesc.Format == DXGI_FORMAT_R16G16B16A16_FLOAT;
@@ -49,6 +49,26 @@ impl Codec {
            Codec::Av1 => "av1_vaapi",
        }
    }
+
+    /// The FFmpeg AMD **AMF** encoder name (the Windows AMD backend). Selected by name (the codec id
+    /// would pick the software encoder). AV1 (`av1_amf`) is RDNA3+/RX 7000+ — probe, never assume.
+    pub fn amf_name(self) -> &'static str {
+        match self {
+            Codec::H264 => "h264_amf",
+            Codec::H265 => "hevc_amf",
+            Codec::Av1 => "av1_amf",
+        }
+    }
+
+    /// The FFmpeg Intel **QSV** encoder name (the Windows Intel backend). Selected by name. AV1
+    /// (`av1_qsv`) is Arc/Xe2+; HEVC Main10 is Gen9.5+ — probe, never assume.
+    pub fn qsv_name(self) -> &'static str {
+        match self {
+            Codec::H264 => "h264_qsv",
+            Codec::H265 => "hevc_qsv",
+            Codec::Av1 => "av1_qsv",
+        }
+    }
 }

 /// A hardware encoder. One per session; runs on the encode thread.
@@ -198,49 +218,83 @@ pub fn open_video(
    #[cfg(target_os = "windows")]
    {
        let _ = cuda; // always false on Windows (no Cuda payload)
-        let _ = bit_depth; // used by the NVENC path below; the software H.264 path is 8-bit only
-        let pref = std::env::var("PUNKTFUNK_ENCODER")
-            .unwrap_or_default()
-            .to_ascii_lowercase();
-        if matches!(pref.as_str(), "nvenc" | "hw" | "nvidia") {
-            // Hardware path: NVENC over D3D11. The DXGI capturer switches to its zero-copy
-            // FramePayload::D3d11 output under the same env var so capture + encode share textures.
-            #[cfg(feature = "nvenc")]
-            {
-                let enc = nvenc::NvencD3d11Encoder::open(
-                    codec,
+                      // NVIDIA → NVENC (direct SDK), AMD → AMF, Intel → QSV (both libavcodec), else → software
+                      // H.264. `auto` (the default) resolves from the DXGI adapter vendor.
+        match windows_resolved_backend() {
+            WindowsBackend::Nvenc => {
+                // Hardware path: NVENC over D3D11. The DXGI capturer switches to its zero-copy
+                // FramePayload::D3d11 output under the same env var so capture + encode share textures.
+                #[cfg(feature = "nvenc")]
+                {
+                    nvenc::NvencD3d11Encoder::open(
+                        codec,
+                        format,
+                        width,
+                        height,
+                        fps,
+                        bitrate_bps,
+                        bit_depth,
+                    )
+                    .map(|e| Box::new(e) as Box<dyn Encoder>)
+                }
+                #[cfg(not(feature = "nvenc"))]
+                {
+                    anyhow::bail!(
+                        "NVENC requested/detected but this host was built without it — rebuild \
+                         with `--features nvenc` (needs the NVENC SDK's nvencodeapi.lib at link time)"
+                    )
+                }
+            }
+            backend @ (WindowsBackend::Amf | WindowsBackend::Qsv) => {
+                // AMD AMF / Intel QSV via libavcodec (the Windows analogue of the Linux VAAPI path).
+                #[cfg(feature = "amf-qsv")]
+                {
+                    let vendor = if matches!(backend, WindowsBackend::Amf) {
+                        ffmpeg_win::WinVendor::Amf
+                    } else {
+                        ffmpeg_win::WinVendor::Qsv
+                    };
+                    ffmpeg_win::FfmpegWinEncoder::open(
+                        vendor,
+                        codec,
+                        format,
+                        width,
+                        height,
+                        fps,
+                        bitrate_bps,
+                        bit_depth,
+                    )
+                    .map(|e| Box::new(e) as Box<dyn Encoder>)
+                }
+                #[cfg(not(feature = "amf-qsv"))]
+                {
+                    let _ = backend;
+                    anyhow::bail!(
+                        "AMD/Intel (AMF/QSV) encode requested/detected but this host was built \
+                         without it — rebuild with `--features amf-qsv` (needs ffmpeg-next + a \
+                         FFMPEG_DIR with the AMF/QSV encoders at build time)"
+                    )
+                }
+            }
+            WindowsBackend::Software => {
+                anyhow::ensure!(
+                    codec == Codec::H264,
+                    "the Windows software encoder supports H.264 only; client negotiated {codec:?} \
+                     (build a GPU backend: --features nvenc or amf-qsv, or request H264)"
+                );
+                let _ = bit_depth; // the software H.264 path is 8-bit only
+                                   // Software H.264 realistically caps far below the negotiated hardware rates.
+                const SW_BITRATE_CEIL: u64 = 100_000_000;
+                sw::OpenH264Encoder::open(
                    format,
                    width,
                    height,
                    fps,
-                    bitrate_bps,
-                    bit_depth,
-                )?;
-                return Ok(Box::new(enc) as Box<dyn Encoder>);
-            }
-            #[cfg(not(feature = "nvenc"))]
-            {
-                anyhow::bail!(
-                    "NVENC requested but this host was built without it — rebuild with \
-                     `--features nvenc` (needs the NVENC SDK's nvencodeapi.lib at link time)"
-                );
+                    bitrate_bps.min(SW_BITRATE_CEIL),
+                )
+                .map(|e| Box::new(e) as Box<dyn Encoder>)
            }
        }
-        anyhow::ensure!(
-            codec == Codec::H264,
-            "the Windows software encoder supports H.264 only; client negotiated {codec:?} \
-             (set PUNKTFUNK_ENCODER=nvenc for a GPU host, or request H264)"
-        );
-        // Software H.264 realistically caps far below the negotiated hardware rates.
-        const SW_BITRATE_CEIL: u64 = 100_000_000;
-        let enc = sw::OpenH264Encoder::open(
-            format,
-            width,
-            height,
-            fps,
-            bitrate_bps.min(SW_BITRATE_CEIL),
-        )?;
-        Ok(Box::new(enc) as Box<dyn Encoder>)
    }
    #[cfg(not(any(target_os = "linux", target_os = "windows")))]
    {
@@ -339,7 +393,7 @@ pub fn linux_zero_copy_is_vaapi() -> bool {
 /// Which codecs the active GPU can actually ENCODE. Used to build the GameStream codec
 /// advertisement so a client never negotiates a codec the GPU can't do (AV1 encode is narrow —
 /// Intel Arc/Xe2+, AMD RDNA3+/RDNA4 — so it must be probed, not assumed).
-#[cfg(target_os = "linux")]
+#[cfg(any(target_os = "linux", target_os = "windows"))]
 #[derive(Clone, Copy, Debug)]
 pub struct CodecSupport {
    pub h264: bool,
@@ -370,6 +424,123 @@ pub fn vaapi_codec_support() -> CodecSupport {
    })
 }

+// ---------------------------------------------------------------------------------------------
+// Windows backend selection (the analogue of the Linux nvidia_present / linux_zero_copy_is_vaapi
+// logic). NVIDIA → NVENC, AMD → AMF, Intel → QSV; `auto` (default) reads the DXGI adapter vendor.
+// ---------------------------------------------------------------------------------------------
+
+#[cfg(target_os = "windows")]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum WindowsBackend {
+    Nvenc,
+    Amf,
+    Qsv,
+    Software,
+}
+
+#[cfg(target_os = "windows")]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum GpuVendor {
+    Nvidia,
+    Amd,
+    Intel,
+}
+
+/// Resolve the active Windows encode backend from `PUNKTFUNK_ENCODER` (`auto` → the DXGI adapter
+/// vendor). Shared by [`open_video`] and the GameStream codec advertisement so both agree.
+#[cfg(target_os = "windows")]
+pub(crate) fn windows_resolved_backend() -> WindowsBackend {
+    let pref = std::env::var("PUNKTFUNK_ENCODER")
+        .unwrap_or_default()
+        .to_ascii_lowercase();
+    match pref.as_str() {
+        "nvenc" | "hw" | "nvidia" | "cuda" => WindowsBackend::Nvenc,
+        "amf" | "amd" => WindowsBackend::Amf,
+        "qsv" | "intel" => WindowsBackend::Qsv,
+        "sw" | "software" | "openh264" => WindowsBackend::Software,
+        _ => match windows_gpu_vendor() {
+            Some(GpuVendor::Nvidia) => WindowsBackend::Nvenc,
+            Some(GpuVendor::Amd) => WindowsBackend::Amf,
+            Some(GpuVendor::Intel) => WindowsBackend::Qsv,
+            None => WindowsBackend::Software,
+        },
+    }
+}
+
+/// True if the active Windows backend is the libavcodec AMF/QSV path (so the codec advertisement
+/// consults a real GPU probe rather than the NVENC static superset). Always false when the
+/// `amf-qsv` feature is off — there's then no ffmpeg backend to probe.
+#[cfg(target_os = "windows")]
+pub fn windows_backend_is_ffmpeg() -> bool {
+    cfg!(feature = "amf-qsv")
+        && matches!(
+            windows_resolved_backend(),
+            WindowsBackend::Amf | WindowsBackend::Qsv
+        )
+}
+
+/// Detect the host GPU vendor from the first hardware DXGI adapter (Windows has no `/dev/nvidia*`
+/// probe). Cached. NVIDIA=0x10DE, AMD=0x1002, Intel=0x8086; the software/WARP adapter is skipped.
+#[cfg(target_os = "windows")]
+fn windows_gpu_vendor() -> Option<GpuVendor> {
+    use std::sync::OnceLock;
+    use windows::Win32::Graphics::Dxgi::{
+        CreateDXGIFactory1, IDXGIFactory1, DXGI_ADAPTER_FLAG_SOFTWARE,
+    };
+    static CACHE: OnceLock<Option<GpuVendor>> = OnceLock::new();
+    *CACHE.get_or_init(|| unsafe {
+        let factory: IDXGIFactory1 = CreateDXGIFactory1().ok()?;
+        let mut i = 0u32;
+        while let Ok(adapter) = factory.EnumAdapters1(i) {
+            i += 1;
+            // windows-rs 0.62: GetDesc1 returns the desc by value (no out-param).
+            let Ok(desc) = adapter.GetDesc1() else {
+                continue;
+            };
+            if (desc.Flags & DXGI_ADAPTER_FLAG_SOFTWARE.0 as u32) != 0 {
+                continue; // skip the Microsoft Basic Render / WARP adapter
+            }
+            match desc.VendorId {
+                0x10DE => return Some(GpuVendor::Nvidia),
+                0x1002 => return Some(GpuVendor::Amd),
+                0x8086 => return Some(GpuVendor::Intel),
+                _ => continue,
+            }
+        }
+        None
+    })
+}
+
+/// Probe the active Windows AMF/QSV backend for its encodable codecs (cached; opens a tiny encoder
+/// per codec, once). Mirrors [`vaapi_codec_support`]; called only when [`windows_backend_is_ffmpeg`]
+/// is true. AV1 is narrow (AMD RDNA3+, Intel Arc/Xe2+), so it must be probed, not assumed.
+#[cfg(all(target_os = "windows", feature = "amf-qsv"))]
+pub fn windows_codec_support() -> CodecSupport {
+    use std::sync::OnceLock;
+    static CACHE: OnceLock<CodecSupport> = OnceLock::new();
+    *CACHE.get_or_init(|| {
+        let vendor = match windows_resolved_backend() {
+            WindowsBackend::Qsv => ffmpeg_win::WinVendor::Qsv,
+            _ => ffmpeg_win::WinVendor::Amf,
+        };
+        let caps = CodecSupport {
+            h264: ffmpeg_win::probe_can_encode(vendor, Codec::H264),
+            h265: ffmpeg_win::probe_can_encode(vendor, Codec::H265),
+            av1: ffmpeg_win::probe_can_encode(vendor, Codec::Av1),
+        };
+        tracing::info!(
+            backend = ?vendor,
+            h264 = caps.h264,
+            h265 = caps.h265,
+            av1 = caps.av1,
+            "Windows AMF/QSV encode capabilities probed"
+        );
+        caps
+    })
+}
+
+#[cfg(all(target_os = "windows", feature = "amf-qsv"))]
+mod ffmpeg_win;
 #[cfg(target_os = "linux")]
 mod linux;
 #[cfg(all(target_os = "windows", feature = "nvenc"))]
@@ -50,29 +50,41 @@ pub fn serverinfo_xml(host: &Host, https: bool, paired: bool) -> String {
 fn codec_mode_support() -> u32 {
    #[cfg(target_os = "linux")]
    if crate::encode::linux_zero_copy_is_vaapi() {
-        use super::{SCM_AV1_MAIN8, SCM_H264, SCM_HEVC};
-        let caps = crate::encode::vaapi_codec_support();
-        let mut m = 0;
-        if caps.h264 {
-            m |= SCM_H264;
+        if let Some(m) = probed_mask(crate::encode::vaapi_codec_support()) {
+            return m;
        }
-        if caps.h265 {
-            m |= SCM_HEVC;
-        }
-        if caps.av1 {
-            m |= SCM_AV1_MAIN8;
-        }
-        // Only trust a probe that actually found an encoder. An empty result means VAAPI wasn't
-        // usable at probe time (no VA display — a GPU-less CI box, or a misconfigured host), NOT
-        // that the GPU encodes nothing; advertise the static superset (pre-probe behaviour) rather
-        // than claiming zero codecs.
-        if m != 0 {
+    }
+    // Windows AMD/Intel (AMF/QSV): advertise only what the GPU actually encodes (AV1 is narrow, an
+    // old iGPU might lack HEVC). NVENC and the GPU-less software path keep the static superset.
+    #[cfg(all(target_os = "windows", feature = "amf-qsv"))]
+    if crate::encode::windows_backend_is_ffmpeg() {
+        if let Some(m) = probed_mask(crate::encode::windows_codec_support()) {
            return m;
        }
    }
    SERVER_CODEC_MODE_SUPPORT
 }

+/// Turn a probed [`CodecSupport`](crate::encode::CodecSupport) into a `ServerCodecModeSupport` mask,
+/// or `None` if the probe found nothing — meaning the GPU wasn't usable at probe time (GPU-less CI,
+/// a misconfigured/wrong-vendor host), NOT that it encodes zero codecs; the caller then advertises
+/// the static superset (pre-probe behaviour) rather than claiming nothing.
+#[cfg(any(target_os = "linux", all(target_os = "windows", feature = "amf-qsv")))]
+fn probed_mask(caps: crate::encode::CodecSupport) -> Option<u32> {
+    use super::{SCM_AV1_MAIN8, SCM_H264, SCM_HEVC};
+    let mut m = 0;
+    if caps.h264 {
+        m |= SCM_H264;
+    }
+    if caps.h265 {
+        m |= SCM_HEVC;
+    }
+    if caps.av1 {
+        m |= SCM_AV1_MAIN8;
+    }
+    (m != 0).then_some(m)
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -603,7 +603,8 @@ fn uninstall() -> Result<()> {
    Ok(())
 }

-/// Write a default `host.env` if none exists, so a fresh install streams with NVENC out of the box.
+/// Write a default `host.env` if none exists, so a fresh install streams out of the box. The encoder
+/// defaults to `auto` — the host picks NVENC (NVIDIA) / AMF (AMD) / QSV (Intel) from the GPU vendor.
 fn ensure_default_host_env() -> Result<()> {
    let path = host_env_path();
    if path.exists() {
@@ -616,7 +617,9 @@ fn ensure_default_host_env() -> Result<()> {
        # KEY=VALUE per line; '#' comments. Restart the service after editing:\n\
        #   punktfunk-host service stop && punktfunk-host service start\n\
        \n\
-        PUNKTFUNK_ENCODER=nvenc\n\
+        # Encode backend: auto (default) detects the GPU vendor — NVIDIA->nvenc, AMD->amf, Intel->qsv.\n\
+        # Force one with nvenc | amf | qsv | sw (software H.264). amf/qsv need an FFmpeg-built host.\n\
+        PUNKTFUNK_ENCODER=auto\n\
        PUNKTFUNK_VIDEO_SOURCE=virtual\n\
        PUNKTFUNK_SECURE_DDA=1\n\
        RUST_LOG=info\n\
@@ -625,7 +628,7 @@ fn ensure_default_host_env() -> Result<()> {
        # compat). Use `serve` for a SECURE native-only host (no GameStream #5/#9 surface).\n\
        # PUNKTFUNK_HOST_CMD=serve --gamestream\n\
        \n\
-        # Force a specific NVENC render GPU by name substring (multi-GPU boxes only):\n\
+        # Force a specific render GPU by name substring (multi-GPU boxes only):\n\
        # PUNKTFUNK_RENDER_ADAPTER=4090\n";
    std::fs::write(&path, default).with_context(|| format!("write {}", path.display()))?;
    println!("Wrote default config: {}", path.display());