perf(pyrowave): elevated GPU scheduling + global-priority encode queue

PyroWave's wavelet encode runs on the GPU's compute/shader cores, so a GPU-bound
game starves it: submit spikes from ~2 ms to ~15 ms under a 95%+ game load and
the stream fps collapses. NVENC is immune (separate encoder ASIC). Two levers to
let the encode get scheduled ahead of the game's rendering:

- Windows process GPU scheduling: D3DKMTSetProcessSchedulingPriorityClass, env
  PUNKTFUNK_GPU_PRIORITY = off|above-normal|high (default)|realtime. Best-effort,
  once per process, non-fatal on refusal (enc/windows/pyrowave.rs).
- Global-priority Vulkan encode queue (Granite patch 0005): request a
  VK_KHR_global_priority queue (PYROWAVE_QUEUE_PRIORITY = off|high|realtime,
  default realtime), downgrading REALTIME→HIGH→none on NOT_PERMITTED so a refused
  class never regresses the encoder to HEVC.

HONEST STATUS: on an RTX 4090 / Windows / WDDM neither moved the ~15 ms spikes —
the graphics-vs-compute preemption granularity is the wall, not the priority
level. Kept because both are correct, harmless (graceful fallback), and may help
other GPUs/drivers. For a GPU-saturated game the working levers are reducing the
encode's GPU cost (4:2:0/8-bit) or H.265; PyroWave holds full rate on the desktop
and in games that leave the GPU headroom.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

crates/pf-encode/Cargo.toml

@@ -67,6 +67,9 @@ windows = { version = "0.62", features = [
     "Win32_Storage_FileSystem",
     "Win32_System_LibraryLoader",
     "Win32_System_Threading",
+    # D3DKMTSetProcessSchedulingPriorityClass — raise the host's WDDM GPU scheduling priority
+    # above a running game so PyroWave's compute-shader encode isn't starved (enc/windows/pyrowave.rs).
+    "Wdk_Graphics_Direct3D",
 ] }
 
 [features]

crates/pf-encode/src/enc/windows/pyrowave.rs

@@ -67,6 +67,65 @@ fn budget_for(bitrate_bps: u64, fps: u32) -> usize {
     ((bitrate_bps / (8 * fps.max(1) as u64)) as usize).max(64 * 1024)
 }
 
+/// Raise this process's WDDM GPU scheduling priority so the wavelet encode isn't starved by a
+/// GPU-bound game. PyroWave encodes on the GPU's compute/shader cores — the exact resource a game
+/// saturates — so under load `pyrowave_encoder_encode_gpu_synchronous` spikes from ~2 ms to
+/// 15-18 ms (measured, RTX 4090 at 95 % game load) and the stream fps collapses; NVENC is immune
+/// because it runs on the separate encoder ASIC. HIGH sits above a game's NORMAL/ABOVE_NORMAL so
+/// the WDDM scheduler services the encode's short compute bursts ahead of the game's rendering.
+/// (REALTIME is deliberately avoided: it needs a privilege and would preempt the desktop
+/// compositor too.) Best-effort + once-per-process: if the class isn't grantable we log and run at
+/// normal priority — no session-fatal path.
+fn raise_process_gpu_priority() {
+    use std::sync::Once;
+    static ONCE: Once = Once::new();
+    ONCE.call_once(|| {
+        use windows::Wdk::Graphics::Direct3D::{
+            D3DKMTSetProcessSchedulingPriorityClass, D3DKMT_SCHEDULINGPRIORITYCLASS_ABOVE_NORMAL,
+            D3DKMT_SCHEDULINGPRIORITYCLASS_HIGH, D3DKMT_SCHEDULINGPRIORITYCLASS_REALTIME,
+        };
+        // `PUNKTFUNK_GPU_PRIORITY` = off | above-normal | high (default) | realtime. REALTIME can
+        // force finer WDDM preemption than HIGH but needs a privilege and preempts the compositor,
+        // so it stays opt-in; `off` skips the call entirely.
+        let (class, label) = match std::env::var("PUNKTFUNK_GPU_PRIORITY")
+            .ok()
+            .as_deref()
+            .map(str::trim)
+            .map(str::to_ascii_lowercase)
+            .as_deref()
+        {
+            Some("off") => {
+                tracing::info!("PyroWave: PUNKTFUNK_GPU_PRIORITY=off — leaving GPU scheduling priority at default");
+                return;
+            }
+            Some("above-normal") | Some("above_normal") => {
+                (D3DKMT_SCHEDULINGPRIORITYCLASS_ABOVE_NORMAL, "ABOVE_NORMAL")
+            }
+            Some("realtime") => (D3DKMT_SCHEDULINGPRIORITYCLASS_REALTIME, "REALTIME"),
+            _ => (D3DKMT_SCHEDULINGPRIORITYCLASS_HIGH, "HIGH"),
+        };
+        // SAFETY: `GetCurrentProcess` returns the current-process pseudo-handle; the D3DKMT call
+        // only sets this process's GPU scheduling class — it creates/frees nothing.
+        let status = unsafe {
+            D3DKMTSetProcessSchedulingPriorityClass(GetCurrentProcess(), class)
+        };
+        if status.is_ok() {
+            tracing::info!(
+                priority = label,
+                "PyroWave: raised process GPU scheduling priority (WDDM) so the wavelet encode is \
+                 serviced ahead of game rendering on the shared shader cores"
+            );
+        } else {
+            tracing::warn!(
+                ?status,
+                priority = label,
+                "PyroWave: could not raise GPU scheduling priority (not grantable) — the encode \
+                 may be starved under heavy game GPU load"
+            );
+        }
+    });
+}
+
 pub struct PyroWaveEncoder {
     // pyrowave owns the whole Vulkan device (create_device_by_compat) — no ash on this side.
     pw_dev: pw::pyrowave_device,
@@ -112,6 +171,9 @@ impl PyroWaveEncoder {
         chroma: crate::ChromaFormat,
         bit_depth: u8,
     ) -> Result<Self> {
+        // Prioritize the host's GPU work over a running game so the compute-shader encode gets
+        // scheduled promptly instead of queuing behind a full frame of the game's rendering.
+        raise_process_gpu_priority();
         let chroma444 = chroma.is_444();
         // A negotiated 10-bit session rides 16-bit UNORM planes carrying the P010-style
         // studio codes the capturer's HDR CSC writes (design/pyrowave-444-hdr.md §2.2) —

crates/pyrowave-sys/patches/0005-global-priority-queue.patch

@@ -0,0 +1,121 @@
+Elevated global-priority encode queue — PUNKTFUNK LOCAL PATCH.
+
+Not upstream. PyroWave's wavelet encode runs on the GPU's compute/shader cores, so a GPU-bound
+game starves it: `pyrowave_encoder_encode_gpu_synchronous` spikes from ~2 ms to ~15 ms under a
+95%+ game load and the stream framerate collapses (NVENC is immune — it uses the separate encoder
+ASIC). A WDDM *process* scheduling-priority raise does not help (it orders packet submission, not
+hardware preemption). This requests a global-priority Vulkan *queue* — the actual NVIDIA compute-
+preemption lever — on the pyrowave encode device.
+
+`Context::create_device` (the pyrowave path uses plain vkCreateDevice, not the vpCreateDevice
+profile path): enable VK_KHR_global_priority (or the EXT alias), chain a
+VkDeviceQueueGlobalPriorityCreateInfoKHR into every queue-create-info, and a create loop that tries
+`PYROWAVE_QUEUE_PRIORITY` (off|high|realtime, default realtime) → HIGH → no-priority, downgrading on
+VK_ERROR_NOT_PERMITTED_KHR so a refused class NEVER regresses the encoder. Gated on !inherit_info
+(only the fresh encoder device). This Granite copy is vendored solely for the PyroWave codec.
+
+NOTE: on an RTX 4090 / Windows / WDDM this did not reduce the spikes (the graphics-vs-compute
+preemption granularity is the wall) — kept because it is correct, harmless (graceful fallback), and
+may help other GPUs/drivers. Reduce the encode's GPU cost (4:2:0/8-bit) or use H.265 for a
+GPU-saturated game.
+
+diff --git a/crates/pyrowave-sys/vendor/pyrowave/Granite/vulkan/context.cpp b/crates/pyrowave-sys/vendor/pyrowave/Granite/vulkan/context.cpp
+index 5257fc33..479eeded 100644
+--- a/crates/pyrowave-sys/vendor/pyrowave/Granite/vulkan/context.cpp
++++ b/crates/pyrowave-sys/vendor/pyrowave/Granite/vulkan/context.cpp
+@@ -2115,6 +2115,51 @@ bool Context::create_device(VkPhysicalDevice gpu_, VkSurfaceKHR surface,
+ 		vpGetProfileProperties(profile.profile, &props);
+ #endif
+ 
++	// PUNKTFUNK PATCH (patches/0005-global-priority-queue.patch, NOT upstream): request a high
++	// global-priority queue so PyroWave's compute-shader encode can PREEMPT a GPU-bound game on
++	// the shared shader cores. A WDDM process-priority raise does not help (it only orders packet
++	// submission, not hardware preemption); a global-priority queue is the actual NVIDIA compute-
++	// preemption lever. `PYROWAVE_QUEUE_PRIORITY` = off | high | realtime (default realtime). The
++	// device-create loop below downgrades on NOT_PERMITTED, so a refused class NEVER fails the
++	// encoder — it just runs at default priority. Only the fresh encoder device (no inherit_info)
++	// is affected; this Granite copy is vendored solely for the PyroWave codec.
++	Util::SmallVector<VkQueueGlobalPriorityKHR> pf_priority_candidates;
++	Util::SmallVector<VkDeviceQueueGlobalPriorityCreateInfoKHR> pf_global_priority_infos;
++	if (!inherit_info)
++	{
++		const char *pf_gp_ext = nullptr;
++		if (has_extension(VK_KHR_GLOBAL_PRIORITY_EXTENSION_NAME))
++			pf_gp_ext = VK_KHR_GLOBAL_PRIORITY_EXTENSION_NAME;
++		else if (has_extension(VK_EXT_GLOBAL_PRIORITY_EXTENSION_NAME))
++			pf_gp_ext = VK_EXT_GLOBAL_PRIORITY_EXTENSION_NAME;
++		std::string pf_prio;
++		if (!Util::get_environment("PYROWAVE_QUEUE_PRIORITY", pf_prio))
++			pf_prio = "realtime";
++		for (auto &pf_ch : pf_prio)
++			if (pf_ch >= 'A' && pf_ch <= 'Z')
++				pf_ch = char(pf_ch + 32);
++		if (pf_gp_ext && pf_prio != "off")
++		{
++			if (pf_prio == "high")
++			{
++				pf_priority_candidates.push_back(VK_QUEUE_GLOBAL_PRIORITY_HIGH_KHR);
++			}
++			else
++			{
++				pf_priority_candidates.push_back(VK_QUEUE_GLOBAL_PRIORITY_REALTIME_KHR);
++				pf_priority_candidates.push_back(VK_QUEUE_GLOBAL_PRIORITY_HIGH_KHR);
++			}
++			enabled_extensions.push_back(pf_gp_ext);
++			pf_global_priority_infos.resize(queue_infos.size());
++			for (size_t pf_i = 0; pf_i < queue_infos.size(); pf_i++)
++			{
++				pf_global_priority_infos[pf_i] = { VK_STRUCTURE_TYPE_DEVICE_QUEUE_GLOBAL_PRIORITY_CREATE_INFO_KHR };
++				pf_global_priority_infos[pf_i].globalPriority = pf_priority_candidates.front();
++				queue_infos[pf_i].pNext = &pf_global_priority_infos[pf_i];
++			}
++		}
++	}
++
+ 	if (inherit_info)
+ 	{
+ 		device_info.enabledExtensionCount = inherit_info->enabledExtensionCount;
+@@ -2144,8 +2189,41 @@ bool Context::create_device(VkPhysicalDevice gpu_, VkSurfaceKHR surface,
+ 			if (device == VK_NULL_HANDLE)
+ 				return false;
+ 		}
+-		else if (vkCreateDevice(gpu, &device_info, nullptr, &device) != VK_SUCCESS)
+-			return false;
++		else
++		{
++			// PUNKTFUNK: try the requested global-priority class, downgrade through the
++			// candidate list on NOT_PERMITTED, then finally create with no global priority.
++			VkResult pf_res;
++			if (pf_priority_candidates.empty())
++			{
++				pf_res = vkCreateDevice(gpu, &device_info, nullptr, &device);
++			}
++			else
++			{
++				pf_res = VK_ERROR_NOT_PERMITTED_KHR;
++				for (size_t pf_a = 0; pf_a < pf_priority_candidates.size(); pf_a++)
++				{
++					for (auto &pf_gp : pf_global_priority_infos)
++						pf_gp.globalPriority = pf_priority_candidates[pf_a];
++					pf_res = vkCreateDevice(gpu, &device_info, nullptr, &device);
++					if (pf_res != VK_ERROR_NOT_PERMITTED_KHR)
++						break;
++					LOGW("PyroWave: global queue priority %u not permitted; downgrading.\n",
++						unsigned(pf_priority_candidates[pf_a]));
++				}
++				if (pf_res == VK_ERROR_NOT_PERMITTED_KHR)
++				{
++					for (auto &pf_qi : queue_infos)
++						pf_qi.pNext = nullptr;
++					pf_res = vkCreateDevice(gpu, &device_info, nullptr, &device);
++					LOGW("PyroWave: all global queue priorities refused; default priority.\n");
++				}
++				else
++					LOGI("PyroWave: encode device created with an elevated global queue priority.\n");
++			}
++			if (pf_res != VK_SUCCESS)
++				return false;
++		}
+ 	}
+ 
+ 	if (inherit_info)

crates/pyrowave-sys/vendor/pyrowave/Granite/vulkan/context.cpp

@@ -2115,6 +2115,51 @@ bool Context::create_device(VkPhysicalDevice gpu_, VkSurfaceKHR surface,
 		vpGetProfileProperties(profile.profile, &props);
 #endif
 
+	// PUNKTFUNK PATCH (patches/0005-global-priority-queue.patch, NOT upstream): request a high
+	// global-priority queue so PyroWave's compute-shader encode can PREEMPT a GPU-bound game on
+	// the shared shader cores. A WDDM process-priority raise does not help (it only orders packet
+	// submission, not hardware preemption); a global-priority queue is the actual NVIDIA compute-
+	// preemption lever. `PYROWAVE_QUEUE_PRIORITY` = off | high | realtime (default realtime). The
+	// device-create loop below downgrades on NOT_PERMITTED, so a refused class NEVER fails the
+	// encoder — it just runs at default priority. Only the fresh encoder device (no inherit_info)
+	// is affected; this Granite copy is vendored solely for the PyroWave codec.
+	Util::SmallVector<VkQueueGlobalPriorityKHR> pf_priority_candidates;
+	Util::SmallVector<VkDeviceQueueGlobalPriorityCreateInfoKHR> pf_global_priority_infos;
+	if (!inherit_info)
+	{
+		const char *pf_gp_ext = nullptr;
+		if (has_extension(VK_KHR_GLOBAL_PRIORITY_EXTENSION_NAME))
+			pf_gp_ext = VK_KHR_GLOBAL_PRIORITY_EXTENSION_NAME;
+		else if (has_extension(VK_EXT_GLOBAL_PRIORITY_EXTENSION_NAME))
+			pf_gp_ext = VK_EXT_GLOBAL_PRIORITY_EXTENSION_NAME;
+		std::string pf_prio;
+		if (!Util::get_environment("PYROWAVE_QUEUE_PRIORITY", pf_prio))
+			pf_prio = "realtime";
+		for (auto &pf_ch : pf_prio)
+			if (pf_ch >= 'A' && pf_ch <= 'Z')
+				pf_ch = char(pf_ch + 32);
+		if (pf_gp_ext && pf_prio != "off")
+		{
+			if (pf_prio == "high")
+			{
+				pf_priority_candidates.push_back(VK_QUEUE_GLOBAL_PRIORITY_HIGH_KHR);
+			}
+			else
+			{
+				pf_priority_candidates.push_back(VK_QUEUE_GLOBAL_PRIORITY_REALTIME_KHR);
+				pf_priority_candidates.push_back(VK_QUEUE_GLOBAL_PRIORITY_HIGH_KHR);
+			}
+			enabled_extensions.push_back(pf_gp_ext);
+			pf_global_priority_infos.resize(queue_infos.size());
+			for (size_t pf_i = 0; pf_i < queue_infos.size(); pf_i++)
+			{
+				pf_global_priority_infos[pf_i] = { VK_STRUCTURE_TYPE_DEVICE_QUEUE_GLOBAL_PRIORITY_CREATE_INFO_KHR };
+				pf_global_priority_infos[pf_i].globalPriority = pf_priority_candidates.front();
+				queue_infos[pf_i].pNext = &pf_global_priority_infos[pf_i];
+			}
+		}
+	}
+
 	if (inherit_info)
 	{
 		device_info.enabledExtensionCount = inherit_info->enabledExtensionCount;
@@ -2144,8 +2189,41 @@ bool Context::create_device(VkPhysicalDevice gpu_, VkSurfaceKHR surface,
 			if (device == VK_NULL_HANDLE)
 				return false;
 		}
-		else if (vkCreateDevice(gpu, &device_info, nullptr, &device) != VK_SUCCESS)
+		else
-			return false;
+		{
+			// PUNKTFUNK: try the requested global-priority class, downgrade through the
+			// candidate list on NOT_PERMITTED, then finally create with no global priority.
+			VkResult pf_res;
+			if (pf_priority_candidates.empty())
+			{
+				pf_res = vkCreateDevice(gpu, &device_info, nullptr, &device);
+			}
+			else
+			{
+				pf_res = VK_ERROR_NOT_PERMITTED_KHR;
+				for (size_t pf_a = 0; pf_a < pf_priority_candidates.size(); pf_a++)
+				{
+					for (auto &pf_gp : pf_global_priority_infos)
+						pf_gp.globalPriority = pf_priority_candidates[pf_a];
+					pf_res = vkCreateDevice(gpu, &device_info, nullptr, &device);
+					if (pf_res != VK_ERROR_NOT_PERMITTED_KHR)
+						break;
+					LOGW("PyroWave: global queue priority %u not permitted; downgrading.\n",
+						unsigned(pf_priority_candidates[pf_a]));
+				}
+				if (pf_res == VK_ERROR_NOT_PERMITTED_KHR)
+				{
+					for (auto &pf_qi : queue_infos)
+						pf_qi.pNext = nullptr;
+					pf_res = vkCreateDevice(gpu, &device_info, nullptr, &device);
+					LOGW("PyroWave: all global queue priorities refused; default priority.\n");
+				}
+				else
+					LOGI("PyroWave: encode device created with an elevated global queue priority.\n");
+			}
+			if (pf_res != VK_SUCCESS)
+				return false;
+		}
 	}
 
 	if (inherit_info)

crates/pyrowave-sys/vendor/pyrowave/PUNKTFUNK-VENDOR.txt

@@ -25,3 +25,10 @@ Local patches (crates/pyrowave-sys/patches/, re-applied on re-vendor):
     dropped to ~1 ms (~1025 fps) once the buffers are pooled on the encoder and
     reused. Safe under the synchronous encode model; re-validated by
     pyrowave_win_smoke. Perf fix, not a correctness fix.
+
+  0005-global-priority-queue.patch — Context::create_device requests a global-priority Vulkan
+    compute queue (VK_KHR_global_priority, PYROWAVE_QUEUE_PRIORITY=off|high|realtime, default
+    realtime) so the wavelet encode can preempt a GPU-bound game on the shared shader cores. A
+    create loop downgrades on NOT_PERMITTED so a refused class never regresses the encoder. Did
+    not overcome the graphics-vs-compute preemption wall on an RTX 4090 (kept: correct + harmless,
+    may help other HW/drivers).