fix(host): crash visibility + Windows deliberate-quit teardown; pin outranks quit

Two strands from the "Windows host went Offline after reconnect, zero errors in
the logs" investigation.

Crash visibility — the field reports kept arriving with nothing in the Logs
tab because every way the host can die silently was unlogged:
- A panic hook tees each panic (thread, location, backtrace) through `tracing`
  into the in-memory ring + host.log before the default hook runs — a panicking
  thread otherwise only hit stderr, absent from the web console and gone when
  stderr is detached.
- A last-resort Windows SEH filter (windows/crash.rs) logs an unhandled native
  exception with its code, faulting address, and faulting MODULE — the smoking
  gun that separates our bug from a GPU runtime/driver DLL (amfrt64, the UMD,
  d3d11) crashing under us.
- A 10 s watchdog around the vdisplay monitor teardown logs an ERROR when the
  driver REMOVE/CCD-restore looks wedged (it runs under the manager state lock,
  so a hang there silently blocks every future acquire).

Deliberate-quit teardown (Windows pf-vdisplay) — the Windows manager never
wired the linger skip the Linux registry has: on ⌘D (the QUIT close code) the
monitor lingered 10 s, so a quick reconnect hit the Lingering-preempt's
back-to-back REMOVE→ADD churn. A per-lease quit flag (VirtualDisplay::
set_quit_flag) now tears the monitor down immediately on a deliberate quit, so
the reconnect finds the manager Idle and does a clean fresh ADD.

Pin outranks quit (both platforms) — keep_alive=forever (the gaming-rig preset)
promises "the screen stays alive", so a deliberate quit must skip only the
linger WINDOW, never the pin. Windows release() checks keep_alive_forever()
before the quit; the Linux registry had the inverse bug (force_immediate tore
down even a pinned display) — fixed via effective_linger() with a unit test.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-08 14:16:19 +02:00
parent ab6790ef6c
commit 0b7e4a00ee
8 changed files with 276 additions and 20 deletions
+4
View File
@@ -201,6 +201,10 @@ windows = { version = "0.62", features = [
# CoCreateInstance(PolicyConfigClient) — set the default audio playback/recording endpoints via the
# undocumented IPolicyConfig (audio/windows/audio_control.rs) so mic + desktop audio auto-wire.
"Win32_System_Com",
# SetUnhandledExceptionFilter + EXCEPTION_POINTERS — the last-resort native-crash logger
# (src/windows/crash.rs); Kernel gates the CONTEXT type EXCEPTION_POINTERS embeds.
"Win32_System_Diagnostics_Debug",
"Win32_System_Kernel",
] }
# The SCM plumbing for the `service` subcommand (define_windows_service! / dispatcher / control
# handler / ServiceManager install). Wraps the Win32 service API; the supervision loop itself uses
+32
View File
@@ -25,6 +25,9 @@ mod discovery;
mod wol;
// Goal-1 stage 6: top-level platform-only modules live under `src/linux/` and `src/windows/`; `#[path]`
// keeps the `crate::*` module names flat (every existing path is unchanged).
#[cfg(target_os = "windows")]
#[path = "windows/crash.rs"]
mod crash;
#[cfg(target_os = "linux")]
#[path = "linux/dmabuf_fence.rs"]
mod dmabuf_fence;
@@ -111,6 +114,35 @@ fn main() {
.init();
}
// Tee every panic through `tracing` BEFORE the default hook: a panicking thread otherwise
// prints only to stderr — absent from the web console's Logs tab (the ring) and gone entirely
// when stderr is detached — so a field report reads "host died, zero errors in the logs".
// The default hook still runs afterwards for the usual stderr message/abort behavior.
let default_panic = std::panic::take_hook();
std::panic::set_hook(Box::new(move |info| {
// Manual payload downcast (`payload_as_str` needs Rust 1.91; workspace MSRV is 1.82).
let payload = info
.payload()
.downcast_ref::<&str>()
.copied()
.or_else(|| info.payload().downcast_ref::<String>().map(String::as_str))
.unwrap_or("<non-string panic payload>");
tracing::error!(
thread = std::thread::current().name().unwrap_or("<unnamed>"),
location = %info
.location()
.map(ToString::to_string)
.unwrap_or_else(|| "<unknown>".into()),
backtrace = %std::backtrace::Backtrace::force_capture(),
"PANIC: {payload}"
);
default_panic(info);
}));
// Native crashes (an access violation inside a GPU runtime/driver DLL) are logged by a
// last-resort SEH filter for the same reason — they otherwise kill the host with no trace.
#[cfg(target_os = "windows")]
crash::install();
if let Err(e) = real_main() {
tracing::error!("{e:#}");
std::process::exit(1);
+7
View File
@@ -3162,6 +3162,13 @@ fn virtual_stream(ctx: SessionContext) -> Result<()> {
// reapplies the client's saved per-monitor config (DPI scaling) on reconnect. No-op on Linux backends
// and for anonymous/GameStream clients (no fingerprint → the driver auto-allocates).
vd.set_client_identity(endpoint::peer_fingerprint(&conn));
// Deliberate-quit wiring (Windows pf-vdisplay; no-op elsewhere): every lease the backend mints —
// the retry-hold below AND the capturer's — carries the session's quit flag, so a user "stop"
// (⌘D → the QUIT close code) tears the virtual monitor down the moment the pipeline drops instead
// of lingering 10 s. The reconnect then finds the manager Idle and does a clean fresh ADD (with
// the user's think-time as driver settle) rather than the Lingering-preempt's REMOVE→ADD churn.
// `keep_alive = forever` (gaming-rig) outranks the quit — the monitor pins as before.
vd.set_quit_flag(quit.clone());
// Per-session launch (non-Windows): hand the resolved command to the backend instance so
// gamescope's bare spawn nests it — per-instance, no process-global env, so concurrent sessions
// can't stomp each other's launch target. The other backends' default `set_launch_command` is a
+8
View File
@@ -122,6 +122,14 @@ pub trait VirtualDisplay: Send {
/// Default: no-op — only the Windows pf-vdisplay backend uses it (Linux compositors own their virtual
/// output identity). `None` = anonymous/unpaired/GameStream → the backend's auto (slot-based) identity.
fn set_client_identity(&mut self, _fingerprint: Option<[u8; 32]>) {}
/// Hand the backend the session's deliberate-quit flag (set when the client closes with the QUIT
/// application code — a user "stop", not a network drop) so the last lease's drop can tear the
/// display down IMMEDIATELY, skipping the keep-alive linger — the Windows analogue of the Linux
/// registry's `Linger::Immediate` path. Carried on the backend instance; set once before
/// [`create`](Self::create). Default: no-op — only the Windows pf-vdisplay backend needs it (its
/// leases live in the `VirtualDisplayManager`, which the registry's quit plumbing does not reach;
/// Linux backends get the flag through `registry::acquire`).
fn set_quit_flag(&mut self, _quit: std::sync::Arc<std::sync::atomic::AtomicBool>) {}
/// The stable identity slot the backend resolved for the most recent [`create`](Self::create) —
/// the per-client id the identity policy assigned (`Some`), or `None` for shared/anonymous. The
/// registry reads it right after `create` to key the display's group **arrangement** (manual
+36 -9
View File
@@ -95,8 +95,9 @@ pub fn acquire(
}
#[cfg(not(target_os = "linux"))]
{
// Windows leases in the manager (its own linger); the deliberate-quit skip is not wired
// through there yet, so the flag is accepted but unused off Linux.
// Windows leases in the manager (its own linger); its deliberate-quit skip is wired through
// `VirtualDisplay::set_quit_flag` on the backend instance (set by the session before any
// `create`, so the retry-hold lease gets it too) — not through this parameter.
let _ = quit;
vd.create(mode)
}
@@ -602,18 +603,25 @@ mod linux {
Ok(output_for(node_id, preferred_mode, gen, quit, false))
}
/// The linger a releasing session actually gets. A deliberate quit (`force_immediate` — the
/// client closed with the quit code, a user "stop") downgrades a linger WINDOW to an immediate
/// teardown; a bare disconnect honors the policy. `keep_alive = forever` (the gaming-rig
/// preset) OUTRANKS the quit: its promise is "the screen stays alive", so a deliberate quit
/// still pins — only an explicit `/display/release` frees it.
fn effective_linger(force_immediate: bool, policy: Linger) -> Linger {
match (force_immediate, policy) {
(true, Linger::Forever) => Linger::Forever,
(true, _) => Linger::Immediate,
(false, l) => l,
}
}
/// The [`DisplayLease`] `Drop` path: release the session's hold on the pooled display. The
/// lifecycle machine decides linger / pin / teardown; a torn-down entry's keepalive drops *after*
/// the lock is released.
fn release(gen: u64, force_immediate: bool) {
let Some(r) = REG.get() else { return };
// A deliberate quit (the client closed with the quit code — a user "stop") tears the display
// down NOW, overriding the keep-alive linger; a bare disconnect honors the policy.
let linger = if force_immediate {
Linger::Immediate
} else {
linger()
};
let linger = effective_linger(force_immediate, linger());
let (torn_down, restore) = {
let mut es = r.entries.lock().unwrap();
let Some(idx) = es.iter().position(|e| e.gen == gen) else {
@@ -965,6 +973,25 @@ mod linux {
Box::new(move || f.store(true, Ordering::SeqCst))
}
#[test]
fn deliberate_quit_skips_the_linger_window_but_never_a_pin() {
use std::time::Duration;
// Quit downgrades a linger window (and a no-linger policy stays immediate)…
assert_eq!(
effective_linger(true, Linger::For(Duration::from_secs(10))),
Linger::Immediate
);
assert_eq!(effective_linger(true, Linger::Immediate), Linger::Immediate);
// …but never a pin: keep_alive=forever (gaming-rig) promises the screen stays alive.
assert_eq!(effective_linger(true, Linger::Forever), Linger::Forever);
// A bare disconnect honors the policy untouched.
assert_eq!(
effective_linger(false, Linger::For(Duration::from_secs(10))),
Linger::For(Duration::from_secs(10))
);
assert_eq!(effective_linger(false, Linger::Forever), Linger::Forever);
}
#[test]
fn topology_restore_floats_to_a_sibling_then_runs_on_the_last_teardown() {
let ran = Arc::new(AtomicBool::new(false));
@@ -395,6 +395,7 @@ impl VirtualDisplayManager {
&'static self,
mode: Mode,
client_fp: Option<[u8; 32]>,
quit: Option<Arc<AtomicBool>>,
) -> Result<VirtualOutput> {
self.ensure_linger_timer();
let mut state = self.state.lock().unwrap();
@@ -474,7 +475,7 @@ impl VirtualDisplayManager {
backend = self.driver.name(),
"virtual monitor reused (concurrent / reconfigure session)"
);
return Ok(self.output_for(mon));
return Ok(self.output_for(mon, quit));
}
// Idle or kept: repurpose a kept monitor / create a fresh one → Active{refs:1}. (In practice a
@@ -516,13 +517,14 @@ impl VirtualDisplayManager {
},
MgrState::Active { .. } => unreachable!("handled above"),
};
let out = self.output_for(&mon);
let out = self.output_for(&mon, quit);
*state = MgrState::Active { mon, refs: 1 };
Ok(out)
}
/// Build the [`VirtualOutput`] (preferred mode + capture target + a fresh gen-stamped lease) for `mon`.
fn output_for(&'static self, mon: &Monitor) -> VirtualOutput {
/// `quit` is the session's deliberate-quit flag, read by the lease `Drop` (see [`Self::release`]).
fn output_for(&'static self, mon: &Monitor, quit: Option<Arc<AtomicBool>>) -> VirtualOutput {
VirtualOutput {
node_id: 0,
preferred_mode: Some((mon.mode.width, mon.mode.height, mon.mode.refresh_hz)),
@@ -530,6 +532,7 @@ impl VirtualDisplayManager {
keepalive: Box::new(MonitorLease {
mgr: self,
gen: mon.gen,
quit,
}),
// The Windows manager owns the monitor lifecycle (refcount/linger/pin), so the registry
// (which delegates to it via `vd.create`) treats it as Owned.
@@ -760,6 +763,28 @@ impl VirtualDisplayManager {
/// # Safety
/// `dev` must be the live control handle.
unsafe fn teardown(&self, dev: HANDLE, mut mon: Monitor) {
// Wedge visibility: this runs synchronously — usually UNDER the `state` lock (linger timer,
// reconnect preempt, quit-skip), so a REMOVE/CCD-restore that never returns (field signature:
// Windows AMD reconnects going silently dead) blocks every future `acquire` with NOTHING in the
// log. One ERROR line after 10 s turns that silent wedge into a diagnosis.
let done = Arc::new(AtomicBool::new(false));
{
let done = done.clone();
let target = mon.target_id;
thread::Builder::new()
.name("vdisplay-teardown-watch".into())
.spawn(move || {
thread::sleep(Duration::from_secs(10));
if !done.load(Ordering::SeqCst) {
tracing::error!(
target_id = target,
"virtual-display teardown still running after 10s — the driver \
REMOVE/CCD restore looks WEDGED; new sessions will block until it returns"
);
}
})
.ok();
}
mon.stop.store(true, Ordering::Relaxed);
if let Some(j) = mon.pinger.take() {
let _ = j.join();
@@ -784,12 +809,20 @@ impl VirtualDisplayManager {
"virtual-display monitor removed"
);
}
done.store(true, Ordering::SeqCst);
}
/// Release a session's hold (the [`MonitorLease`] `Drop`): refcount-- ; the last session leaving
/// LINGERs before teardown. A STALE lease (its monitor was preempted + recreated under it) is a
/// no-op, so it can't tear down the CURRENT monitor.
fn release(&self, gen: u64) {
/// LINGERs before teardown — unless `quit_now` (the client closed with the QUIT code, a user
/// "stop"), which tears the monitor down IMMEDIATELY instead of lingering. That both restores
/// the physical displays at the moment the user quits and means a follow-up reconnect finds the
/// manager Idle — a clean fresh ADD with the user's think-time as driver settle — instead of
/// tripping the Lingering-preempt's back-to-back REMOVE→ADD. `keep_alive = forever` (the
/// gaming-rig preset) OUTRANKS the quit: its promise is "the screen stays alive", so a
/// deliberate quit still pins — only `/display/release` frees a pinned monitor. A STALE lease
/// (its monitor was preempted + recreated under it) is a no-op, so it can't tear down the
/// CURRENT monitor.
fn release(&self, gen: u64, quit_now: bool) {
let mut state = self.state.lock().unwrap();
let stale = match &*state {
MgrState::Active { mon, .. }
@@ -805,14 +838,41 @@ impl VirtualDisplayManager {
mon,
refs: refs - 1,
},
// Last session left: keep the monitor forever (Pinned) under `keep_alive = forever`,
// else linger for the policy window before the timer tears it down.
// Last session left: keep the monitor forever (Pinned) under `keep_alive = forever`
// checked BEFORE the quit, because the gaming-rig preset's contract is "the screen
// stays alive": a deliberate quit skips only the linger window, never the pin.
MgrState::Active { mon, .. } if keep_alive_forever() => {
tracing::info!(
"virtual-display: last session left — PINNED (keep_alive=forever); free via /display/release"
);
MgrState::Pinned { mon }
}
// Last session left on a deliberate quit: tear down NOW (linger skipped). Teardown
// runs UNDER the state lock — same shape as the linger timer, and for the same reason: a
// racing `acquire` must WAIT the teardown out rather than see Idle and ADD into the
// driver's in-flight REMOVE. `device_handle()` is only None if the control device was
// never opened — impossible with a monitor live — but fall back to Lingering (the timer
// retries) rather than leak the monitor.
MgrState::Active { mon, .. } if quit_now => match self.device_handle() {
Some(dev) => {
tracing::info!(
"virtual-display: last session left (deliberate quit) — tearing down now, linger skipped"
);
// SAFETY: `teardown` requires `dev` to be the live control handle; `dev` is the
// cached process-lifetime `OwnedHandle` from `device_handle()` (the `Some` checked
// above; cached handles are never closed — a dead one is retired, kept alive). `mon`
// was moved out of the `Active` state under the `state` lock, so it is exclusively
// owned here — no aliasing.
unsafe { self.teardown(dev, mon) };
MgrState::Idle
}
None => MgrState::Lingering {
mon,
until: Instant::now() + Duration::from_millis(linger_ms()),
},
},
// Last session left, no quit signal: linger for the policy window before the timer
// tears it down.
MgrState::Active { mon, .. } => {
let ms = linger_ms();
tracing::info!(
@@ -948,11 +1008,17 @@ impl VirtualDisplayManager {
struct MonitorLease {
mgr: &'static VirtualDisplayManager,
gen: u64,
/// The session's deliberate-quit flag (the client closed with the QUIT application code — a user
/// "stop", not a network drop). Read at drop time: a quit release tears the monitor down NOW
/// instead of lingering, mirroring the Linux registry's `Linger::Immediate`. `None` = no signal
/// (GameStream sessions, the mgmt reconfigure path) → the linger policy applies.
quit: Option<Arc<AtomicBool>>,
}
impl Drop for MonitorLease {
fn drop(&mut self) {
self.mgr.release(self.gen);
let quit_now = self.quit.as_ref().is_some_and(|q| q.load(Ordering::SeqCst));
self.mgr.release(self.gen, quit_now);
}
}
@@ -595,12 +595,19 @@ pub struct PfVdisplayDisplay {
/// The connecting client's cert fingerprint (`None` = anonymous/GameStream → the manager's auto id).
/// Set by [`set_client_identity`](VirtualDisplay::set_client_identity) before `create`.
client_fp: Option<[u8; 32]>,
/// The session's deliberate-quit flag (`None` = no signal → the linger policy applies). Set by
/// [`set_quit_flag`](VirtualDisplay::set_quit_flag) before `create`; rides into every lease this
/// backend mints so a user "stop" tears the monitor down immediately instead of lingering.
quit: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
}
impl PfVdisplayDisplay {
pub fn new() -> Result<Self> {
super::manager::init(Box::new(PfVdisplayDriver)).open_backend()?;
Ok(Self { client_fp: None })
Ok(Self {
client_fp: None,
quit: None,
})
}
}
@@ -613,8 +620,12 @@ impl VirtualDisplay for PfVdisplayDisplay {
self.client_fp = fingerprint;
}
fn set_quit_flag(&mut self, quit: std::sync::Arc<std::sync::atomic::AtomicBool>) {
self.quit = Some(quit);
}
fn create(&mut self, mode: Mode) -> Result<VirtualOutput> {
super::manager::vdm().acquire(mode, self.client_fp)
super::manager::vdm().acquire(mode, self.client_fp, self.quit.clone())
}
}
+101
View File
@@ -0,0 +1,101 @@
//! Last-resort crash visibility (Windows): an unhandled-SEH filter that logs a native crash —
//! an access violation inside a driver/runtime DLL (amfrt64, the GPU user-mode driver, d3d11,
//! IddCx plumbing) — through `tracing` before the process dies.
//!
//! Motivated by the "host went Offline with zero errors in the logs" class of field report: a
//! native crash kills the process (dropping the mDNS advert → every client tile flips Offline)
//! while the log ring's last entry is a healthy INFO line, so the report arrives with no evidence
//! at all. One ERROR line naming the exception code and the faulting module turns that into a
//! diagnosis. The Rust-panic analogue (a panic hook that tees into `tracing`) lives in `main()`.
// Every `unsafe` block in this file carries a `// SAFETY:` proof (unsafe-proof program).
#![deny(clippy::undocumented_unsafe_blocks)]
use windows::Win32::Foundation::HMODULE;
use windows::Win32::System::Diagnostics::Debug::{
SetUnhandledExceptionFilter, EXCEPTION_CONTINUE_SEARCH, EXCEPTION_POINTERS,
};
use windows::Win32::System::LibraryLoader::{
GetModuleFileNameW, GetModuleHandleExW, GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
};
/// Install the process-wide unhandled-exception filter. Call once at startup, after logging init
/// (the filter reports through `tracing`, so installing it earlier would log into the void).
pub fn install() {
// SAFETY: registers a process-wide top-level exception filter; `on_unhandled` is a plain
// `extern "system"` fn with the LPTOP_LEVEL_EXCEPTION_FILTER signature and static lifetime.
// The returned previous filter is deliberately dropped — we are the first/only installer.
unsafe {
SetUnhandledExceptionFilter(Some(on_unhandled));
}
}
/// STATUS_ACCESS_VIOLATION — the overwhelmingly common native-crash code; its first two
/// `ExceptionInformation` slots carry the access kind (0 read / 1 write / 8 execute) and the
/// target address, which we surface because they distinguish a wild pointer from a guard page.
const STATUS_ACCESS_VIOLATION: i32 = 0xC0000005u32 as i32;
/// The filter itself. Best-effort by design: it formats and logs, which allocates — if the crash
/// is heap corruption this can fault again, in which case the OS terminates us exactly as it was
/// about to anyway. Returns `EXCEPTION_CONTINUE_SEARCH` so default handling (WER / a debugger /
/// the service supervisor seeing the exit) still runs.
unsafe extern "system" fn on_unhandled(info: *const EXCEPTION_POINTERS) -> i32 {
let mut code: i32 = 0;
let mut addr: usize = 0;
let mut av_kind: Option<usize> = None;
let mut av_target: Option<usize> = None;
// SAFETY: `info` (and `ExceptionRecord`) are supplied by the OS for the duration of this
// callback; both are checked non-null before the read, and only plain fields are copied out.
unsafe {
if !info.is_null() && !(*info).ExceptionRecord.is_null() {
let r = &*(*info).ExceptionRecord;
code = r.ExceptionCode.0;
addr = r.ExceptionAddress as usize;
if code == STATUS_ACCESS_VIOLATION && r.NumberParameters >= 2 {
av_kind = Some(r.ExceptionInformation[0]);
av_target = Some(r.ExceptionInformation[1]);
}
}
}
let module = module_at(addr);
tracing::error!(
code = %format!("0x{:08x}", code as u32),
address = %format!("0x{addr:016x}"),
module = %module.as_deref().unwrap_or("<unknown>"),
av_kind = av_kind.map(|k| match k {
0 => "read",
1 => "write",
8 => "execute",
_ => "other",
}),
av_target = av_target.map(|t| format!("0x{t:016x}")),
"FATAL: unhandled native exception — the host process is about to die"
);
EXCEPTION_CONTINUE_SEARCH
}
/// Resolve the module (DLL/EXE) containing `addr` — the smoking gun that separates "our bug" from
/// "the GPU runtime crashed under us" (amfrt64.dll, atiumd64.dll, nvwgf2umx.dll, d3d11.dll, …).
fn module_at(addr: usize) -> Option<String> {
if addr == 0 {
return None;
}
let mut hmod = HMODULE::default();
// SAFETY: FROM_ADDRESS reinterprets the "module name" parameter as an address inside the
// module — `addr as *const u16` is exactly that; UNCHANGED_REFCOUNT means no AddRef, so the
// returned HMODULE needs no FreeLibrary. `&mut hmod` is a live out-pointer for the call.
unsafe {
GetModuleHandleExW(
GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
windows::core::PCWSTR(addr as *const u16),
&mut hmod,
)
.ok()?;
}
let mut buf = [0u16; 512];
// SAFETY: `hmod` is the module handle resolved above; `buf` is a live, writable slice for the
// duration of the call. Returns the number of UTF-16 units written (0 on failure).
let n = unsafe { GetModuleFileNameW(Some(hmod), &mut buf) } as usize;
(n > 0).then(|| String::from_utf16_lossy(&buf[..n.min(buf.len())]))
}