From 51d5f6cb2998c6724948c48adbc7091c198bbeab Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Thu, 30 Jul 2026 16:12:22 +0200 Subject: [PATCH] =?UTF-8?q?feat(host/windows):=20boot-loop=20auto-rollback?= =?UTF-8?q?=20=E2=80=94=20the=20supervisor=20re-runs=20the=20cached=20prev?= =?UTF-8?q?ious=20installer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The U3.2 leg of planning:host-update-from-web-console.md. The service worker loop — the one piece that survives an upgrade — detects a fresh update-intent whose target version is the crash-looping child (≥3 rapid restarts, intent <30 min), then exactly once: picks the newest cached non-target installer, requires a valid Authenticode signature, writes a rolled-back result record for the console, deletes the intent, and spawns the installer silently. No cached installer or a failing signature degrade to today's backoff loop with a loud log — never a brick, never an installer loop. Co-Authored-By: Claude Fable 5 --- crates/punktfunk-host/src/update.rs | 2 +- crates/punktfunk-host/src/update/windows.rs | 5 +- crates/punktfunk-host/src/windows/service.rs | 108 +++++++++++++++++++ 3 files changed, 112 insertions(+), 3 deletions(-) diff --git a/crates/punktfunk-host/src/update.rs b/crates/punktfunk-host/src/update.rs index 9b335db0..886a05ff 100644 --- a/crates/punktfunk-host/src/update.rs +++ b/crates/punktfunk-host/src/update.rs @@ -22,7 +22,7 @@ pub(crate) mod jobs; mod linux; pub(crate) mod manifest; #[cfg(target_os = "windows")] -mod windows; +pub(crate) mod windows; use crate::store::index::PublicKey; use manifest::Manifest; diff --git a/crates/punktfunk-host/src/update/windows.rs b/crates/punktfunk-host/src/update/windows.rs index daecca16..4e6776bc 100644 --- a/crates/punktfunk-host/src/update/windows.rs +++ b/crates/punktfunk-host/src/update/windows.rs @@ -249,8 +249,9 @@ fn preflight_disk(at: &Path, needed: u64) -> Result<(), String> { /// Authenticode: valid embedded signature (untrusted root tolerated — self-signed `CN=unom`), /// signing-leaf SHA-256 ∈ `pins` when pins are present. The leaf comes out of the same -/// `WinVerifyTrust` state via `WTHelperGetProvSignerFromChain`. -fn verify_authenticode(path: &Path, pins: &[String]) -> Result<(), String> { +/// `WinVerifyTrust` state via `WTHelperGetProvSignerFromChain`. (`pub(crate)`: the service +/// supervisor's boot-loop rollback re-checks the cached previous installer with it.) +pub(crate) fn verify_authenticode(path: &Path, pins: &[String]) -> Result<(), String> { use windows::core::{GUID, PCWSTR}; use windows::Win32::Foundation::{CERT_E_UNTRUSTEDROOT, S_OK}; use windows::Win32::Security::WinTrust::{ diff --git a/crates/punktfunk-host/src/windows/service.rs b/crates/punktfunk-host/src/windows/service.rs index 2bfcf5ea..f7e635f4 100644 --- a/crates/punktfunk-host/src/windows/service.rs +++ b/crates/punktfunk-host/src/windows/service.rs @@ -360,6 +360,9 @@ fn supervise(stop: HANDLE, session_ev: HANDLE) -> Result<()> { let job = make_job().context("create job object")?; let mut restarts: u32 = 0; + // One-shot latch for the update boot-loop rollback (plan U3.2) — a rollback that itself + // fails must not spawn installers in a loop. + let mut rollback_attempted = false; loop { if wait_one(stop, 0) { break; @@ -462,6 +465,7 @@ fn supervise(stop: HANDLE, session_ev: HANDLE) -> Result<()> { } restarts += 1; + maybe_boot_loop_rollback(restarts, &mut rollback_attempted); let backoff = restarts.min(10) * 500; // 0.5s..5s if wait_one(stop, backoff) { break; @@ -1129,3 +1133,107 @@ fn run_quiet(cmd: &str, args: &[&str]) -> bool { .map(|s| s.success()) .unwrap_or(false) } + +/// Update boot-loop rollback (planning: host-update-from-web-console.md §6, plan U3.2): a +/// FRESH update-intent record plus a crash-looping child that IS the intent's target version +/// means the just-installed host doesn't stay up. The supervisor — which survives the upgrade, +/// making it the right place — rolls back by re-running the CACHED previous installer: once +/// per intent, Authenticode-checked, outcome written where the console reads it. The service +/// process is not inside the kill-on-close job, so the spawned installer survives the service +/// stop it is about to perform. +fn maybe_boot_loop_rollback(restarts: u32, attempted: &mut bool) { + if *attempted || restarts < 3 { + return; + } + let intent_path = crate::update::jobs::intent_path(); + let Some(intent) = crate::update::jobs::read_intent(&intent_path) else { + return; + }; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + // A stale intent never triggers rollbacks, and a boot-looping OLD binary (version ≠ the + // intent's target) is not an update failure — boot reconciliation owns that story. + if now.saturating_sub(intent.started_unix) > 30 * 60 || env!("PUNKTFUNK_VERSION") != intent.to { + return; + } + *attempted = true; + + let updates = pf_paths::config_dir().join("updates"); + let previous = std::fs::read_dir(&updates) + .ok() + .into_iter() + .flatten() + .flatten() + .map(|e| e.path()) + .filter(|p| { + p.file_name() + .and_then(|n| n.to_str()) + .map(|n| { + n.starts_with("punktfunk-host-setup-") + && n.ends_with(".exe") + && !n.contains(intent.to.as_str()) + }) + .unwrap_or(false) + }) + .max_by_key(|p| { + p.metadata() + .and_then(|m| m.modified()) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH) + }); + let Some(previous) = previous else { + tracing::error!( + to = %intent.to, + "updated host is crash-looping and no cached previous installer exists — \ + leaving the intent for reconcile; manual reinstall required" + ); + return; + }; + // Validity-only Authenticode check: the failed update's manifest pins are gone with it, + // and the cached file was fully verified (manifest sha256 + pins) when first downloaded. + if let Err(e) = crate::update::windows::verify_authenticode(&previous, &[]) { + tracing::error!( + installer = %previous.display(), + error = %e, + "cached previous installer fails its signature check — not rolling back" + ); + return; + } + + let log = pf_paths::config_dir() + .join("logs") + .join(format!("update-rollback-from-{}.log", intent.to)); + let record = crate::update::jobs::ResultRecord { + ok: false, + from: intent.from.clone(), + to: intent.to.clone(), + finished_unix: now, + stage: Some("rolled-back".into()), + error: Some(format!( + "the updated host crash-looped after install; rolled back via {}", + previous.display() + )), + log_path: Some(log.display().to_string()), + staged: false, + }; + let _ = crate::update::jobs::write_json_atomic(&crate::update::jobs::result_path(), &record); + // Delete the intent BEFORE spawning: the incoming (previous) host must boot clean, and a + // missing intent keeps this one-shot even across a service restart. + let _ = std::fs::remove_file(&intent_path); + + tracing::warn!( + failed_version = %intent.to, + back_to = %previous.display(), + "updated host is crash-looping — rolling back via the cached previous installer" + ); + match std::process::Command::new(&previous) + .args(["/VERYSILENT", "/SUPPRESSMSGBOXES", "/NORESTART", "/SP-"]) + .arg(format!("/LOG={}", log.display())) + .spawn() + { + // Detached on purpose: it stops this service and reinstalls the previous version. + Ok(child) => drop(child), + Err(e) => tracing::error!(error = %e, "failed to spawn the rollback installer"), + } +}