From c3cdee9bf54954912d3dc58404e25fe518dfb5c6 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 7 Aug 2026 10:10:59 +0200 Subject: [PATCH 1/3] fix(ci/prune): the 2-minute image prune stops deleting images mid-pull MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docker image prune -af --filter until=2h keyed on image CREATION time, so a base image built days ago that merely had no container at that instant was "aged" — including one a job had just pulled and not yet created. Measured 2026-08-07: three job failures, each coinciding with a prune tick to the second ("No such image: …punktfunk-rust-ci:latest", every step cancelled), plus a 4-7 GB re-pull of every idle base image within minutes. The routine tick now retires only what this host actually accretes — per-SHA app tags older than 2h (their creation time IS the local build time) — then sweeps dangling layers, which cannot touch a tagged image. The blanket -a prune survives only in the near-ENOSPC burst guard, where one re-pull beats every concurrent job dying. docker-reclaim.{sh,service,timer} are the hourly leak reclaimer that so far lived hand-installed on home-runner-1 only; home-runner-2 went without it and re-accumulated 176 leaked volumes (~60 GB) until jobs died of ENOSPC on 2026-08-06/07. Checked in so both hosts install the same files from here. --- scripts/ci/docker-prune.sh | 37 ++++++++++++++------ scripts/ci/docker-reclaim.service | 20 +++++++++++ scripts/ci/docker-reclaim.sh | 57 +++++++++++++++++++++++++++++++ scripts/ci/docker-reclaim.timer | 16 +++++++++ 4 files changed, 120 insertions(+), 10 deletions(-) create mode 100644 scripts/ci/docker-reclaim.service create mode 100644 scripts/ci/docker-reclaim.sh create mode 100644 scripts/ci/docker-reclaim.timer diff --git a/scripts/ci/docker-prune.sh b/scripts/ci/docker-prune.sh index b6b4f4dc..bff35bf6 100644 --- a/scripts/ci/docker-prune.sh +++ b/scripts/ci/docker-prune.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash -# CI runner disk hygiene — invoked by docker-prune.service (every 30 min). Lives in a real script +# CI runner disk hygiene — invoked by docker-prune.service (every 2 min). Lives in a real script # rather than inline ExecStart= lines because systemd does its OWN $-expansion on ExecStart and # empties shell vars / $(...) before /bin/sh sees them (silently breaking the logic under `|| true`). # -# See docker-prune.service for the full why. The headline: the act_runner cache server's blob store -# lives INSIDE the long-running runner container's writable layer, where `docker prune` can't reach -# it — left alone it grows to tens of GB and fills the disk on its own. +# See docker-prune.service for the full why. Sibling: docker-reclaim.sh (hourly) handles what +# act_runner *leaks* — per-job volumes, stale networks, old build cache. This one handles what +# CI legitimately *produces* and then abandons: per-SHA app tags and the layers they pin. set -u export PATH=/usr/bin:/bin:/usr/local/bin:$PATH @@ -23,11 +23,26 @@ MIN_FREE_GB=${MIN_FREE_GB:-60} # ...or this little is left, whichever t # 2026-07-29: zero burst clears fired in six hours # while deb still died of ENOSPC between polls. -# 1) Routine: trim aged images / build cache / stopped containers. sha- tags aren't -# dangling, so -a is required. until=2h, not 6h: on a busy day every image is younger than six -# hours, so the filter matched nothing and a run reclaimed 0B while `docker system df` was -# reporting 20+ GB reclaimable. Two hours still protects a re-run of the push being worked on. -docker image prune -af --filter until=2h || true +# 1) Routine: retire aged per-SHA app tags, then sweep what untagging released. +# ⚠ NEVER `docker image prune -a` on this tick. `until=` filters on image CREATION time, so a +# CI *base* image (built days ago) that merely has no container this instant counts as "aged" — +# including one a job JUST PULLED whose container does not exist yet. Measured 2026-08-07: +# this tick ran 07:36:09–:29 and a rust job's `docker create` failed at 07:36:29 with +# "No such image: …punktfunk-rust-ci:latest" — three sampled failures that morning, each +# coinciding with a prune run to the second — and every idle base image was re-pulled within +# minutes (4–7 GB each), churning the LAN registry for nothing. +# The only tag debris this host actually accretes is the per-SHA app tags (web/docs — their +# creation time IS the local build time, so a 2h age gate is exact), and a dangling-only prune +# cannot touch a tagged image, so neither step can race a starting job. +now=$(date +%s) +docker images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep ':sha-' | while read -r ref; do + created=$(docker image inspect -f '{{.Created}}' "$ref" 2>/dev/null) || continue + cts=$(date -d "$created" +%s 2>/dev/null) || continue + if [ $((now - cts)) -ge 7200 ]; then + docker rmi "$ref" >/dev/null 2>&1 || true + fi +done +docker image prune -f || true docker builder prune -af --filter until=2h || true docker buildx prune -af --filter until=2h || true docker container prune -f --filter until=2h || true @@ -44,7 +59,9 @@ docker network prune -f --filter until=2h || true # what matters is absolute headroom for three concurrent target/ dirs, not a ratio — and the # ratio moves whenever the disk is resized (it went 123 G -> 175 G on 2026-07-29) while the # headroom three jobs need does not. In-use images are protected by the daemon, so a burst clear -# cannot pull the rug from a live job. +# cannot pull the rug from a live job — but the blanket `-a` prune below CAN race an image that +# is pulled-but-not-yet-created (the section 1 lesson). That narrow window is accepted HERE +# only: when the alternative is every concurrent job dying of ENOSPC, one job re-pulling loses. PCT=$(df --output=pcent / | tr -dc '0-9') FREE_GB=$(df --output=avail -BG / | tr -dc '0-9') # Two flat tests into a flag rather than one multi-line `{ …; } || { …; }` condition: the brace-group diff --git a/scripts/ci/docker-reclaim.service b/scripts/ci/docker-reclaim.service new file mode 100644 index 00000000..7e794407 --- /dev/null +++ b/scripts/ci/docker-reclaim.service @@ -0,0 +1,20 @@ +# Hourly reclaim of Docker resources act_runner LEAKS (per-job volumes, stale networks, old build +# cache). Sibling of docker-prune.service, which handles what CI legitimately produces and then +# abandons; the split matters because this one must stay conservative enough to run while jobs are +# live (dangling-only volumes, age-gated networks) — see docker-reclaim.sh for the full why. +# +# Install: see the header of docker-reclaim.sh (note the installed unit name is +# ci-docker-reclaim.service — existing fleet hosts already run it under that name). + +[Unit] +Description=Reclaim disk leaked by Gitea act_runner (per-job volumes, networks, stale build cache) +Documentation=https://git.unom.io/unom/punktfunk +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/ci-docker-reclaim.sh +# Never let maintenance starve a running build. +Nice=10 +IOSchedulingClass=idle diff --git a/scripts/ci/docker-reclaim.sh b/scripts/ci/docker-reclaim.sh new file mode 100644 index 00000000..9b50423f --- /dev/null +++ b/scripts/ci/docker-reclaim.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Reclaim the disk that Gitea act_runner leaks on this host. +# +# Why this exists: act_runner creates a per-job network and a pair of named volumes, and leaks both +# when a job is killed or the runner restarts. By 2026-07-25 that had accumulated 252 unused volumes +# (11.7 GB) and 94 stale networks — some dating to task 5626 while current tasks were ~25233 — and +# concurrent builds then exhausted the disk, failing CI with "No space left on device" at both the +# cargo and the Docker/overlayfs layer. The stale networks are also what once broke the docs deploy +# by exhausting Docker's default address pool and swallowing the DMZ 192.168.50.0/24 range. +# +# This ran on home-runner-1 only, hand-installed; home-runner-2 went without it and by 2026-08-07 +# had re-accumulated 176 leaked volumes (~60 GB) + 22 GB build cache and spent two days failing +# jobs at ENOSPC. Hence checked in: BOTH runner hosts install it, from here. +# +# Install on a runner host (root): +# install -m755 scripts/ci/docker-reclaim.sh /usr/local/sbin/ci-docker-reclaim.sh +# install -m644 scripts/ci/docker-reclaim.service /etc/systemd/system/ci-docker-reclaim.service +# install -m644 scripts/ci/docker-reclaim.timer /etc/systemd/system/ci-docker-reclaim.timer +# systemctl daemon-reload && systemctl enable --now ci-docker-reclaim.timer +# +# Deliberately NOT `docker volume prune -a`: that would also delete any intentional named volume +# that merely has no container attached at the moment the timer fires — e.g. the `docker-mirror` +# pull-through registry cache or the runner cache during a restart — silently destroying it. Only +# volumes act_runner named are removed here. +# +# Also deliberately NOT pruning images: on this host the per-SHA CI tags share all their layers with +# `:latest`, so removing them reclaims nothing while forcing re-pulls. `docker system df`'s +# "RECLAIMABLE" column counts shared layers once per image and overstates the win badly. +# (docker-prune.sh owns tag retirement — age-gated and never `image prune -a`, see its header.) +set -uo pipefail + +log() { echo "ci-docker-reclaim: $*"; } + +before_avail=$(df --output=avail -BM / | tail -1 | tr -dc '0-9') + +# 1. Leaked per-job volumes — dangling AND named by act_runner. In-use volumes are never listed as +# dangling, so a running job's volumes cannot be hit. +mapfile -t stale_vols < <(docker volume ls -qf dangling=true 2>/dev/null | grep '^GITEA-ACTIONS-TASK-' || true) +if ((${#stale_vols[@]})); then + printf '%s\n' "${stale_vols[@]}" | xargs -r docker volume rm >/dev/null 2>&1 + log "removed ${#stale_vols[@]} leaked act_runner volumes" +else + log "no leaked act_runner volumes" +fi + +# 2. Unused networks older than 2h — never touches a live job's network (it is in use), and the age +# filter keeps a just-created one safe against a race with a starting job. +net_out=$(docker network prune -f --filter until=2h 2>&1 | grep -c '^GITEA-ACTIONS' || true) +log "removed ${net_out:-0} stale job networks" + +# 3. Build cache older than 48h. Recent cache is what makes builds fast, so it is kept. +cache_freed=$(docker builder prune -f --filter until=48h 2>&1 | awk '/^Total:/ {print $2}') +log "build cache freed: ${cache_freed:-0B}" + +after_avail=$(df --output=avail -BM / | tail -1 | tr -dc '0-9') +log "avail ${before_avail}M -> ${after_avail}M (reclaimed $((after_avail - before_avail))M)" +df -h / | tail -1 | sed 's/^/ci-docker-reclaim: /' diff --git a/scripts/ci/docker-reclaim.timer b/scripts/ci/docker-reclaim.timer new file mode 100644 index 00000000..ad4edfab --- /dev/null +++ b/scripts/ci/docker-reclaim.timer @@ -0,0 +1,16 @@ +# Hourly is the right cadence for LEAKS: they only accrue when jobs die abnormally, and the +# per-tick docker-prune.timer (every 2 min) already carries the burst guard for genuine +# disk-pressure emergencies. Install: see the header of docker-reclaim.sh. + +[Unit] +Description=Hourly reclaim of act_runner-leaked Docker disk + +[Timer] +OnCalendar=hourly +# Catch up after a reboot rather than waiting for the next slot. +Persistent=true +# Spread it off the hour so it does not collide with scheduled CI. +RandomizedDelaySec=300 + +[Install] +WantedBy=timers.target -- 2.54.0 From 39cfb7234cef8a95b3912d4c1f14627435d1110f Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 7 Aug 2026 10:11:02 +0200 Subject: [PATCH 2/3] fix(ci/docker): a cache-hit builders job stops failing on a login it never uses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The LAN-registry docker login only serves the Push step (Reconcile and Tag-for-release authenticate via curl -u), but it ran unguarded — so a hit=true leg landing on a host with a misconfigured docker daemon failed at login with nothing to push (run 16044/16013 f44 leg). Gate it like Build/Push. --- .gitea/workflows/docker.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitea/workflows/docker.yml b/.gitea/workflows/docker.yml index 8cb826d6..003f4491 100644 --- a/.gitea/workflows/docker.yml +++ b/.gitea/workflows/docker.yml @@ -148,7 +148,12 @@ jobs: -t "$CI_REGISTRY_PUSH/${{ matrix.image }}:latest" \ ci + # Gated like Build/Push: only the docker CLI needs this login (Reconcile and Tag-for-release + # authenticate via curl -u), so a cache-hit job with nothing to push must not be able to fail + # on a login it never uses — proven on run 16013, where a host with a misconfigured daemon + # failed exactly here on a hit=true leg. - name: Log in to the LAN registry + if: steps.exists.outputs.hit == 'false' run: | echo "$CI_REGISTRY_PASSWORD" | docker login "$CI_REGISTRY_PUSH" -u ci --password-stdin env: @@ -236,7 +241,9 @@ jobs: -t "$CI_REGISTRY_PUSH/$IMAGE:latest" \ . + # Same gate as the builders job above: the login only serves Push. - name: Log in to the LAN registry + if: steps.exists.outputs.hit == 'false' run: | echo "$CI_REGISTRY_PASSWORD" | docker login "$CI_REGISTRY_PUSH" -u ci --password-stdin env: -- 2.54.0 From 138a1f1b2ff97aa4391089f22c4a1a25c1b46b87 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 7 Aug 2026 10:11:04 +0200 Subject: [PATCH 3/3] fix(host/windows): the staging-dir SID checks document their unsafe blocks clippy's undocumented_unsafe_blocks (deny) flagged the three blocks that 81039581 introduced: the SAFETY comment sat outside the closure, so IsValidSid/EqualSid inside it read as undocumented, and from_raw_parts shared a comment that only covered the GetLengthSid line above it. Windows host clippy is the only leg that lints this cfg(windows) code, red since. --- crates/punktfunk-host/src/windows/install.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/punktfunk-host/src/windows/install.rs b/crates/punktfunk-host/src/windows/install.rs index a1f04552..3c60e1de 100644 --- a/crates/punktfunk-host/src/windows/install.rs +++ b/crates/punktfunk-host/src/windows/install.rs @@ -159,13 +159,15 @@ fn ensure_admin_only_source(dir: &Path) -> Result<()> { let verdict = (|| -> Result<()> { rc.ok().context("GetNamedSecurityInfoW(owner + DACL)")?; let privileged = privileged_sids()?; - // SAFETY: `owner` points into the descriptor returned above and is valid for this scope. let is_privileged = |sid: PSID| -> bool { + // SAFETY: every `sid` handed in points into the descriptor returned above (or at an + // ACE inside it) and is valid for this scope; IsValidSid is itself the probe. if sid.is_invalid() || !unsafe { IsValidSid(sid) }.as_bool() { return false; } privileged .iter() + // SAFETY: `sid` passed IsValidSid above; `p` is an owned, length-exact SID copy. .any(|p| unsafe { EqualSid(sid, PSID(p.as_ptr().cast_mut().cast())) }.is_ok()) }; @@ -237,6 +239,7 @@ fn privileged_sids() -> Result>> { .with_context(|| format!("ConvertStringSidToSidW({s})"))?; // SAFETY: psid is a valid SID; copy it out so the caller owns plain bytes. let len = unsafe { GetLengthSid(psid) } as usize; + // SAFETY: GetLengthSid just measured exactly `len` readable bytes at `psid`. let bytes = unsafe { std::slice::from_raw_parts(psid.0 as *const u8, len) }.to_vec(); // SAFETY: ConvertStringSidToSidW allocates with LocalAlloc. unsafe { -- 2.54.0