fix(ci): runner hygiene stops eating its own jobs #84
@@ -148,7 +148,12 @@ jobs:
|
||||
-t "$CI_REGISTRY_PUSH/${{ matrix.image }}:latest" \
|
||||
ci
|
||||
|
||||
# Gated like Build/Push: only the docker CLI needs this login (Reconcile and Tag-for-release
|
||||
# authenticate via curl -u), so a cache-hit job with nothing to push must not be able to fail
|
||||
# on a login it never uses — proven on run 16013, where a host with a misconfigured daemon
|
||||
# failed exactly here on a hit=true leg.
|
||||
- name: Log in to the LAN registry
|
||||
if: steps.exists.outputs.hit == 'false'
|
||||
run: |
|
||||
echo "$CI_REGISTRY_PASSWORD" | docker login "$CI_REGISTRY_PUSH" -u ci --password-stdin
|
||||
env:
|
||||
@@ -236,7 +241,9 @@ jobs:
|
||||
-t "$CI_REGISTRY_PUSH/$IMAGE:latest" \
|
||||
.
|
||||
|
||||
# Same gate as the builders job above: the login only serves Push.
|
||||
- name: Log in to the LAN registry
|
||||
if: steps.exists.outputs.hit == 'false'
|
||||
run: |
|
||||
echo "$CI_REGISTRY_PASSWORD" | docker login "$CI_REGISTRY_PUSH" -u ci --password-stdin
|
||||
env:
|
||||
|
||||
@@ -159,13 +159,15 @@ fn ensure_admin_only_source(dir: &Path) -> Result<()> {
|
||||
let verdict = (|| -> Result<()> {
|
||||
rc.ok().context("GetNamedSecurityInfoW(owner + DACL)")?;
|
||||
let privileged = privileged_sids()?;
|
||||
// SAFETY: `owner` points into the descriptor returned above and is valid for this scope.
|
||||
let is_privileged = |sid: PSID| -> bool {
|
||||
// SAFETY: every `sid` handed in points into the descriptor returned above (or at an
|
||||
// ACE inside it) and is valid for this scope; IsValidSid is itself the probe.
|
||||
if sid.is_invalid() || !unsafe { IsValidSid(sid) }.as_bool() {
|
||||
return false;
|
||||
}
|
||||
privileged
|
||||
.iter()
|
||||
// SAFETY: `sid` passed IsValidSid above; `p` is an owned, length-exact SID copy.
|
||||
.any(|p| unsafe { EqualSid(sid, PSID(p.as_ptr().cast_mut().cast())) }.is_ok())
|
||||
};
|
||||
|
||||
@@ -237,6 +239,7 @@ fn privileged_sids() -> Result<Vec<Vec<u8>>> {
|
||||
.with_context(|| format!("ConvertStringSidToSidW({s})"))?;
|
||||
// SAFETY: psid is a valid SID; copy it out so the caller owns plain bytes.
|
||||
let len = unsafe { GetLengthSid(psid) } as usize;
|
||||
// SAFETY: GetLengthSid just measured exactly `len` readable bytes at `psid`.
|
||||
let bytes = unsafe { std::slice::from_raw_parts(psid.0 as *const u8, len) }.to_vec();
|
||||
// SAFETY: ConvertStringSidToSidW allocates with LocalAlloc.
|
||||
unsafe {
|
||||
|
||||
+27
-10
@@ -1,11 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI runner disk hygiene — invoked by docker-prune.service (every 30 min). Lives in a real script
|
||||
# CI runner disk hygiene — invoked by docker-prune.service (every 2 min). Lives in a real script
|
||||
# rather than inline ExecStart= lines because systemd does its OWN $-expansion on ExecStart and
|
||||
# empties shell vars / $(...) before /bin/sh sees them (silently breaking the logic under `|| true`).
|
||||
#
|
||||
# See docker-prune.service for the full why. The headline: the act_runner cache server's blob store
|
||||
# lives INSIDE the long-running runner container's writable layer, where `docker prune` can't reach
|
||||
# it — left alone it grows to tens of GB and fills the disk on its own.
|
||||
# See docker-prune.service for the full why. Sibling: docker-reclaim.sh (hourly) handles what
|
||||
# act_runner *leaks* — per-job volumes, stale networks, old build cache. This one handles what
|
||||
# CI legitimately *produces* and then abandons: per-SHA app tags and the layers they pin.
|
||||
set -u
|
||||
export PATH=/usr/bin:/bin:/usr/local/bin:$PATH
|
||||
|
||||
@@ -23,11 +23,26 @@ MIN_FREE_GB=${MIN_FREE_GB:-60} # ...or this little is left, whichever t
|
||||
# 2026-07-29: zero burst clears fired in six hours
|
||||
# while deb still died of ENOSPC between polls.
|
||||
|
||||
# 1) Routine: trim aged images / build cache / stopped containers. sha-<commit> tags aren't
|
||||
# dangling, so -a is required. until=2h, not 6h: on a busy day every image is younger than six
|
||||
# hours, so the filter matched nothing and a run reclaimed 0B while `docker system df` was
|
||||
# reporting 20+ GB reclaimable. Two hours still protects a re-run of the push being worked on.
|
||||
docker image prune -af --filter until=2h || true
|
||||
# 1) Routine: retire aged per-SHA app tags, then sweep what untagging released.
|
||||
# ⚠ NEVER `docker image prune -a` on this tick. `until=` filters on image CREATION time, so a
|
||||
# CI *base* image (built days ago) that merely has no container this instant counts as "aged" —
|
||||
# including one a job JUST PULLED whose container does not exist yet. Measured 2026-08-07:
|
||||
# this tick ran 07:36:09–:29 and a rust job's `docker create` failed at 07:36:29 with
|
||||
# "No such image: …punktfunk-rust-ci:latest" — three sampled failures that morning, each
|
||||
# coinciding with a prune run to the second — and every idle base image was re-pulled within
|
||||
# minutes (4–7 GB each), churning the LAN registry for nothing.
|
||||
# The only tag debris this host actually accretes is the per-SHA app tags (web/docs — their
|
||||
# creation time IS the local build time, so a 2h age gate is exact), and a dangling-only prune
|
||||
# cannot touch a tagged image, so neither step can race a starting job.
|
||||
now=$(date +%s)
|
||||
docker images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep ':sha-' | while read -r ref; do
|
||||
created=$(docker image inspect -f '{{.Created}}' "$ref" 2>/dev/null) || continue
|
||||
cts=$(date -d "$created" +%s 2>/dev/null) || continue
|
||||
if [ $((now - cts)) -ge 7200 ]; then
|
||||
docker rmi "$ref" >/dev/null 2>&1 || true
|
||||
fi
|
||||
done
|
||||
docker image prune -f || true
|
||||
docker builder prune -af --filter until=2h || true
|
||||
docker buildx prune -af --filter until=2h || true
|
||||
docker container prune -f --filter until=2h || true
|
||||
@@ -44,7 +59,9 @@ docker network prune -f --filter until=2h || true
|
||||
# what matters is absolute headroom for three concurrent target/ dirs, not a ratio — and the
|
||||
# ratio moves whenever the disk is resized (it went 123 G -> 175 G on 2026-07-29) while the
|
||||
# headroom three jobs need does not. In-use images are protected by the daemon, so a burst clear
|
||||
# cannot pull the rug from a live job.
|
||||
# cannot pull the rug from a live job — but the blanket `-a` prune below CAN race an image that
|
||||
# is pulled-but-not-yet-created (the section 1 lesson). That narrow window is accepted HERE
|
||||
# only: when the alternative is every concurrent job dying of ENOSPC, one job re-pulling loses.
|
||||
PCT=$(df --output=pcent / | tr -dc '0-9')
|
||||
FREE_GB=$(df --output=avail -BG / | tr -dc '0-9')
|
||||
# Two flat tests into a flag rather than one multi-line `{ …; } || { …; }` condition: the brace-group
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# Hourly reclaim of Docker resources act_runner LEAKS (per-job volumes, stale networks, old build
|
||||
# cache). Sibling of docker-prune.service, which handles what CI legitimately produces and then
|
||||
# abandons; the split matters because this one must stay conservative enough to run while jobs are
|
||||
# live (dangling-only volumes, age-gated networks) — see docker-reclaim.sh for the full why.
|
||||
#
|
||||
# Install: see the header of docker-reclaim.sh (note the installed unit name is
|
||||
# ci-docker-reclaim.service — existing fleet hosts already run it under that name).
|
||||
|
||||
[Unit]
|
||||
Description=Reclaim disk leaked by Gitea act_runner (per-job volumes, networks, stale build cache)
|
||||
Documentation=https://git.unom.io/unom/punktfunk
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/ci-docker-reclaim.sh
|
||||
# Never let maintenance starve a running build.
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
# Reclaim the disk that Gitea act_runner leaks on this host.
|
||||
#
|
||||
# Why this exists: act_runner creates a per-job network and a pair of named volumes, and leaks both
|
||||
# when a job is killed or the runner restarts. By 2026-07-25 that had accumulated 252 unused volumes
|
||||
# (11.7 GB) and 94 stale networks — some dating to task 5626 while current tasks were ~25233 — and
|
||||
# concurrent builds then exhausted the disk, failing CI with "No space left on device" at both the
|
||||
# cargo and the Docker/overlayfs layer. The stale networks are also what once broke the docs deploy
|
||||
# by exhausting Docker's default address pool and swallowing the DMZ 192.168.50.0/24 range.
|
||||
#
|
||||
# This ran on home-runner-1 only, hand-installed; home-runner-2 went without it and by 2026-08-07
|
||||
# had re-accumulated 176 leaked volumes (~60 GB) + 22 GB build cache and spent two days failing
|
||||
# jobs at ENOSPC. Hence checked in: BOTH runner hosts install it, from here.
|
||||
#
|
||||
# Install on a runner host (root):
|
||||
# install -m755 scripts/ci/docker-reclaim.sh /usr/local/sbin/ci-docker-reclaim.sh
|
||||
# install -m644 scripts/ci/docker-reclaim.service /etc/systemd/system/ci-docker-reclaim.service
|
||||
# install -m644 scripts/ci/docker-reclaim.timer /etc/systemd/system/ci-docker-reclaim.timer
|
||||
# systemctl daemon-reload && systemctl enable --now ci-docker-reclaim.timer
|
||||
#
|
||||
# Deliberately NOT `docker volume prune -a`: that would also delete any intentional named volume
|
||||
# that merely has no container attached at the moment the timer fires — e.g. the `docker-mirror`
|
||||
# pull-through registry cache or the runner cache during a restart — silently destroying it. Only
|
||||
# volumes act_runner named are removed here.
|
||||
#
|
||||
# Also deliberately NOT pruning images: on this host the per-SHA CI tags share all their layers with
|
||||
# `:latest`, so removing them reclaims nothing while forcing re-pulls. `docker system df`'s
|
||||
# "RECLAIMABLE" column counts shared layers once per image and overstates the win badly.
|
||||
# (docker-prune.sh owns tag retirement — age-gated and never `image prune -a`, see its header.)
|
||||
set -uo pipefail
|
||||
|
||||
log() { echo "ci-docker-reclaim: $*"; }
|
||||
|
||||
before_avail=$(df --output=avail -BM / | tail -1 | tr -dc '0-9')
|
||||
|
||||
# 1. Leaked per-job volumes — dangling AND named by act_runner. In-use volumes are never listed as
|
||||
# dangling, so a running job's volumes cannot be hit.
|
||||
mapfile -t stale_vols < <(docker volume ls -qf dangling=true 2>/dev/null | grep '^GITEA-ACTIONS-TASK-' || true)
|
||||
if ((${#stale_vols[@]})); then
|
||||
printf '%s\n' "${stale_vols[@]}" | xargs -r docker volume rm >/dev/null 2>&1
|
||||
log "removed ${#stale_vols[@]} leaked act_runner volumes"
|
||||
else
|
||||
log "no leaked act_runner volumes"
|
||||
fi
|
||||
|
||||
# 2. Unused networks older than 2h — never touches a live job's network (it is in use), and the age
|
||||
# filter keeps a just-created one safe against a race with a starting job.
|
||||
net_out=$(docker network prune -f --filter until=2h 2>&1 | grep -c '^GITEA-ACTIONS' || true)
|
||||
log "removed ${net_out:-0} stale job networks"
|
||||
|
||||
# 3. Build cache older than 48h. Recent cache is what makes builds fast, so it is kept.
|
||||
cache_freed=$(docker builder prune -f --filter until=48h 2>&1 | awk '/^Total:/ {print $2}')
|
||||
log "build cache freed: ${cache_freed:-0B}"
|
||||
|
||||
after_avail=$(df --output=avail -BM / | tail -1 | tr -dc '0-9')
|
||||
log "avail ${before_avail}M -> ${after_avail}M (reclaimed $((after_avail - before_avail))M)"
|
||||
df -h / | tail -1 | sed 's/^/ci-docker-reclaim: /'
|
||||
@@ -0,0 +1,16 @@
|
||||
# Hourly is the right cadence for LEAKS: they only accrue when jobs die abnormally, and the
|
||||
# per-tick docker-prune.timer (every 2 min) already carries the burst guard for genuine
|
||||
# disk-pressure emergencies. Install: see the header of docker-reclaim.sh.
|
||||
|
||||
[Unit]
|
||||
Description=Hourly reclaim of act_runner-leaked Docker disk
|
||||
|
||||
[Timer]
|
||||
OnCalendar=hourly
|
||||
# Catch up after a reboot rather than waiting for the next slot.
|
||||
Persistent=true
|
||||
# Spread it off the hour so it does not collide with scheduled CI.
|
||||
RandomizedDelaySec=300
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Reference in New Issue
Block a user