diff --git a/scripts/ci/docker-prune.sh b/scripts/ci/docker-prune.sh index 05b80aaf..4207b22f 100644 --- a/scripts/ci/docker-prune.sh +++ b/scripts/ci/docker-prune.sh @@ -9,34 +9,48 @@ set -u export PATH=/usr/bin:/bin:/usr/local/bin:$PATH -RUNNER=$(docker ps -q -f name=gitea-runner-runner | head -1) -ACTCACHE=/root/.cache/actcache/cache # path INSIDE the runner container (HOME=/root there) -CAP_MB=20000 # clear the actcache once its blob dir exceeds ~20 GB -BURST_PCT=80 # full clear once the disk is this % full +# The cache-server's blob store is a HOST directory: the fleet runs a standalone cache-server +# service (compose.yml) that bind-mounts this path to /data, and every replica points at it over +# HTTP (`external_server`). It used to live inside a runner container's writable layer, which is +# why this reached in with `docker exec` — that is no longer where it is, and the container name it +# looked for (`gitea-runner-runner`) does not exist either now that the replicas are named +# `gitea-runner-fleet-runner-N-1`. Both halves silently did nothing: the filter matched zero +# containers, so the cap and the burst-clear below were dead code. A plain host path needs neither. +CACHE_DIR=${CACHE_DIR:-/home/runner/gitea-runner-fleet/cache} +CAP_MB=${CAP_MB:-20000} # clear the cache store once it exceeds ~20 GB +BURST_PCT=${BURST_PCT:-80} # full clear once the disk is this % full +MIN_FREE_GB=${MIN_FREE_GB:-45} # ...or this little is left, whichever trips first # 1) Routine: trim aged images / build cache / stopped containers. sha- tags aren't -# dangling, so -a is required; until=6h keeps very recent ones for quick re-runs. -docker image prune -af --filter until=6h || true -docker builder prune -af --filter until=6h || true -docker buildx prune -af --filter until=6h || true -docker container prune -f --filter until=6h || true +# dangling, so -a is required. until=2h, not 6h: on a busy day every image is younger than six +# hours, so the filter matched nothing and a run reclaimed 0B while `docker system df` was +# reporting 20+ GB reclaimable. Two hours still protects a re-run of the push being worked on. +docker image prune -af --filter until=2h || true +docker builder prune -af --filter until=2h || true +docker buildx prune -af --filter until=2h || true +docker container prune -f --filter until=2h || true -# 2) Cap the act_runner cache server store (the real disk filler). Clearing the blobs is safe — -# act_runner repopulates it and cache keys are content-hashed, so this only drops stale entries. -if [ -n "$RUNNER" ]; then - SZ=$(docker exec "$RUNNER" du -sm "$ACTCACHE" 2>/dev/null | cut -f1) +# 2) Cap the cache-server store. Clearing the blobs is safe — act_runner repopulates it and cache +# keys are content-hashed, so this only drops stale entries. +if [ -d "$CACHE_DIR" ]; then + SZ=$(du -sm "$CACHE_DIR" 2>/dev/null | cut -f1) if [ -n "${SZ:-}" ] && [ "$SZ" -ge "$CAP_MB" ]; then - docker exec "$RUNNER" sh -c "rm -rf $ACTCACHE/*" && echo "actcache cleared (was ${SZ} MB)" + rm -rf "${CACHE_DIR:?}"/* && echo "cache-server store cleared (was ${SZ} MB)" fi fi -# 3) Burst guard: a push-storm can fill the disk within one interval. Once >=BURST_PCT% full, prune -# ALL idle images/cache AND clear the actcache, regardless of age. In-use images are protected. +# 3) Burst guard: a push-storm fills the disk WITHIN one interval — three concurrent Rust builds, +# each with a multi-GB target/, on top of a ~40 GB containerd image baseline. Trigger on a free +# -space FLOOR as well as a percentage: the percentage alone is the wrong instrument here, +# because 80% of 123 G still leaves only ~25 G, which three jobs can swallow before the next +# poll. In-use images are protected by the daemon, so this cannot pull the rug from a live job. PCT=$(df --output=pcent / | tr -dc '0-9') -if [ -n "$PCT" ] && [ "$PCT" -ge "$BURST_PCT" ]; then - echo "disk ${PCT}% >= ${BURST_PCT}% — burst clear" +FREE_GB=$(df --output=avail -BG / | tr -dc '0-9') +if { [ -n "$PCT" ] && [ "$PCT" -ge "$BURST_PCT" ]; } || + { [ -n "$FREE_GB" ] && [ "$FREE_GB" -lt "$MIN_FREE_GB" ]; }; then + echo "disk ${PCT}% used, ${FREE_GB}G free (thresholds ${BURST_PCT}% / ${MIN_FREE_GB}G) — burst clear" docker image prune -af || true docker builder prune -af || true docker buildx prune -af || true - [ -n "$RUNNER" ] && docker exec "$RUNNER" sh -c "rm -rf $ACTCACHE/*" || true + [ -d "$CACHE_DIR" ] && rm -rf "${CACHE_DIR:?}"/* || true fi diff --git a/scripts/ci/docker-prune.timer b/scripts/ci/docker-prune.timer index dacb7842..ef78378b 100644 --- a/scripts/ci/docker-prune.timer +++ b/scripts/ci/docker-prune.timer @@ -1,13 +1,19 @@ -# Runs docker-prune.service every 30 min. The runner is hammered with build bursts that can refill -# the disk fast (and the actcache cap needs to react well within an hour), so 30 min beats hourly. +# Runs docker-prune.service every 10 min. It was every 30, and that is how the burst guard never +# fired once: three concurrent Rust builds fill the disk and drain it again well inside a 30-minute +# window, so the poll kept landing on a healthy `df` while jobs died of ENOSPC in between. (An arch +# build failed with `as: BFD assertion fail` — assembler noise for "can't write, No space left on +# device" — and by the time anyone looked, the disk was back to 37% used.) +# +# Ten minutes is mitigation, not a fix: the real levers are runner capacity (three replicas sharing +# one 123 G disk) or a bigger disk. RandomizedDelaySec stays low so the guard is prompt. # Persistent=true catches up after downtime. Install: see the header of docker-prune.service. [Unit] -Description=Run docker-prune every 30 min (CI runner disk hygiene + actcache cap + burst guard) +Description=Run docker-prune every 10 min (CI runner disk hygiene + cache cap + burst guard) [Timer] -OnCalendar=*:0/30 -RandomizedDelaySec=120 +OnCalendar=*:0/10 +RandomizedDelaySec=30 Persistent=true [Install]