Merge pull request 'v0.25.0 shipped an Arch host no up-to-date box can install — and the pipeline had no way to tell' (#109) from worktree-arch-ffmpeg9-repackage into main

Reviewed-on: #109

.gitea/workflows/arch.yml

@@ -48,7 +48,26 @@ on:
     # `punktfunk-canary` pacman repo as X.Y.Z-0.<run#> (sorts below the eventual X.Y.Z-1),
     # tags to `punktfunk` — separate repos, so neither channel can shadow the other.
     tags: ['v*']
+  # REBUILDING A PUBLISHED RELEASE, because on a rolling distro the ground moves under one.
+  # Arch went FFmpeg 8 -> 9 (every libav soname +1) four minutes before v0.25.0 was tagged, so
+  # the release's punktfunk-host was linked in a builder image that still had 8 and shipped
+  # `libavcodec.so=62-64`. No up-to-date Arch box can satisfy that — and pacman prepares the
+  # whole transaction at once, so it did not merely block our package, it blocked those users'
+  # entire `pacman -Syu`. The repair is a rebuild of the SAME upstream version at a HIGHER
+  # pkgrel; nothing else reaches a box that already has the broken build recorded in its db.
+  # The workflow file at the tag can never carry inputs added after it was tagged, so dispatch
+  # this from `main`: it checks the tag's SOURCE out, publishes to the STABLE repo, and
+  # replaces the release-page assets. Same lever for any future "the distro moved" rebuild.
   workflow_dispatch:
+    inputs:
+      release_tag:
+        description: 'Rebuild this published release (e.g. v0.25.0) into the stable `punktfunk` repo. Empty = ordinary canary build of the dispatched ref.'
+        required: false
+        default: ''
+      pkgrel:
+        description: 'pkgrel for that rebuild — MUST be above the published one (2, 3, …); a same-pkgrel republish is invisible to pacman. Ignored without release_tag.'
+        required: false
+        default: '2'
 
 env:
   REGISTRY: git.unom.io
@@ -94,7 +113,52 @@ jobs:
           }
           bun --version
 
+      # THE BUILDER'S FFmpeg IS PART OF THE PACKAGE CONTRACT, not merely a build detail.
+      # packaging/arch/PKGBUILD binds punktfunk-host to the exact libav sonames it linked
+      # (`libavcodec.so=63-64` …), so a builder one FFmpeg major behind Arch emits a package
+      # that NOBODY can install — and takes the user's whole `pacman -Syu` down with it, since
+      # pacman prepares the transaction as a unit. That is exactly how v0.25.0 shipped: PR #108
+      # re-keyed this image for FFmpeg 9, the release tag fired four minutes later, and the job
+      # still got the FFmpeg-8 `:latest`. The image is a cache and is allowed to lag — but never
+      # on this one axis. So heal it in-job and shout, instead of building a dead package.
+      # (Runs BEFORE checkout: a stale image should be repaired before anything depends on it.)
+      - name: FFmpeg soname parity with today's Arch (heals a stale builder image)
+        run: |
+          export LC_ALL=C   # `Provides` is a localized field name
+          # Piped (never a TTY here) pacman prints each field on ONE line, unwrapped.
+          sonames() { sed -n 's/^Provides *: *//p' | tr ' ' '\n' | grep -E '^lib(av|sw)[a-z]*\.so=' | sort | tr '\n' ' '; }
+          # A SEPARATE --dbpath: this refreshes only a throwaway view of the repos, so the
+          # container's own db never enters the partial-upgrade state a bare `pacman -Sy` leaves.
+          mkdir -p /tmp/pf-archsync
+          if ! pacman -Sy --dbpath /tmp/pf-archsync --logfile /dev/null >/dev/null 2>&1; then
+            echo "::warning::could not refresh the Arch db — skipping the FFmpeg parity check"
+            exit 0
+          fi
+          HAVE="$(pacman -Qi ffmpeg | sonames)"
+          WANT="$(pacman -Si --dbpath /tmp/pf-archsync ffmpeg | sonames)"
+          echo "builder ffmpeg $(pacman -Q ffmpeg | cut -d' ' -f2): $HAVE"
+          echo "arch    ffmpeg $(pacman -Si --dbpath /tmp/pf-archsync ffmpeg | sed -n 's/^Version *: *//p'): $WANT"
+          if [ "$HAVE" = "$WANT" ]; then
+            echo "OK: the builder links the FFmpeg every up-to-date Arch box already has"
+            exit 0
+          fi
+          echo "::warning::arch-ci is stale ACROSS AN FFMPEG SONAME BUMP — upgrading it for this run."
+          echo "::warning::Bump the 'refreshed:' date in ci/arch-ci.Dockerfile so the IMAGE carries it."
+          pacman -Syu --noconfirm || true
+          HAVE="$(pacman -Qi ffmpeg | sonames)"
+          if [ "$HAVE" != "$WANT" ]; then
+            echo "::error::builder still links $HAVE while Arch ships $WANT."
+            echo "::error::Building on would publish a package no Arch box can install."
+            exit 1
+          fi
+          echo "healed: builder now links $HAVE"
+
       - uses: actions/checkout@v4
+        with:
+          # A dispatched release rebuild takes its WORKFLOW from the ref you dispatch (the only
+          # way it can carry inputs the tag predates) and its SOURCE from the tag. Empty string
+          # = checkout's own default, i.e. the triggering ref, for every other trigger.
+          ref: ${{ github.event.inputs.release_tag }}
 
       # Cache cargo's git dir too, not just the registry: the workspace includes
       # clients/windows, whose windows-reactor/windows deps are git-pinned — cargo must CLONE
@@ -127,12 +191,30 @@ jobs:
         # Keep the leading `0.` — it is what sorts a canary BELOW the eventual `X.Y.Z-1` stable
         # release. (A pkgrel is digits+dots only, so `0.` is the only prefix available; raising
         # it to `1.` would sort canaries ABOVE the release and is not an option.)
+        env:
+          RELEASE_TAG: ${{ github.event.inputs.release_tag }}
+          REBUILD_PKGREL: ${{ github.event.inputs.pkgrel }}
         run: |
           eval "$(bash scripts/ci/pf-version.sh)"   # -> PF_BASE (one minor ahead of latest stable)
-          case "$GITHUB_REF" in
-            refs/tags/v*) V="${GITHUB_REF_NAME#v}"; R="1";                       REPO=punktfunk ;;
-            *)            V="$PF_BASE";  R="0.$(printf '%08d' "$GITHUB_RUN_NUMBER")";  REPO=punktfunk-canary ;;
-          esac
+          if [ -n "${RELEASE_TAG:-}" ]; then
+            # Dispatched rebuild of a published release (see the workflow_dispatch note at the
+            # top): same upstream version, higher pkgrel, straight into the stable repo.
+            # ⚠ Keep that pkgrel SINGLE-DIGIT. Gitea's Arch registry picks the version its .db
+            # advertises by STRING order (the same trap the canary zero-padding below exists for),
+            # so "0.25.0-10" sorts BELOW "0.25.0-2" and the rebuild would never be advertised.
+            V="${RELEASE_TAG#v}"
+            R="${REBUILD_PKGREL:-2}"
+            REPO=punktfunk
+            case "$R" in
+              ''|*[!0-9.]*) echo "::error::pkgrel '$R' is not digits+dots"; exit 1 ;;
+              1) echo "::error::pkgrel 1 is the published build — a rebuild MUST go up (2, 3, …)"; exit 1 ;;
+            esac
+          else
+            case "$GITHUB_REF" in
+              refs/tags/v*) V="${GITHUB_REF_NAME#v}"; R="1";                       REPO=punktfunk ;;
+              *)            V="$PF_BASE";  R="0.$(printf '%08d' "$GITHUB_RUN_NUMBER")";  REPO=punktfunk-canary ;;
+            esac
+          fi
           echo "PF_PKGVER=$V" >> "$GITHUB_ENV"
           echo "PF_PKGREL=$R" >> "$GITHUB_ENV"
           echo "REPO=$REPO" >> "$GITHUB_ENV"
@@ -235,6 +317,63 @@ jobs:
             rm -rf dist-gamescope   # never cache a failed build (an empty path is not saved)
           fi
 
+      # THE GATE THIS PIPELINE WAS MISSING. The soname assert above proves the libav dep is
+      # VERSIONED; it cannot prove the version is one that EXISTS. v0.25.0 passed it and still
+      # shipped `libavcodec.so=62-64` to a world that had moved to 63 — every affected user got
+      # "unable to satisfy dependency … required by punktfunk-host", and because pacman prepares
+      # one transaction, their whole system upgrade stopped there. So ask the only question that
+      # matters before publishing: would a real, up-to-date Arch box install this?
+      #
+      # An empty --dbpath is what makes the answer honest. It means "nothing is installed", so
+      # pacman must satisfy every dependency FROM THE REPOS exactly as a user's box does. Checking
+      # against the builder's own installed set instead would let a stale ffmpeg satisfy the stale
+      # bound and hide the break completely — the very illusion that shipped v0.25.0. `--print`
+      # resolves and prints; it downloads nothing and installs nothing. Verified against the real
+      # broken artifact on an ffmpeg-9 box: it reproduces the user-visible failure verbatim.
+      - name: Assert every package installs on an up-to-date Arch box
+        run: |
+          export LC_ALL=C
+          mkdir -p /tmp/pf-instcheck
+          if ! pacman -Sy --dbpath /tmp/pf-instcheck --logfile /dev/null >/dev/null 2>&1; then
+            echo "::error::could not sync the Arch db — cannot prove these packages install"
+            exit 1
+          fi
+          check() {   # check FILE -> 0 installable, 1 not (reason on stdout)
+            pacman -U --print --noconfirm --dbpath /tmp/pf-instcheck --logfile /dev/null "$1" 2>&1
+          }
+          ls dist/*.pkg.tar.zst >/dev/null 2>&1 || { echo "::error::nothing in dist/ to check"; exit 1; }
+          rc=0
+          for pkg in dist/*.pkg.tar.zst; do
+            if out="$(check "$pkg")"; then
+              echo "OK  $(basename "$pkg")  ($(echo "$out" | wc -l) targets resolve)"
+            else
+              rc=1
+              echo "::error::$(basename "$pkg") CANNOT be installed on an up-to-date Arch box:"
+              echo "$out" | sed 's/^/    /'
+            fi
+          done
+          # gamescope stays best-effort, exactly as its build step is: a companion that cannot
+          # install is dropped from the upload with a warning, never a reason to withhold the
+          # packages this workflow exists to publish. (It is also the one package that can be
+          # restored from a cache older than the current Arch snapshot.)
+          for pkg in dist-gamescope/*.pkg.tar.zst; do
+            [ -e "$pkg" ] || continue
+            if out="$(check "$pkg")"; then
+              echo "OK  $(basename "$pkg")  ($(echo "$out" | wc -l) targets resolve)"
+            else
+              echo "::warning::$(basename "$pkg") is not installable on current Arch — NOT publishing it"
+              echo "$out" | sed 's/^/    /'
+              rm -f "$pkg"
+            fi
+          done
+          if [ "$rc" != 0 ]; then
+            echo "::error::refusing to publish: pacman would reject this on a current box, and a"
+            echo "::error::rejected dependency blocks the user's ENTIRE upgrade, not just punktfunk."
+            echo "::error::Usual cause: the arch-ci builder image lags Arch across a soname bump —"
+            echo "::error::bump 'refreshed:' in ci/arch-ci.Dockerfile, let docker.yml republish it, re-run."
+            exit 1
+          fi
+
       # NOTE deliberately NO sysext image is built or published here: a prebuilt HOST binary on
       # SteamOS breaks on the next A/B soname bump (and /var — where sysexts live — is
       # per-partition-set), which is the standing packaging verdict behind the on-device
@@ -262,14 +401,29 @@ jobs:
           done
           echo "published to $OWNER/arch/$REPO"
 
-      # On a real release, also attach the packages to the unified Gitea Release.
-      - name: Attach packages to the Gitea release (stable tags only)
-        if: startsWith(gitea.ref, 'refs/tags/v')
+      # On a real release, also attach the packages to the unified Gitea Release. A dispatched
+      # rebuild attaches to that SAME release object: the release page is a distribution surface
+      # too, and leaving the superseded .pkg.tar.zst sitting on it is one click away from handing
+      # someone the exact break the rebuild exists to fix.
+      - name: Attach packages to the Gitea release (stable tags + release rebuilds)
+        if: startsWith(gitea.ref, 'refs/tags/v') || github.event.inputs.release_tag != ''
         env:
           GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
+          RELEASE_TAG: ${{ github.event.inputs.release_tag }}
         run: |
           . scripts/ci/gitea-release.sh
-          RID=$(ensure_release "$GITHUB_REF_NAME" "$GITHUB_REF_NAME" auto)
+          TAG="${RELEASE_TAG:-$GITHUB_REF_NAME}"
+          RID=$(ensure_release "$TAG" "$TAG" auto)
           for pkg in dist/*.pkg.tar.zst; do
             upsert_asset "$RID" "$pkg"
           done
+          # A rebuild bumps pkgrel, so its FILENAMES differ from the ones already attached, and
+          # upsert_asset only replaces by name — the superseded set would survive untouched.
+          # Drop every pacman asset (and .sha256 sidecar) this upload did not just write.
+          if [ -n "${RELEASE_TAG:-}" ]; then
+            KEEP="$(cd dist && printf '%s ' *.pkg.tar.zst)"
+            # An UNMATCHED glob would come through literally and match nothing in the keep set —
+            # i.e. "delete every pacman asset on the release". Empty out instead; prune refuses.
+            case "$KEEP" in *'*'*) KEEP="" ;; esac
+            prune_release_assets "$RID" .pkg.tar.zst "$KEEP"
+          fi

CHANGELOG.md

@@ -437,6 +437,24 @@ refuses the upgrade instead of bricking the install. All seven libs are listed e
 `--as-needed` currently drops two: an unlinked soname is left bare by makepkg and satisfied by any
 ffmpeg, so listing it costs nothing and a future link picks up the bound automatically.
 
+🛑 **The v0.25.0 Arch packages shipped with that bound pointing at the WRONG FFmpeg — install
+`punktfunk-host 0.25.0-2` or newer.** The soname fix and the FFmpeg-9 build landed as one merge;
+the release tag was pushed four minutes later, while the CI builder image was still being
+rebuilt. arch.yml deliberately runs no `-Syu` ("the image's snapshot IS the build environment"),
+so the release was linked against FFmpeg 8 and published `libavcodec.so=62-64` — a bound no
+up-to-date Arch box can satisfy. It fails *safely* (pacman refuses; nothing bricks), but it fails
+**loudly and broadly**: pacman prepares one transaction, so an unsatisfiable dependency of ours
+stopped affected users' entire `pacman -Syu`. `0.25.0-2` is the identical source rebuilt against
+FFmpeg 9. Only Arch was exposed — every other format derives its dependency from the ELF at build
+time and could not disagree with itself this way.
+
+Two guards now stand where only a convention did. arch.yml compares the builder's libav
+`provides` against the live repos before building and `-Syu`s itself if they differ; and no
+package is published until a **pristine-`--dbpath`** `pacman -U --print` resolves it, which asks
+"would a real, up-to-date Arch box install this?" instead of "does the builder happen to satisfy
+it?" — the distinction that let this ship. Keeping `ci/arch-ci.Dockerfile` current is still the
+cheap path; the guards are the backstop.
+
 ### Linux playback filled the buffer ceiling
 
 The PipeWire playback callback sized its writes from the mapped buffer's **capacity** — PipeWire's

ci/arch-ci.Dockerfile

@@ -19,6 +19,16 @@
 # 63-64), so it would simply refuse to install rather than start. Re-keying this image is the step
 # that makes the ffmpeg-9 bump actually reach the package — a Cargo.toml bump alone does nothing
 # here. Whenever Arch moves to an FFmpeg major, bump the date in the same commit.
+#
+# ⚠ AND KNOW WHY THAT WAS NOT ENOUGH: bumping this date only helps once docker.yml has actually
+# republished the image, and nothing sequences the two workflows. v0.25.0 was tagged four minutes
+# after the ffmpeg-9 merge, so the release build still pulled the FFmpeg-8 `:latest` and published
+# a punktfunk-host that no up-to-date Arch box could install — which blocks the user's ENTIRE
+# `pacman -Syu`, not just our package. arch.yml therefore no longer trusts this image on that one
+# axis: it compares the builder's libav sonames against the repos before building (and `-Syu`s
+# itself if they differ), and refuses to publish anything a pristine-db `pacman -U --print` says
+# is unsatisfiable. This file staying current is still the CHEAP path — those guards are the
+# backstop, not the plan.
 FROM docker.io/library/archlinux:base-devel
 
 # One transaction: the main build/runtime deps (first list) + the gamescope companion's

packaging/arch/README.md

@@ -56,9 +56,39 @@ sudo pacman -Sy punktfunk-web         # optional browser management console
 packages against the key you just trusted. Arch is rolling, so the packages are built against
 current Arch sonames — keep the box itself updated too.)
 
+Step 2 **appends**, so running it twice leaves two `[punktfunk]` blocks and every later pacman
+run opens with `error: could not register 'punktfunk' database (database already registered)`.
+It is harmless — pacman ignores the duplicate and carries on — but to silence it, delete the
+extra block from `/etc/pacman.conf`.
+
 Then the same first-run steps as a source build (printed by the install scriptlet): `input`
 group, `host.env`, `systemctl --user enable --now punktfunk-host` — see the next section.
 
+### If pacman says `unable to satisfy dependency 'libavcodec.so=…'`
+
+```
+:: unable to satisfy dependency 'libavcodec.so=62-64' required by punktfunk-host
+```
+
+`punktfunk-host` links FFmpeg, so it depends on the exact libav sonames it was built against —
+FFmpeg 8 provides `libavcodec.so=62`, FFmpeg 9 provides `libavcodec.so=63`. This message means the
+package on offer was built against a *different* FFmpeg major than your box has. Because pacman
+prepares the whole transaction at once, it stops your entire `pacman -Syu`, not just this package.
+
+The bound is deliberate. Without it the upgrade succeeds and leaves a host binary that cannot
+start at all — exit 127 before `main()`, in a systemd restart loop, with nothing in its own log
+to explain it (`ldd /usr/bin/punktfunk-host | grep 'not found'` is the one-line diagnosis).
+
+1. `sudo pacman -Syyu` — a forced db refresh, in case the matching build is already published.
+   Compare `pacman -Si punktfunk-host` against your `pacman -Q ffmpeg`.
+2. Still refused? Then we published a build made against the wrong FFmpeg — please report it. The
+   repair arrives as a higher **pkgrel** of the same version (`0.25.0-2`), so a later `-Syu`
+   picks it up with nothing to undo.
+3. To let the rest of the system upgrade in the meantime: `sudo pacman -Syu --ignore punktfunk-host`.
+   If pacman still refuses (your *installed* copy is the one carrying the bound), remove it with
+   `sudo pacman -Rdd punktfunk-host`, upgrade, and install it again once the rebuild lands. Either
+   way the host stays down until then — that is the soname break itself, not a second fault.
+
 ## Build from source — Arch Linux (mutable)
 
 ```sh

scripts/ci/gitea-release.sh

@@ -38,6 +38,18 @@ for a in json.load(sys.stdin):
         print(a.get("id",""));break' "$1" 2>/dev/null
 }
 _urlencode() { python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1],safe=""))' "$1"; }
+# _json_stale_asset_ids SUFFIX KEEP_NAMES  (assets JSON on stdin) -> "<id> <name>" lines
+#   The assets matching SUFFIX (or its .sha256 sidecar) that are NOT in the whitespace-separated
+#   KEEP_NAMES. See prune_release_assets.
+_json_stale_asset_ids() {
+  python3 -c 'import json,sys
+suffix, keep = sys.argv[1], set(sys.argv[2].split())
+keep |= {n + ".sha256" for n in keep}
+for a in json.load(sys.stdin):
+    n = a.get("name", "")
+    if n.endswith((suffix, suffix + ".sha256")) and n not in keep:
+        print(a.get("id", ""), n)' "$1" "$2" 2>/dev/null
+}
 
 # _release_notes_path TAG
 #   Print the path of the in-repo release notes for TAG (docs/releases/<TAG>.md) IFF it exists,
@@ -165,6 +177,33 @@ upsert_asset() {
   if _put_asset "$rid" "$sums" "$name.sha256"; then rm -f "$sums"; else rm -f "$sums"; return 1; fi
 }
 
+# prune_release_assets RELEASE_ID SUFFIX KEEP_NAMES
+#   Delete every asset of the release whose name ends in SUFFIX (or SUFFIX.sha256) and is not one
+#   of KEEP_NAMES (whitespace-separated).
+#
+#   WHY: upsert_asset replaces an asset BY NAME, which is idempotent only while the filename is
+#   stable. A REBUILD of an already-published release is exactly the case where it is not — a
+#   distro moved under the release, the artifact is rebuilt at a higher pkgrel, and
+#   `punktfunk-host-0.25.0-2-x86_64.pkg.tar.zst` collides with nothing, so the -1 build stays
+#   attached. A superseded package on a release page is not clutter; it is a live download of the
+#   very build the rebuild exists to replace. Scoped by SUFFIX because a release object is shared
+#   by ~8 packaging workflows running concurrently — each leg may only ever prune names it owns.
+prune_release_assets() {
+  local rid="${1:?release id}" suffix="${2:?suffix}" keep="${3:-}"
+  local api
+  # An empty keep list means "delete every asset matching SUFFIX", which is never what a caller
+  # wants and is exactly what a mis-expanded glob looks like. Refuse rather than clear a release.
+  [ -n "$keep" ] || { echo "gitea-release: prune_release_assets got an empty keep list — refusing" >&2; return 0; }
+  api="$(_gitea_api)"
+  curl -fsS "$api/releases/$rid/assets" -H "Authorization: token ${GITEA_TOKEN:?}" \
+    | _json_stale_asset_ids "$suffix" "$keep" \
+    | while read -r id name; do
+        echo "gitea-release: dropping superseded asset '$name'"
+        curl -fsS -o /dev/null -X DELETE "$api/releases/$rid/assets/$id" \
+          -H "Authorization: token ${GITEA_TOKEN:?}" || true
+      done
+}
+
 # apply_release_notes RELEASE_ID TAG
 #   Force the release body to match docs/releases/<TAG>.md (the source of truth), if that file
 #   exists — a no-op otherwise. PATCHes ONLY the body, so name/prerelease/assets are preserved