diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index fba4d909..159f18e9 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -275,3 +275,31 @@ jobs: run: bun run build - name: Typecheck run: bun run lint + + # web/bun.nix and sdk/bun.nix are GENERATED from their bun.lock (bun2nix) and committed; the Nix + # build fetches node_modules from nothing else. They regenerate only on a local `bun install` that + # runs lifecycle scripts — never under CI's `--ignore-scripts`, and never on a merge or rebase, + # which happily carries a lockfile change past a bun.nix generated before it. That is not + # theoretical: web/bun.nix sat stale on main for 553 commits (2026-07-27 → 2026-08-05) with + # `nix build .#punktfunk-web` broken, and was repaired only by accident when an advisory bump + # happened to rerun a real `bun install`. + # + # Deliberately UNFILTERED and in ci.yml rather than nix.yml: it needs no Nix, takes well under a + # minute, and the whole point is that the drift arrives through commits that look unrelated to + # Nix. The Nix-toolchain gates (flake eval + building the bun packages) live in nix.yml. + bun-nix: + runs-on: ubuntu-24.04 + container: + image: oven/bun:1 + timeout-minutes: 15 + steps: + # oven/bun ships neither git nor a real node, and the slim base has no CA bundle — + # actions/checkout needs all three (see the web job). + - name: Install git + node + CA certs + run: apt-get update && apt-get install -y --no-install-recommends ca-certificates git nodejs + - uses: actions/checkout@v4 + # Regenerates each bun.nix from its committed bun.lock and diffs, and checks that the + # bun2nix version pin agrees across flake.nix and both package.json files (bun.nix has no + # schema stability across bun2nix releases). Fix with: scripts/ci/check-bun-nix.sh --fix + - name: bun.nix drift gate + run: sh scripts/ci/check-bun-nix.sh diff --git a/.gitea/workflows/nix.yml b/.gitea/workflows/nix.yml new file mode 100644 index 00000000..b9d6a41f --- /dev/null +++ b/.gitea/workflows/nix.yml @@ -0,0 +1,132 @@ +# Nix packaging gate. Until this existed, NOTHING in CI ever evaluated flake.nix: the word "nix" +# appeared in exactly one workflow file, and only in a comment about bun2nix breaking a Windows +# step. Every Nix regression therefore reached main invisibly and was found by hand on a Nix box — +# `nix build .#punktfunk-web` was broken for 553 commits before anyone noticed (see the bun-nix job +# in ci.yml for that story). +# +# Two tiers, because a full `nix flake check` builds the whole Rust workspace with crane and would +# run for an hour on every push: +# +# * eval — `nix flake check --no-build`: instantiates every package, app, check, devShell and +# the NixOS module without building them. Catches the failures that actually happen to +# this flake — a renamed file, a callPackage argument that no longer exists, a syntax +# error, a package attribute dropped from packages.nix. +# * bun — actually BUILDS punktfunk-web + punktfunk-scripting. These are the two derivations +# whose inputs churn constantly (every dependency bump moves a lockfile) and they cost +# minutes, not hours, because neither compiles Rust. This is the end-to-end proof that +# the generated bun.nix really does materialise a working node_modules offline — it +# covers what the ci.yml drift gate cannot, e.g. a tarball the registry no longer +# serves, or the codegen going quietly message-less (see packages.nix's inlang note). +# +# The Rust packages (punktfunk-host, punktfunk-client) and punktfunk-gamescope are NOT built here. +# They are the expensive ones and their inputs are already gated by the `rust` job in ci.yml; build +# them by hand on a Nix box, or with the `build-rust` dispatch input below. +# +# ⚠ pull_request is deliberately present. flatpak.yml shipped with push-only triggers and manifest +# breakage reached main invisibly for weeks — do not "simplify" this workflow by dropping it. +# ⚠ The two path lists are duplicated on purpose: a YAML anchor would be tidier, but Gitea's +# workflow parser is not a place to bet on anchor support. Keep them in step by hand. +name: nix + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: [main] + paths: + - "flake.nix" + - "flake.lock" + - "packaging/nix/**" + - "**/bun.lock" + - "**/bun.nix" + - "**/package.json" + - "Cargo.lock" + - "Cargo.toml" + - "rust-toolchain.toml" + - ".gitea/workflows/nix.yml" + - "scripts/ci/check-bun-nix.sh" + pull_request: + paths: + - "flake.nix" + - "flake.lock" + - "packaging/nix/**" + - "**/bun.lock" + - "**/bun.nix" + - "**/package.json" + - "Cargo.lock" + - "Cargo.toml" + - "rust-toolchain.toml" + - ".gitea/workflows/nix.yml" + - "scripts/ci/check-bun-nix.sh" + workflow_dispatch: + inputs: + build-rust: + description: "Also build punktfunk-host + punktfunk-client (slow: full Rust workspace)" + type: boolean + default: false + +jobs: + flake: + runs-on: ubuntu-24.04 + container: + # Official Nix image (Docker Hub, like this fleet's other WAN images: oven/bun:1, + # fedora:43, node:22-bookworm). It ships nix and little else. + image: nixos/nix:latest + timeout-minutes: 90 + env: + # The image defaults to stable Nix with the experimental features off; the flake needs both. + # Set at job level so every step — including the `nix profile install` below — sees it. + NIX_CONFIG: "experimental-features = nix-command flakes" + steps: + # actions/checkout is a JS action and needs node; a plain `run:` step executes through the + # container shell, so this must come BEFORE the checkout (same ordering, and the same + # reason, as flatpak.yml's fedora job). + - name: node + git for the JS actions + run: nix profile install nixpkgs#nodejs nixpkgs#git + + - uses: actions/checkout@v4 + + # Nix reads the flake through libgit2 and refuses a checkout owned by another uid + # ("detected dubious ownership"), which is the normal case for a container job. + - name: Trust the checkout + run: git config --global --add safe.directory "$PWD" + + # First-run diagnostics — cheap, and the difference between "the gate found a real problem" + # and "the runner had no disk" is otherwise a guess. + - name: Environment + run: | + nix --version + df -h /nix /tmp || true + + # Evaluates + instantiates every flake output without building any of it. + - name: nix flake check (eval only) + run: nix flake check --no-build --show-trace + + # The bun packages, built for real. This is the leg that would have caught the stale + # web/bun.nix end to end: the derivation's offline `bun install` runs against a store cache + # built strictly from bun.nix, so a lockfile that cache does not cover fails here. + - name: Build the bun packages + run: nix build --print-build-logs .#punktfunk-web .#punktfunk-scripting + + # Both launchers exec pkgs.bun from the store; confirm they were produced and are real entry + # points rather than dangling wrappers. + - name: Smoke the built launchers + run: | + set -eu + web=$(nix path-info .#punktfunk-web) + scripting=$(nix path-info .#punktfunk-scripting) + test -x "$web/bin/punktfunk-web-server" || { echo "no punktfunk-web-server in $web" >&2; exit 1; } + test -x "$scripting/bin/punktfunk-scripting" || { echo "no punktfunk-scripting in $scripting" >&2; exit 1; } + # The console must be the bun bundle, not a node one — the same assertion packages.nix + # makes at build time, re-checked on the installed output. + grep -q 'Bun\.serve' "$web/share/punktfunk-web/.output/server/index.mjs" \ + || { echo "installed console is not a bun bundle" >&2; exit 1; } + echo "bun packages OK: $web $scripting" + + # Opt-in only: the full Rust workspace through crane, which is the hour-long leg. + # `github.event.inputs.*` (string) rather than `inputs.*` — the portable spelling. + - name: Build the Rust packages (dispatch opt-in) + if: ${{ github.event.inputs.build-rust == 'true' }} + run: nix build --print-build-logs .#punktfunk-host .#punktfunk-client diff --git a/packaging/nix/README.md b/packaging/nix/README.md index 500fea5f..d010a76a 100644 --- a/packaging/nix/README.md +++ b/packaging/nix/README.md @@ -234,18 +234,33 @@ The shell exports `PF_FFVK_VULKAN_INCLUDE` (Vulkan headers for pf-ffvk bindgen) already in the lockfile, via a generated-and-committed `bun.nix` (`web/bun.nix`, `sdk/bun.nix`). There is **no aggregate deps hash to bump** — the previous design put `bun install` in a fixed-output derivation whose single `outputHash` silently went stale on every lockfile change and - broke the build. `bun.nix` regenerates itself: `bun2nix` is a devDependency of both packages and - runs on every `bun install` (web's `postinstall`; the SDK's `prepare`, since sdk/ is the - *published* `@punktfunk/host` package and a `postinstall` would then fire on consumers' installs). - Regenerate by hand with `cd web && bunx bun2nix -o bun.nix` if a lockfile is ever edited directly. + broke the build. `bun2nix` is a devDependency of both packages and regenerates `bun.nix` on every + `bun install` (web's `postinstall`; the SDK's `prepare`, since sdk/ is the *published* + `@punktfunk/host` package and a `postinstall` would then fire on consumers' installs). The `@unom` scope needs no special handling: `web/bun.lock` records those tarballs' full `https://git.unom.io/api/packages/unom/npm/…` URLs and the registry is read-public (the same anonymous pull CI's rpm/deb builds do). - > ⚠ **`bun.nix` has no schema stability across bun2nix versions.** The flake input is pinned - > (`github:nix-community/bun2nix?ref=2.1.2`) and the npm devDependency is pinned to the *same* - > exact version in `web/package.json` + `sdk/package.json`. Move both together, then rerun - > `bun install` in `web/` and `sdk/` to regenerate. + > ⚠⚠ **That devDependency hook is a convenience, NOT the guarantee — `bun.nix` still drifts.** + > It fires only on a local `bun install` that runs lifecycle scripts. It does *not* fire under + > `bun install --ignore-scripts`, which is what every bun install in CI uses; and it cannot fire + > on a **merge or rebase**, where git carries someone else's `bun.lock` change past a `bun.nix` + > generated before it and reports no conflict. That is how `web/bun.nix` shipped on main holding + > `brace-expansion@5.0.7` while `web/bun.lock` said `5.0.8` — for **553 commits** (2026-07-27 → + > 2026-08-05), with `nix build .#punktfunk-web` broken the whole time, until an unrelated + > advisory bump happened to rerun a real `bun install` and closed it by accident. + > + > The enforcement point is **`scripts/ci/check-bun-nix.sh`** (the `bun-nix` job in `ci.yml`, + > unfiltered so it sees the innocuous-looking commits drift arrives through). It regenerates each + > `bun.nix` from its committed `bun.lock` and diffs. Fix any report with: + > + > scripts/ci/check-bun-nix.sh --fix + > + > Never regenerate with a bare `bunx bun2nix`: **`bun.nix` has no schema stability across bun2nix + > versions**, and an unpinned `bunx` uses whatever is newest. The flake input + > (`github:nix-community/bun2nix?ref=2.1.2`) and the npm devDependency in `web/package.json` + + > `sdk/package.json` must name the *same exact version* — the script checks that too, and always + > generates with the pinned one. Move all three together, then rerun it with `--fix`. Everything past the deps fetch is offline (the console's codegen + vite build; the runner's `bun build --target=bun` bundle). Both launchers exec `pkgs.bun` from the store — unlike the diff --git a/scripts/ci/check-bun-nix.sh b/scripts/ci/check-bun-nix.sh new file mode 100755 index 00000000..983a4690 --- /dev/null +++ b/scripts/ci/check-bun-nix.sh @@ -0,0 +1,181 @@ +#!/bin/sh +# Drift gate for the generated bun2nix lockfile expressions (web/bun.nix, sdk/bun.nix). +# +# `bun.nix` is a DERIVED file: bun2nix is a pure function of `bun.lock` (it reads the lockfile text +# and emits one `fetchurl` per package, keyed by the lockfile's own integrity hashes — see +# packaging/nix/README.md). Nothing but the lockfile goes in, so any disagreement between the two +# committed files is drift, and it is always mechanically fixable. +# +# Why this exists: moving the bun packages to bun2nix (1db8f763) removed the *aggregate deps hash* +# that used to go stale, but not the second, quieter way a derived file rots. `bun.nix` regenerates +# only from a local `bun install` that runs lifecycle scripts (web's `postinstall`, the SDK's +# `prepare`). It does NOT regenerate on: +# +# * `bun install --ignore-scripts` — which is what EVERY bun install in CI uses (ci.yml, +# web-screenshots.yml, windows-host.yml, sdk-publish.yml), because web's `postinstall` shells +# out to a `bun` on PATH that CI's portable bun isn't; +# * a merge or rebase — git merges `bun.lock` and `bun.nix` as two unrelated files, so a branch +# that generated `bun.nix` before picking up someone else's lockfile change silently commits +# the pair out of step; +# * a lockfile edited or re-resolved by hand. +# +# That second case is not hypothetical: it is how `web/bun.nix` shipped on main carrying +# brace-expansion@5.0.7 (plus two nested entries the override had already collapsed) while +# `web/bun.lock` said 5.0.8 — the `^5.0.8` override from ec9aa415 landed in the lockfile, the +# bun2nix branch had generated `bun.nix` off the pre-override lock, and the merge kept both. The +# Nix build fetches node_modules strictly from `bun.nix`, so the offline `bun install` inside the +# derivation is then asked for a tarball the store cache does not contain and `punktfunk-web` fails +# to build — with a "package not found" that names npm, not the lockfile that actually drifted. +# +# The gate also enforces the version pin the flake and README only *state*: `bun.nix` has no schema +# stability across bun2nix releases, so the flake input ref and BOTH npm devDependencies must name +# the same exact version. Nothing checked that before; a half-moved pin regenerates the file with a +# generator the flake does not use. +# +# The list of packages to check is read out of packaging/nix/packages.nix (its `bunNix = src + …` +# lines) rather than hardcoded here, so a third bun package is covered the day it is added — and an +# empty list is a hard error, because a gate that checks nothing passes exactly like a clean tree. +# +# Usage: +# scripts/ci/check-bun-nix.sh # verify; non-zero on drift (CI) +# scripts/ci/check-bun-nix.sh --fix # regenerate the committed files in place +set -eu + +FIX=0 +if [ $# -gt 0 ]; then + case "$1" in + --fix) FIX=1 ;; + *) echo "usage: $0 [--fix]" >&2; exit 2 ;; + esac +fi + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd) +PACKAGES_NIX="$ROOT/packaging/nix/packages.nix" +FLAKE="$ROOT/flake.nix" + +command -v bun >/dev/null 2>&1 || { + echo "check-bun-nix: bun is not on PATH (needed to run bun2nix and to read package.json)" >&2 + exit 1 +} +[ -f "$PACKAGES_NIX" ] || { echo "check-bun-nix: no $PACKAGES_NIX" >&2; exit 1; } +[ -f "$FLAKE" ] || { echo "check-bun-nix: no $FLAKE" >&2; exit 1; } + +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +# --- the pinned bun2nix version ------------------------------------------------------------------- +# flake.nix: url = "github:nix-community/bun2nix?ref=2.1.2"; +PINNED=$(sed -n 's/.*github:nix-community\/bun2nix?ref=\([^"]*\)".*/\1/p' "$FLAKE" | head -1) +[ -n "$PINNED" ] || { + echo "check-bun-nix: could not read the bun2nix input ref out of $FLAKE." >&2 + echo "Expected a line like: url = \"github:nix-community/bun2nix?ref=\";" >&2 + exit 1 +} + +# --- which packages carry a generated bun.nix ----------------------------------------------------- +# packages.nix: bunDeps = bun2nix.fetchBunDeps { bunNix = src + "/web/bun.nix"; }; +sed -n 's/.*bunNix *= *src *+ *"\/\(.*\)\/bun\.nix".*/\1/p' "$PACKAGES_NIX" | sort -u > "$TMP/roots" +if [ ! -s "$TMP/roots" ]; then + echo "check-bun-nix: found no \`bunNix = src + \"//bun.nix\"\` in $PACKAGES_NIX." >&2 + echo "Either the bun packages were removed (delete this gate) or the expression changed shape" >&2 + echo "and the gate silently stopped checking anything. Not passing vacuously." >&2 + exit 1 +fi + +fail=0 +checked=0 + +# --- version pin agreement ------------------------------------------------------------------------ +# `bun.nix` has no schema stability across bun2nix versions, so the generator the flake builds with +# and the generator `bun install` runs must be the SAME exact version (packaging/nix/README.md). +while read -r dir; do + pkgjson="$ROOT/$dir/package.json" + [ -f "$pkgjson" ] || { echo "check-bun-nix: no $pkgjson" >&2; fail=1; continue; } + dev=$(bun -e "const d=require(process.argv[1]).devDependencies||{};console.log(d.bun2nix??'')" \ + "$pkgjson") + if [ "$dev" != "$PINNED" ]; then + echo "check-bun-nix: bun2nix version pin disagrees." >&2 + echo " flake.nix input ref : $PINNED" >&2 + echo " $dir/package.json devDependency : ${dev:-}" >&2 + echo "These must be the same exact version — bun.nix has no schema stability across" >&2 + echo "bun2nix releases. Move both together, then rerun this script with --fix." >&2 + fail=1 + fi +done < "$TMP/roots" + +# --- the generator --------------------------------------------------------------------------------- +# Prefer an already-installed bun2nix at the pinned version (fast, offline — the dev case); otherwise +# fetch exactly the pinned one, once, into $TMP. Never a floating `bunx bun2nix`: that would generate +# with whatever is newest, and `bun.nix` has no schema stability across releases. +BUN2NIX="" +while read -r dir; do + cand="$ROOT/$dir/node_modules/bun2nix/index.ts" + [ -f "$cand" ] || continue + have=$(bun -e "console.log(require(process.argv[1]).version??'')" \ + "$ROOT/$dir/node_modules/bun2nix/package.json" 2>/dev/null || echo '') + if [ "$have" = "$PINNED" ]; then BUN2NIX="$cand"; break; fi +done < "$TMP/roots" + +if [ -z "$BUN2NIX" ]; then + # Installed in its own scratch dir, so this never touches the repo's lockfiles or .npmrc. + mkdir -p "$TMP/gen" + if ! ( cd "$TMP/gen" && bun add --exact "bun2nix@$PINNED" ) > "$TMP/geninstall.log" 2>&1; then + echo "check-bun-nix: could not install bun2nix@$PINNED" >&2 + cat "$TMP/geninstall.log" >&2 + exit 1 + fi + BUN2NIX="$TMP/gen/node_modules/bun2nix/index.ts" + [ -f "$BUN2NIX" ] || { echo "check-bun-nix: bun2nix@$PINNED installed but $BUN2NIX is absent" >&2; exit 1; } +fi + +run_bun2nix() { # + bun "$BUN2NIX" --lock-file "$1" --output-file "$2" +} + +# --- regenerate + compare --------------------------------------------------------------------------- +while read -r dir; do + lock="$ROOT/$dir/bun.lock" + nix="$ROOT/$dir/bun.nix" + [ -f "$lock" ] || { echo "check-bun-nix: no $lock (packages.nix expects $dir/bun.nix)" >&2; fail=1; continue; } + + out="$TMP/$(echo "$dir" | tr '/' '_').bun.nix" + run_bun2nix "$lock" "$out" >/dev/null + + if [ "$FIX" -eq 1 ]; then + if [ ! -f "$nix" ] || ! cmp -s "$nix" "$out"; then + cp "$out" "$nix" + echo "check-bun-nix: regenerated $dir/bun.nix from $dir/bun.lock" + else + echo "check-bun-nix: $dir/bun.nix already in sync" + fi + checked=$((checked + 1)) + continue + fi + + if [ ! -f "$nix" ]; then + echo "check-bun-nix: $dir/bun.nix is MISSING — packages.nix fetches node_modules from it." >&2 + fail=1 + continue + fi + # Plain files, not `diff <(…) <(…)`: Gitea's runner executes a step's `run:` under `sh`, and + # dash has no process substitution — it would reject the script at parse time and the gate + # would never compare anything (exactly how the shader SPIR-V gate in ci.yml was lost). + if cmp -s "$nix" "$out"; then + echo "check-bun-nix: $dir/bun.nix matches $dir/bun.lock" + else + echo "check-bun-nix: $dir/bun.nix is STALE — it does not match $dir/bun.lock." >&2 + echo "The Nix build fetches node_modules only from bun.nix, so punktfunk's bun packages" >&2 + echo "would build against the wrong dependency set (or fail to fetch it at all)." >&2 + echo "Regenerate and commit it: scripts/ci/check-bun-nix.sh --fix" >&2 + echo "--- diff (committed -> regenerated from bun.lock) ---" >&2 + diff -u "$nix" "$out" >&2 || true + fail=1 + fi + checked=$((checked + 1)) +done < "$TMP/roots" + +[ "$checked" -gt 0 ] || { echo "check-bun-nix: checked nothing — refusing to report success" >&2; exit 1; } +if [ "$fail" -eq 0 ] && [ "$FIX" -eq 0 ]; then + echo "check-bun-nix: $checked bun package(s) in sync, bun2nix pinned at $PINNED everywhere" +fi +exit "$fail"