diff --git a/.gitea/workflows/deploy-services.yml b/.gitea/workflows/deploy-services.yml index bdecf2e7..55ca72a2 100644 --- a/.gitea/workflows/deploy-services.yml +++ b/.gitea/workflows/deploy-services.yml @@ -1,17 +1,23 @@ -# Deploy-only: bring up the two unom-1 pieces that live in THIS repo but whose normal -# deploys are coupled to heavy build workflows — docs to docker.yml's 5-image matrix, -# the flatpak server to flatpak.yml's full flatpak-builder run. This workflow does -# NEITHER build: it just (re)places the compose files and pulls the already-published -# images, so unom/infra's deploy-all can bring a fresh unom-1 fully up in a single -# dispatch without triggering those rebuilds. +# Deploy-only: bring up the unom-1 pieces that live in THIS repo but whose normal deploys +# are coupled to heavy build workflows — docs to docker.yml's 5-image matrix, the flatpak +# server to flatpak.yml's full flatpak-builder run, the nix cache to nix.yml's full Rust +# build. This workflow does NONE of those builds: it just (re)places the compose files and +# pulls the already-published images, so unom/infra's deploy-all can bring a fresh unom-1 +# fully up in a single dispatch without triggering those rebuilds. # -# docs -> pulls git.unom.io/unom/punktfunk-docs:latest (built by docker.yml) and -# brings it up on :3220. -# flatpak -> brings up the caddy:2-alpine static server on :3230. The OSTree repo -# CONTENT (./site) is NOT shipped here — it is regenerated by flatpak.yml -# on the next client build, or restored from the unom-1 backup -# (unom/infra scripts/restore-unom-1.sh, `files` tag). A fresh box serves -# an empty repo until then; that is expected. +# docs -> pulls git.unom.io/unom/punktfunk-docs:latest (built by docker.yml) and +# brings it up on :3220. +# flatpak -> brings up the caddy:2-alpine static server on :3230. The OSTree repo +# CONTENT (./site) is NOT shipped here — it is regenerated by flatpak.yml +# on the next client build, or restored from the unom-1 backup +# (unom/infra scripts/restore-unom-1.sh, `files` tag). A fresh box serves +# an empty repo until then; that is expected. +# nix-cache -> brings up the caddy:2-alpine Nix binary cache on :3250. Same content/config +# split: the cache CONTENT is republished by nix.yml on the next main push +# that moves the flake. An empty cache is harmless — every path 404s and +# users build from source, which is the pre-cache status quo. +# winget -> brings up the winget REST source on :3240; catalogue shipped by +# windows-host.yml on stable tags. # # Dispatched by unom/infra scripts/deploy-all.sh: `dispatch-and-wait.sh punktfunk # deploy-services.yml`. Uses the same secret set docker.yml/flatpak.yml already rely on: @@ -100,6 +106,46 @@ jobs: cd ~/unom-flatpak docker compose -f compose.production.yml up -d + nix-cache: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: Sync nix cache compose + server + uses: appleboy/scp-action@917f8b81dfc1ccd331fef9e2d61bdc6c8be94634 # v0.1.7 + with: + host: ${{ inputs.deploy_host || secrets.DEPLOY_HOST }} + username: ${{ secrets.DEPLOY_USER }} + port: ${{ secrets.DEPLOY_PORT }} + key: ${{ secrets.DEPLOY_SSH_KEY }} + # Land all three flat in ~/unom-nix-cache/ (drop the packaging/nix/server/ prefix). + source: "packaging/nix/server/compose.production.yml,packaging/nix/server/Caddyfile,packaging/nix/server/prune.sh" + target: "~/unom-nix-cache" + strip_components: 3 + overwrite: true + + - name: Start nix binary cache server + uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5 + with: + host: ${{ inputs.deploy_host || secrets.DEPLOY_HOST }} + username: ${{ secrets.DEPLOY_USER }} + port: ${{ secrets.DEPLOY_PORT }} + key: ${{ secrets.DEPLOY_SSH_KEY }} + script: | + set -euo pipefail + # ./site (the cache CONTENT) is NOT shipped here — nix.yml rsyncs it on each main push + # that moves the flake, same content/config split as the flatpak repo and the winget + # catalogue. Ensure the bind-mount source exists so caddy starts; an empty cache is + # harmless, it just 404s every path and users build from source as they do today. + mkdir -p ~/unom-nix-cache/site/nar + cd ~/unom-nix-cache + docker compose -f compose.production.yml up -d + # A cache that 404s a miss is healthy; one that cannot answer at all is not. + sleep 3 + curl -fsS http://127.0.0.1:3250/nix-cache-info \ + || echo "NOTE: no cache content yet - push to main with the flake touched to populate it" + winget: runs-on: ubuntu-24.04 timeout-minutes: 10 diff --git a/.gitea/workflows/nix.yml b/.gitea/workflows/nix.yml index d8a99526..310fbd18 100644 --- a/.gitea/workflows/nix.yml +++ b/.gitea/workflows/nix.yml @@ -4,8 +4,9 @@ # `nix build .#punktfunk-web` was broken for 553 commits before anyone noticed (see the bun-nix job # in ci.yml for that story). # -# Two tiers, because a full `nix flake check` builds the whole Rust workspace with crane and would -# run for an hour on every push: +# Three tiers, because a full `nix flake check` builds the whole Rust workspace with crane and would +# run for an hour on every push — so the two cheap tiers gate every PR and the expensive one runs +# only where its cost buys something (a published cache): # # * eval — `nix flake check --no-build`: instantiates every package, app, check and devShell # without building them. Catches the failures that actually happen to this flake — a @@ -32,15 +33,28 @@ # covers what the ci.yml drift gate cannot, e.g. a tarball the registry no longer # serves, or the codegen going quietly message-less (see packages.nix's inlang note). # -# The Rust packages (punktfunk-host, punktfunk-client) and punktfunk-gamescope are NOT built here. -# They are the expensive ones and their inputs are already gated by the `rust` job in ci.yml; build -# them by hand on a Nix box, or with the `build-rust` dispatch input below. +# * cache — PUSH TO MAIN ONLY. Builds the Rust packages + gamescope for real and publishes every +# punktfunk store path to the binary cache at https://nix.unom.io, so a NixOS user gets +# prebuilt binaries instead of an hour of rustc and a gamescope compile. This is the +# expensive tier and it is why the job timeout is 180 rather than 90. # -# ⚠ punktfunk-gamescope deserves the dispatch run more than it looks: `host.gamescopeHdr` DEFAULTS -# TRUE, so it is on the critical path of every `services.punktfunk.host.enable = true` build, while -# being the one package nothing here compiles. It patches whatever gamescope the pinned nixpkgs -# carries, so a nixpkgs bump — not a change of ours — is what breaks it, and the first person to -# find out would be an operator whose system rebuild fails. Run the dispatch after a flake.lock bump. +# It needs NO extra trigger for releases: a release bumps the workspace version in +# Cargo.toml, which is already in the path filter below, so the tag's content is +# published by the version-bump commit on main. +# +# Only OUR paths are published — see the step for why that is both correct and the +# difference between ~300 MB and several GB per publish. +# +# The Rust packages and punktfunk-gamescope are still not built on PRs: they are the expensive ones +# and their inputs are already gated by the `rust` job in ci.yml. Build them on a PR by hand on a +# Nix box, or with the `build-rust` / `build-gamescope` dispatch inputs below. +# +# ⚠ punktfunk-gamescope matters more than it looks: `host.gamescopeHdr` DEFAULTS TRUE, so it is on +# the critical path of every `services.punktfunk.host.enable = true` build. It patches whatever +# gamescope the pinned nixpkgs carries, so a nixpkgs bump — not a change of ours — is what breaks +# it, and the first person to find out would be an operator whose system rebuild fails. The `cache` +# tier now compiles it on every main push, so a flake.lock bump that breaks it goes red HERE; the +# dispatch input below is for checking it on a branch before merging. # # ⚠ pull_request is deliberately present. flatpak.yml shipped with push-only triggers and manifest # breakage reached main invisibly for weeks — do not "simplify" this workflow by dropping it. @@ -107,8 +121,15 @@ jobs: # real node (so actions/checkout works with no pre-checkout install dance), and audit.yml # already pulls it on this fleet, so it is proven to resolve here. Nix is installed below. image: node:22-bookworm - timeout-minutes: 90 + # 180, not 90: the `cache` tier on a main push compiles the whole Rust workspace AND gamescope + # from source inside the nix sandbox, where the sccache every other Rust job leans on cannot + # reach (no network in a derivation, and RUSTC_WRAPPER is not set inside one). + timeout-minutes: 180 env: + # Where the published cache lives on unom-1, and the URL users substitute from. Kept next to + # the flatpak repo (3230) and winget source (3240) — see packaging/nix/server/. + DEPLOY_DIR: unom-nix-cache + CACHE_URL: https://nix.unom.io # The flake needs both experimental features. Also baked into the installer's --extra-conf # below; this covers any step that shells out before that config is read. NIX_CONFIG: "experimental-features = nix-command flakes" @@ -126,11 +147,12 @@ jobs: steps: - uses: actions/checkout@v4 - # The Determinate installer needs curl + xz; git so nix can read the flake from the checkout. - # (node:22-bookworm is the full image and already has all three — this is belt-and-braces - # against a future slim-image swap, and costs one cached apt call.) + # The Determinate installer needs curl + xz; git so nix can read the flake from the checkout; + # rsync + ssh to ship the built cache to unom-1. (node:22-bookworm is the full image and + # already has all but rsync — this is belt-and-braces against a future slim-image swap, and + # costs one cached apt call.) - name: Installer prerequisites - run: apt-get update && apt-get install -y --no-install-recommends ca-certificates curl xz-utils git + run: apt-get update && apt-get install -y --no-install-recommends ca-certificates curl xz-utils git rsync openssh-client # `--init none` is the container mode: no systemd, no daemon. Running as root, nix then talks # to the store directly. Determinate Nix is also what the Nix box (.21) runs, so CI and the @@ -183,6 +205,133 @@ jobs: || { echo "installed console is not a bun bundle" >&2; exit 1; } echo "bun packages OK: $web $scripting" + # ── binary cache (push to main only) ─────────────────────────────────────────────────────── + # + # Decided against a bucket on storage.unom.io even though sccache already uses it and the + # credentials already exist: it is local RustFS on the home uplink with no CDN in front, so + # every NixOS user's download would come off the same pipe every CI runner uses — and S3 + # answers 403, not 404, for a missing key, which nix treats as a hard error rather than a + # cache miss (see packaging/nix/server/Caddyfile). unom-1 already serves the flatpak repo + # this way from a cloud IP; a Nix cache is the same static-files-behind-caddy shape. + # + # Gitea itself cannot host this at all: it has 23 package registry types and none is Nix, and + # the binary cache protocol wants fixed anonymous paths at a URL root (/nix-cache-info, + # /.narinfo, /nar/.nar.xz) that /api/packages/{owner}/generic/… cannot express. + - name: Cache publish preflight + id: cachecfg + if: ${{ github.event_name == 'push' }} + env: + NIX_CACHE_SIGNING_KEY: ${{ secrets.NIX_CACHE_SIGNING_KEY }} + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + # Guard BEFORE the build, not before the upload: an unconfigured cache must not cost an + # hour of rustc first. No-ops cleanly until the secret exists, exactly as flatpak.yml's + # repo deploy does, so this workflow stays green through setup. + run: | + set -eu + if [ -n "${NIX_CACHE_SIGNING_KEY:-}" ] && [ -n "${DEPLOY_HOST:-}" ]; then + echo "go=true" >> "$GITHUB_OUTPUT" + else + echo "go=false" >> "$GITHUB_OUTPUT" + echo "::warning::NIX_CACHE_SIGNING_KEY/DEPLOY_HOST not set — skipping the binary cache publish (see packaging/nix/README.md)." + fi + + - name: Build the publishable packages + if: ${{ steps.cachecfg.outputs.go == 'true' }} + # Everything a user can install. punktfunk-gamescope earns its place here more than any + # other: host.gamescopeHdr DEFAULTS TRUE, so without it in the cache every + # `services.punktfunk.host.enable = true` still compiles a compositor from source. + run: | + "$NIX" build --print-build-logs \ + .#punktfunk-host .#punktfunk-client .#punktfunk-tray \ + .#punktfunk-web .#punktfunk-scripting .#punktfunk-gamescope + # This is now the heaviest job on the fleet — a full workspace build plus gamescope fills + # the store with tens of GB, and this fleet ran a runner out of disk on 2026-08-06. Record + # the headroom AFTER the build too, or a future "no space left on device" is a guess. + df -h / /nix /tmp || true + + - name: Sign + publish to nix.unom.io + if: ${{ steps.cachecfg.outputs.go == 'true' }} + env: + NIX_CACHE_SIGNING_KEY: ${{ secrets.NIX_CACHE_SIGNING_KEY }} + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + DEPLOY_USER: ${{ secrets.DEPLOY_USER }} + DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }} + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + run: | + # `set -eu`, NOT `set -euo pipefail`: act_runner may execute a step's `run:` under dash in + # these containers (see scripts/ci/ensure-sccache.sh), and dash dies on `-o pipefail` with + # "Illegal option". The two places below where a pipeline's LEFT side must be able to fail + # the step are written as redirects instead, so nothing depends on pipefail. + set -eu + PKGS=".#punktfunk-host .#punktfunk-client .#punktfunk-tray .#punktfunk-web .#punktfunk-scripting .#punktfunk-gamescope" + + # 1) Pick what to publish. PUBLISH ONLY OUR OWN PATHS — this is the difference between + # ~300 MB and several GB per run, and it is not a corner cut: a runtime closure here + # is our binaries plus stock nixpkgs (ffmpeg, gtk4, glibc, …), and every stock path is + # already on cache.nixos.org, served by a real CDN. Mirroring them onto unom-1 would + # cost disk and home-to-cloud bandwidth to serve a WORSE copy of what users already + # have. Nothing in nixpkgs is named punktfunk, so the name filter is exact. + paths="$("$NIX" path-info -r $PKGS | grep -- '-punktfunk' || true)" + [ -n "$paths" ] || { echo "::error::no punktfunk store paths in the closure — the name filter is broken"; exit 1; } + echo "$paths" + # The filter is a string match, so it would fail SILENTLY if a pname ever changed — and + # the package most likely to drift is gamescope, the most expensive one to lose. Assert + # every built output is actually covered rather than discovering it as a user rebuild. + for out in $("$NIX" build --print-out-paths $PKGS); do + printf '%s\n' "$paths" | grep -qxF "$out" \ + || { echo "::error::$out is not matched by the '-punktfunk' filter — publish would silently omit it"; exit 1; } + done + + # 2) Sign into a local binary cache. The secret is the whole `name:base64` line from + # `nix key generate-secret`; the matching public key is what users pin (README). + KEYDIR="$(mktemp -d)"; chmod 700 "$KEYDIR" + printf '%s' "$NIX_CACHE_SIGNING_KEY" > "$KEYDIR/key"; chmod 600 "$KEYDIR/key" + printf '%s\n' "$paths" | xargs "$NIX" copy --to "file://$PWD/nix-cache?secret-key=$KEYDIR/key" + # Publish the PUBLIC half beside the cache and echo it here. Users must pin this key, so + # it needs to be fetchable from the cache itself rather than only from a doc that can + # drift — and on the first run this log line is where the value for README.md comes from. + # Redirect, not `| tee`: without pipefail a failing nix would be masked by tee's success + # and publish an EMPTY public key, which every user would then pin. + "$NIX" key convert-secret-to-public < "$KEYDIR/key" > nix-cache/punktfunk-cache.pub + cat nix-cache/punktfunk-cache.pub + rm -rf "$KEYDIR" + echo "publishing $(find nix-cache -name '*.narinfo' | wc -l) paths, $(du -sh nix-cache | cut -f1)" + + # 3) Ship it. Same deploy key and retry discipline as flatpak.yml — this runner's link to + # unom-1 drops TCP dials under load. + install -d -m700 ~/.ssh + printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy; chmod 600 ~/.ssh/deploy + SSH="ssh -i $HOME/.ssh/deploy -p ${DEPLOY_PORT:-22} -o StrictHostKeyChecking=accept-new" + DEST="${DEPLOY_USER}@${DEPLOY_HOST}" + bash scripts/ci/retry.sh 5 $SSH "$DEST" "mkdir -p ~/$DEPLOY_DIR/site/nar" + # ⚠ ORDER IS LOAD-BEARING: NARs first, narinfos second. A narinfo whose NAR has not landed + # yet is a HARD download failure for whoever fetches it in that window; a NAR nothing + # points at yet is simply invisible. rsync renames each file into place atomically, so a + # cancelled run (this workflow has cancel-in-progress) can only ever under-publish. + # No --delete: superseded paths are aged out by prune.sh below instead, so a client + # mid-download is never pulled out from under. + bash scripts/ci/retry.sh 5 rsync -az --info=stats1 -e "$SSH" nix-cache/nar/ "$DEST:$DEPLOY_DIR/site/nar/" + bash scripts/ci/retry.sh 5 rsync -az -e "$SSH" nix-cache/nix-cache-info nix-cache/punktfunk-cache.pub nix-cache/*.narinfo "$DEST:$DEPLOY_DIR/site/" + bash scripts/ci/retry.sh 5 rsync -az -e "$SSH" \ + packaging/nix/server/compose.production.yml packaging/nix/server/Caddyfile packaging/nix/server/prune.sh \ + "$DEST:$DEPLOY_DIR/" + bash scripts/ci/retry.sh 5 $SSH "$DEST" "cd ~/$DEPLOY_DIR && docker compose -f compose.production.yml up -d" + + # 4) Bound it. The flatpak repo next door reached 3.84 GB publishing this same way with + # no sweep, on a box that has run out of disk before; this one gets the sweep from the + # first publish. Never allowed to fail the job — the cache is already live by now, and + # a growing disk is a slower problem than a red main. + bash scripts/ci/retry.sh 3 $SSH "$DEST" "sh ~/$DEPLOY_DIR/prune.sh ~/$DEPLOY_DIR/site 180" \ + || echo "::warning::cache prune failed — published cache may be growing unbounded" + + # 5) Prove the published cache actually answers, rather than assuming the rsync landed. + # A substituter that 200s on nix-cache-info but 403s on a miss is the failure mode that + # breaks users' builds, so check both. + bash scripts/ci/retry.sh 5 curl -fsS "$CACHE_URL/nix-cache-info" + miss="$(curl -sS -o /dev/null -w '%{http_code}' "$CACHE_URL/0000000000000000000000000000000000.narinfo")" + [ "$miss" = 404 ] || { echo "::error::cache returns $miss for an absent path; nix needs 404 or every user build fails"; exit 1; } + echo "published → $CACHE_URL" + # Opt-in only: the full Rust workspace through crane, which is the hour-long leg. # `github.event.inputs.*` (string) rather than `inputs.*` — the portable spelling. - name: Build the Rust packages (dispatch opt-in) diff --git a/docs-site/content/docs/install.md b/docs-site/content/docs/install.md index 8190df91..2691f8dc 100644 --- a/docs-site/content/docs/install.md +++ b/docs-site/content/docs/install.md @@ -32,8 +32,9 @@ On **Windows**, the host ships as a signed installer instead — see [Windows](# Each registry is public — no auth, you just trust the repo's signing key. Adding the repo is a one-time step covered in the linked guide; after that, normal `apt upgrade` / `dnf upgrade` / -`pacman -Syu` (or `sudo punktfunk-sysext update` on Bazzite) tracks new builds. On **NixOS** there -is no repo to add — you add the flake as an input and enable its module, see [NixOS](#nixos). +`pacman -Syu` (or `sudo punktfunk-sysext update` on Bazzite) tracks new builds. On **NixOS** you add +the flake as an input and enable its module rather than adding a package repo — but do add the +[binary cache](#nixos), or every build compiles from source. > **Stable vs canary.** The repos in the per-distro guides are the **stable** channel — it only > moves when a `vX.Y.Z` release is cut. For the latest `main` build (fast, possibly broken), point @@ -93,6 +94,22 @@ The repo's `flake.nix` is a supported install path: it builds `punktfunk-host`, `punktfunk-web` and `punktfunk-scripting`, and ships a NixOS module. **`x86_64-linux` only**, and NixOS **24.11 or newer**. +**Add the binary cache first.** Without it, a build compiles the whole Rust workspace *and* +gamescope from source — about an hour. With it you get prebuilt binaries: + +```nix +nix.settings = { + substituters = [ "https://nix.unom.io" ]; + trusted-public-keys = [ "punktfunk-cache-1:" ]; # curl https://nix.unom.io/punktfunk-cache.pub +}; +``` + +Off NixOS, put the same two values in `/etc/nix/nix.conf` as `extra-substituters` / +`extra-trusted-public-keys`. One caveat worth knowing before you copy a flake snippet from +elsewhere: setting `inputs.punktfunk.inputs.nixpkgs.follows = "nixpkgs"` changes every store path +and so misses the cache entirely — details in +[packaging/nix](https://git.unom.io/unom/punktfunk/src/branch/main/packaging/nix/README.md#binary-cache-do-this-before-your-first-build). + You can run it straight from the flake without NixOS (on other distros, wrap it in [nixGL](https://github.com/nix-community/nixGL) so the GPU drivers resolve): diff --git a/packaging/nix/README.md b/packaging/nix/README.md index 48844f00..27813187 100644 --- a/packaging/nix/README.md +++ b/packaging/nix/README.md @@ -45,6 +45,69 @@ GPU drivers are resolved at runtime from `/run/opengl-driver/lib`. On non-NixOS [nixGL](https://github.com/nix-community/nixGL) so that path is populated (`nixGL nix run …`); on NixOS the module (below) sets `hardware.graphics.enable = true` for you. +> **Do this first, or the commands above compile the world:** without the binary cache, `nix build` +> here means the whole Rust workspace *and* a gamescope build from source — roughly an hour on a +> fast machine. See below. + +--- + +## Binary cache (do this before your first build) + +CI publishes every punktfunk package to **`https://nix.unom.io`** on each push to `main` that moves +the flake, so you get prebuilt binaries instead of an hour of `rustc`. It covers +`punktfunk-host`, `-client`, `-tray`, `-web`, `-scripting` and `-gamescope` — everything the flake +builds from source. Everything else in the closure is stock nixpkgs and comes from `cache.nixos.org` +as usual, so the cache is deliberately small and adding it costs you nothing on unrelated builds. + +**NixOS** — in your system configuration: + +```nix +nix.settings = { + substituters = [ "https://nix.unom.io" ]; + trusted-public-keys = [ "punktfunk-cache-1:" ]; +}; +``` + +**Anywhere else** — in `/etc/nix/nix.conf` (or `~/.config/nix/nix.conf` if you are a trusted user): + +```conf +extra-substituters = https://nix.unom.io +extra-trusted-public-keys = punktfunk-cache-1: +``` + +The current public key is served by the cache itself, so you can always check it against the source +of truth: + +```sh +curl https://nix.unom.io/punktfunk-cache.pub +``` + +Verify the cache is being used — this should print the store paths without compiling anything: + +```sh +nix build --dry-run git+https://git.unom.io/unom/punktfunk#punktfunk-host +``` + +### ⚠ `nixpkgs.follows` turns the cache off + +Every store path is keyed by the exact inputs it was built from. Pointing punktfunk's nixpkgs at +yours changes those inputs, so **every** path misses and you compile the workspace anyway: + +```nix +# Convenient, but it costs you the entire binary cache: +inputs.punktfunk.inputs.nixpkgs.follows = "nixpkgs"; +``` + +That is a real trade, not a bug — `follows` buys you one shared nixpkgs in the closure instead of +two. Take it if closure size matters more to you than build time; leave it out to get binaries. + +### Why not `cachix`? + +Nothing against it — punktfunk simply self-hosts every other channel (flatpak, deb, rpm, Arch, +docker, winget), and a Nix cache is static files behind a web server, so it rides the same unom-1 +box and the same deploy key as the rest. Nothing about the cache is punktfunk-specific: it speaks +plain HTTP binary-cache protocol, so any nix client works with it. + --- ## NixOS module @@ -55,6 +118,8 @@ Add the flake and enable the host and/or client: { inputs.punktfunk.url = "git+https://git.unom.io/unom/punktfunk"; # (optional) share your nixpkgs: inputs.punktfunk.inputs.nixpkgs.follows = "nixpkgs"; + # ⚠ this DISABLES the binary cache — different inputs, different store paths, so every + # package is rebuilt from source (~1h). See "Binary cache" above. outputs = { self, nixpkgs, punktfunk, ... }: { nixosConfigurations.myhost = nixpkgs.lib.nixosSystem { @@ -369,8 +434,66 @@ RUNPATH (`/run/opengl-driver/lib`) and the GTK GApps wrapper (GSettings schemas are present. Fixes discovered during that bring-up: `CMAKE_POLICY_VERSION_MINIMUM=3.5` (CMake ≥ 4), system `libopus` (audiopus_sys), and the session Skia note above. -In CI (`.gitea/workflows/nix.yml`): `nix flake check --no-build` evaluates every output *including* -the module check above, and `punktfunk-web` + `punktfunk-scripting` are built for real. The Rust -packages and `punktfunk-gamescope` are `workflow_dispatch` opt-ins (`build-rust`, -`build-gamescope`) — run the latter after a `flake.lock` bump, since it patches whatever gamescope -the pinned nixpkgs carries. +In CI (`.gitea/workflows/nix.yml`), three tiers: `nix flake check --no-build` evaluates every output +*including* the module check above; `punktfunk-web` + `punktfunk-scripting` are built for real on +every PR; and on a push to `main` the Rust packages and `punktfunk-gamescope` are built and +published to the binary cache. A `flake.lock` bump that breaks the gamescope patches therefore goes +red on main rather than in an operator's rebuild. The `build-rust` / `build-gamescope` +`workflow_dispatch` inputs remain, for checking those on a branch before merging. + +--- + +## Cache infrastructure (maintainers) + +`https://nix.unom.io` is a `caddy:2-alpine` container on unom-1 serving a static directory — +`packaging/nix/server/` — exactly like the flatpak repo (3230) and the winget source (3240). A Nix +binary cache *is* just `nix-cache-info` + `.narinfo` + `nar/.nar.xz` behind a web +server; there is no cache daemon to run. + +**Why not Gitea, and why not `storage.unom.io`:** + +- Gitea has 23 package registry types and none is Nix. It is not a missing label — the protocol + needs fixed anonymous paths at a URL *root* (`/nix-cache-info`, `/.narinfo`, + `/nar/…`), and `/api/packages/{owner}/generic/{name}/{version}/{file}` cannot express them. +- The RustFS S3 at `storage.unom.io` *would* work mechanically (nix speaks `s3://…?endpoint=`, and + the sccache credentials already exist), but it is a local box on the home uplink with no CDN, so + every user download competes with CI. It also answers **403** for a missing key unless the bucket + policy grants anonymous `ListBucket` — and nix treats anything other than **404** as a hard error + rather than a cache miss, which would break users' builds for packages the cache never held. + +**One-time setup — in this order.** The publish step ends by fetching `nix.unom.io` to prove the +cache really answers (and answers **404**, not 403, for a path it does not hold), so stand the +service up *before* you set the secret that switches publishing on. The secret is the last step for +exactly that reason: until it exists the publish no-ops with a warning and `main` stays green, +the same way flatpak.yml's repo deploy does. + +1. **Edge proxy:** `nix.unom.io { reverse_proxy 192.168.50.50:3250 }` on home-reverse-proxy-1. +2. **Port allowlist:** add `3250` to `caddy_target_ports` in `unom/infra` (proxmox/unom-1) + + terraform apply. +3. **DNS:** ensure `nix.unom.io` resolves to the edge proxy. +4. Dispatch `deploy-services.yml` (or `unom/infra`'s `deploy-all`) to bring the container up. It + serves an empty cache — every path 404s, which is exactly what a healthy empty cache does. +5. Generate the signing key on a Nix box and store the secret half as the repo Actions secret + `NIX_CACHE_SIGNING_KEY` (the whole `name:base64` line): + ```sh + nix key generate-secret --key-name punktfunk-cache-1 + ``` +6. Push to `main` touching the flake. The publish step prints the **public** key — paste it into the + "Binary cache" section above (and `docs-site/content/docs/install.md`) and commit. + +**Operational notes:** + +- Only punktfunk's own store paths are published (`nix path-info -r … | grep -- '-punktfunk'`). + Everything else in a closure is stock nixpkgs, already on `cache.nixos.org` behind a real CDN; + mirroring it would cost disk and home-to-cloud bandwidth to serve a worse copy. The publish step + asserts every built output is matched by that filter, so a future `pname` change fails the build + instead of silently dropping a package from the cache. +- `rsync` runs **without** `--delete` (a client mid-download is never pulled out from under), and + NARs are uploaded *before* narinfos — a narinfo whose NAR has not landed is a hard download + failure for whoever fetches it in that window, while an unreferenced NAR is merely invisible. +- Growth is bounded by `packaging/nix/server/prune.sh` (evicts narinfos untouched for 180 days, + then sweeps NARs nothing references). The flatpak repo next door reached 3.84 GB publishing the + same way with no sweep, on a box that has run out of disk before — hence the sweep from the first + publish. Run its self-check with `sh packaging/nix/server/prune.sh --self-test`. +- A user on a pinned rev older than the eviction window falls back to building from source, which + is the pre-cache status quo. diff --git a/packaging/nix/server/Caddyfile b/packaging/nix/server/Caddyfile new file mode 100644 index 00000000..fc6be23f --- /dev/null +++ b/packaging/nix/server/Caddyfile @@ -0,0 +1,32 @@ +# Inner Caddy (plain HTTP on :3250); the edge proxy on home-reverse-proxy-1 does TLS for +# nix.unom.io. Serves the punktfunk Nix binary cache — which is nothing but a static tree: +# +# /nix-cache-info store dir + priority, fetched once per substituter +# /<32-char-hash>.narinfo one per store path +# /nar/.nar.xz the archives themselves +# +# ⚠ A MISSING PATH MUST 404, NOT 403. Nix reads 404 as "not in this cache, try the next +# substituter" and treats anything else as a hard error that fails the build — so a cache +# answering 403 for unknown hashes breaks every user who adds it, including for packages it +# was never meant to serve. `file_server` 404s correctly; do not put an auth wrapper in front +# of this without preserving that. This is also the concrete reason the cache is NOT a bucket +# on storage.unom.io: S3 answers 403 for a missing key unless the bucket policy grants +# anonymous ListBucket, and that box is on the home uplink besides. +:3250 { + root * /srv + file_server browse + + # Everything except nix-cache-info is content-addressed by the store hash and can never + # change meaning — a narinfo for a given hash is as immutable as the NAR it points at. + @immutable path /nar/* *.narinfo + header @immutable Cache-Control "public, max-age=31536000, immutable" + + # The only mutable file, and cheap to revalidate: nix reads it once per substituter per run. + @info path /nix-cache-info + header @info Cache-Control "public, max-age=300" + + # nix does not care about Content-Type, but a browser poking at the cache should not be + # offered a download for what is a two-line text file. + @text path *.narinfo /nix-cache-info + header @text Content-Type "text/plain; charset=utf-8" +} diff --git a/packaging/nix/server/compose.production.yml b/packaging/nix/server/compose.production.yml new file mode 100644 index 00000000..770633e6 --- /dev/null +++ b/packaging/nix/server/compose.production.yml @@ -0,0 +1,17 @@ +# Static file server for the punktfunk Nix binary cache, on unom-1 (the DMZ services VM) — +# the same shape as packaging/flatpak/server, because a Nix cache and an OSTree repo are both +# just immutable files behind a web server. Caddy on home-reverse-proxy-1 terminates TLS for +# nix.unom.io and reverse_proxies to 192.168.50.50:3250 (the port must be in unom/infra +# caddy_target_ports). This inner Caddy serves the bind-mounted ./site tree over plain HTTP. +# nix.yml rsyncs into ./site and runs `docker compose up -d` (idempotent). +# +# Port 3250: docs is 3220, flatpak 3230, winget 3240 — keep the run going. +services: + nix-cache: + image: caddy:2-alpine + restart: unless-stopped + ports: + - "3250:3250" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - ./site:/srv:ro diff --git a/packaging/nix/server/prune.sh b/packaging/nix/server/prune.sh new file mode 100755 index 00000000..f97afa92 --- /dev/null +++ b/packaging/nix/server/prune.sh @@ -0,0 +1,93 @@ +#!/bin/sh +# Bound the published Nix binary cache on unom-1. +# +# WHY THIS EXISTS UP FRONT, rather than being added after the box fills: the flatpak repo next +# door taught this exact lesson the expensive way. It publishes with rsync WITHOUT --delete (so +# a client mid-download is never broken), nothing ever removed the superseded objects, and it +# reached 3.84 GB on a box that had already run out of disk once. This cache has the same +# publish model and the same growth shape — every build whose inputs moved adds a fresh set of +# store paths and keeps the old ones — so it gets the sweep from day one. +# +# The order below is the whole correctness argument: +# +# 1. Delete narinfos older than $DAYS. rsync -a carries the CI-side mtime over, and `nix copy` +# rewrites every narinfo it publishes on every run, so a path that is still being published +# keeps getting a fresh mtime. Age therefore means "no publish has referenced this in +# $DAYS", which is exactly the eviction signal wanted. +# 2. THEN delete NARs no surviving narinfo points at. +# +# Doing it the other way round — or aging the NARs independently — can strand a live narinfo +# pointing at a deleted NAR, and that is strictly worse than a cache miss: nix reports a missing +# NAR as a hard download failure, not as "not cached, build it yourself". +# +# POSIX sh: this runs over ssh on unom-1 (Debian), invoked by .gitea/workflows/nix.yml. +# +# Usage: sh prune.sh [max-age-days] (default 180) +# sh prune.sh --self-test +set -eu + +self_test() { + # Smallest thing that fails if the ordering or the reference sweep breaks. + t="$(mktemp -d)" + trap 'rm -rf "$t"' EXIT + mkdir -p "$t/nar" + printf 'StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 41\n' >"$t/nix-cache-info" + + # A live path, a stale one, and a NAR nothing ever pointed at. + printf 'StorePath: /nix/store/aaa-live\nURL: nar/live.nar.xz\n' >"$t/aaa.narinfo" + printf 'StorePath: /nix/store/bbb-stale\nURL: nar/stale.nar.xz\n' >"$t/bbb.narinfo" + : >"$t/nar/live.nar.xz" + : >"$t/nar/stale.nar.xz" + : >"$t/nar/orphan.nar.xz" + # Two paths that SHARE a NAR, one stale and one live: the shared NAR must survive. A sweep + # that deleted NARs per-evicted-narinfo instead of by surviving references would drop it. + printf 'StorePath: /nix/store/ccc-live\nURL: nar/shared.nar.xz\n' >"$t/ccc.narinfo" + printf 'StorePath: /nix/store/ddd-stale\nURL: nar/shared.nar.xz\n' >"$t/ddd.narinfo" + : >"$t/nar/shared.nar.xz" + + # Age the stale ones well past any plausible threshold (portable -t form: YYYYMMDDhhmm). + touch -t 200001010000 "$t/bbb.narinfo" "$t/ddd.narinfo" "$t/nar/stale.nar.xz" + + prune "$t" 180 + + fail() { echo "SELF-TEST FAILED: $1" >&2; exit 1; } + [ -f "$t/aaa.narinfo" ] || fail "evicted a fresh narinfo" + [ -f "$t/nar/live.nar.xz" ] || fail "evicted a referenced NAR" + [ -f "$t/nix-cache-info" ] || fail "deleted nix-cache-info" + [ ! -f "$t/bbb.narinfo" ] || fail "kept a stale narinfo" + [ ! -f "$t/nar/stale.nar.xz" ] || fail "kept a NAR nothing references any more" + [ ! -f "$t/nar/orphan.nar.xz" ] || fail "kept an orphan NAR" + [ -f "$t/nar/shared.nar.xz" ] || fail "deleted a NAR a surviving narinfo still references" + echo "prune.sh self-test OK" +} + +prune() { + root="$1" + days="$2" + cd "$root" + + before="$(du -sh . 2>/dev/null | cut -f1)" + + # 1. Age out the narinfos. + find . -maxdepth 1 -name '*.narinfo' -mtime "+$days" -delete + + # 2. Sweep NARs nothing points at any more. Both lists are relative to $root and spelled the + # same way ("nar/") so `comm` can diff them. + keep="$(mktemp)" + have="$(mktemp)" + # `|| true`: an empty cache (or one whose narinfos were all just evicted) makes the glob + # match nothing, and an empty keep-list is the correct answer there, not an error. + cat ./*.narinfo 2>/dev/null | sed -n 's|^URL: ||p' | sort -u >"$keep" || true + find nar -type f 2>/dev/null | sed 's|^\./||' | sort >"$have" || true + comm -13 "$keep" "$have" | tr '\n' '\0' | xargs -0 -r rm -f + rm -f "$keep" "$have" + + # Leave the numbers in the deploy log — this is the only place the published size is visible. + echo "cache pruned (narinfos older than ${days}d): ${before:-?} -> $(du -sh . 2>/dev/null | cut -f1) in $(pwd)" +} + +case "${1:-}" in +--self-test) self_test ;; +"") echo "usage: prune.sh [max-age-days] | --self-test" >&2; exit 2 ;; +*) prune "$1" "${2:-180}" ;; +esac