NixOS users compiled the whole workspace because we published no binaries #313
@@ -1,17 +1,23 @@
|
||||
# Deploy-only: bring up the two unom-1 pieces that live in THIS repo but whose normal
|
||||
# deploys are coupled to heavy build workflows — docs to docker.yml's 5-image matrix,
|
||||
# the flatpak server to flatpak.yml's full flatpak-builder run. This workflow does
|
||||
# NEITHER build: it just (re)places the compose files and pulls the already-published
|
||||
# images, so unom/infra's deploy-all can bring a fresh unom-1 fully up in a single
|
||||
# dispatch without triggering those rebuilds.
|
||||
# Deploy-only: bring up the unom-1 pieces that live in THIS repo but whose normal deploys
|
||||
# are coupled to heavy build workflows — docs to docker.yml's 5-image matrix, the flatpak
|
||||
# server to flatpak.yml's full flatpak-builder run, the nix cache to nix.yml's full Rust
|
||||
# build. This workflow does NONE of those builds: it just (re)places the compose files and
|
||||
# pulls the already-published images, so unom/infra's deploy-all can bring a fresh unom-1
|
||||
# fully up in a single dispatch without triggering those rebuilds.
|
||||
#
|
||||
# docs -> pulls git.unom.io/unom/punktfunk-docs:latest (built by docker.yml) and
|
||||
# brings it up on :3220.
|
||||
# flatpak -> brings up the caddy:2-alpine static server on :3230. The OSTree repo
|
||||
# CONTENT (./site) is NOT shipped here — it is regenerated by flatpak.yml
|
||||
# on the next client build, or restored from the unom-1 backup
|
||||
# (unom/infra scripts/restore-unom-1.sh, `files` tag). A fresh box serves
|
||||
# an empty repo until then; that is expected.
|
||||
# docs -> pulls git.unom.io/unom/punktfunk-docs:latest (built by docker.yml) and
|
||||
# brings it up on :3220.
|
||||
# flatpak -> brings up the caddy:2-alpine static server on :3230. The OSTree repo
|
||||
# CONTENT (./site) is NOT shipped here — it is regenerated by flatpak.yml
|
||||
# on the next client build, or restored from the unom-1 backup
|
||||
# (unom/infra scripts/restore-unom-1.sh, `files` tag). A fresh box serves
|
||||
# an empty repo until then; that is expected.
|
||||
# nix-cache -> brings up the caddy:2-alpine Nix binary cache on :3250. Same content/config
|
||||
# split: the cache CONTENT is republished by nix.yml on the next main push
|
||||
# that moves the flake. An empty cache is harmless — every path 404s and
|
||||
# users build from source, which is the pre-cache status quo.
|
||||
# winget -> brings up the winget REST source on :3240; catalogue shipped by
|
||||
# windows-host.yml on stable tags.
|
||||
#
|
||||
# Dispatched by unom/infra scripts/deploy-all.sh: `dispatch-and-wait.sh punktfunk
|
||||
# deploy-services.yml`. Uses the same secret set docker.yml/flatpak.yml already rely on:
|
||||
@@ -100,6 +106,46 @@ jobs:
|
||||
cd ~/unom-flatpak
|
||||
docker compose -f compose.production.yml up -d
|
||||
|
||||
nix-cache:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Sync nix cache compose + server
|
||||
uses: appleboy/scp-action@917f8b81dfc1ccd331fef9e2d61bdc6c8be94634 # v0.1.7
|
||||
with:
|
||||
host: ${{ inputs.deploy_host || secrets.DEPLOY_HOST }}
|
||||
username: ${{ secrets.DEPLOY_USER }}
|
||||
port: ${{ secrets.DEPLOY_PORT }}
|
||||
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
||||
# Land all three flat in ~/unom-nix-cache/ (drop the packaging/nix/server/ prefix).
|
||||
source: "packaging/nix/server/compose.production.yml,packaging/nix/server/Caddyfile,packaging/nix/server/prune.sh"
|
||||
target: "~/unom-nix-cache"
|
||||
strip_components: 3
|
||||
overwrite: true
|
||||
|
||||
- name: Start nix binary cache server
|
||||
uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5
|
||||
with:
|
||||
host: ${{ inputs.deploy_host || secrets.DEPLOY_HOST }}
|
||||
username: ${{ secrets.DEPLOY_USER }}
|
||||
port: ${{ secrets.DEPLOY_PORT }}
|
||||
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
||||
script: |
|
||||
set -euo pipefail
|
||||
# ./site (the cache CONTENT) is NOT shipped here — nix.yml rsyncs it on each main push
|
||||
# that moves the flake, same content/config split as the flatpak repo and the winget
|
||||
# catalogue. Ensure the bind-mount source exists so caddy starts; an empty cache is
|
||||
# harmless, it just 404s every path and users build from source as they do today.
|
||||
mkdir -p ~/unom-nix-cache/site/nar
|
||||
cd ~/unom-nix-cache
|
||||
docker compose -f compose.production.yml up -d
|
||||
# A cache that 404s a miss is healthy; one that cannot answer at all is not.
|
||||
sleep 3
|
||||
curl -fsS http://127.0.0.1:3250/nix-cache-info \
|
||||
|| echo "NOTE: no cache content yet - push to main with the flake touched to populate it"
|
||||
|
||||
winget:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
|
||||
+164
-15
@@ -4,8 +4,9 @@
|
||||
# `nix build .#punktfunk-web` was broken for 553 commits before anyone noticed (see the bun-nix job
|
||||
# in ci.yml for that story).
|
||||
#
|
||||
# Two tiers, because a full `nix flake check` builds the whole Rust workspace with crane and would
|
||||
# run for an hour on every push:
|
||||
# Three tiers, because a full `nix flake check` builds the whole Rust workspace with crane and would
|
||||
# run for an hour on every push — so the two cheap tiers gate every PR and the expensive one runs
|
||||
# only where its cost buys something (a published cache):
|
||||
#
|
||||
# * eval — `nix flake check --no-build`: instantiates every package, app, check and devShell
|
||||
# without building them. Catches the failures that actually happen to this flake — a
|
||||
@@ -32,15 +33,28 @@
|
||||
# covers what the ci.yml drift gate cannot, e.g. a tarball the registry no longer
|
||||
# serves, or the codegen going quietly message-less (see packages.nix's inlang note).
|
||||
#
|
||||
# The Rust packages (punktfunk-host, punktfunk-client) and punktfunk-gamescope are NOT built here.
|
||||
# They are the expensive ones and their inputs are already gated by the `rust` job in ci.yml; build
|
||||
# them by hand on a Nix box, or with the `build-rust` dispatch input below.
|
||||
# * cache — PUSH TO MAIN ONLY. Builds the Rust packages + gamescope for real and publishes every
|
||||
# punktfunk store path to the binary cache at https://nix.unom.io, so a NixOS user gets
|
||||
# prebuilt binaries instead of an hour of rustc and a gamescope compile. This is the
|
||||
# expensive tier and it is why the job timeout is 180 rather than 90.
|
||||
#
|
||||
# ⚠ punktfunk-gamescope deserves the dispatch run more than it looks: `host.gamescopeHdr` DEFAULTS
|
||||
# TRUE, so it is on the critical path of every `services.punktfunk.host.enable = true` build, while
|
||||
# being the one package nothing here compiles. It patches whatever gamescope the pinned nixpkgs
|
||||
# carries, so a nixpkgs bump — not a change of ours — is what breaks it, and the first person to
|
||||
# find out would be an operator whose system rebuild fails. Run the dispatch after a flake.lock bump.
|
||||
# It needs NO extra trigger for releases: a release bumps the workspace version in
|
||||
# Cargo.toml, which is already in the path filter below, so the tag's content is
|
||||
# published by the version-bump commit on main.
|
||||
#
|
||||
# Only OUR paths are published — see the step for why that is both correct and the
|
||||
# difference between ~300 MB and several GB per publish.
|
||||
#
|
||||
# The Rust packages and punktfunk-gamescope are still not built on PRs: they are the expensive ones
|
||||
# and their inputs are already gated by the `rust` job in ci.yml. Build them on a PR by hand on a
|
||||
# Nix box, or with the `build-rust` / `build-gamescope` dispatch inputs below.
|
||||
#
|
||||
# ⚠ punktfunk-gamescope matters more than it looks: `host.gamescopeHdr` DEFAULTS TRUE, so it is on
|
||||
# the critical path of every `services.punktfunk.host.enable = true` build. It patches whatever
|
||||
# gamescope the pinned nixpkgs carries, so a nixpkgs bump — not a change of ours — is what breaks
|
||||
# it, and the first person to find out would be an operator whose system rebuild fails. The `cache`
|
||||
# tier now compiles it on every main push, so a flake.lock bump that breaks it goes red HERE; the
|
||||
# dispatch input below is for checking it on a branch before merging.
|
||||
#
|
||||
# ⚠ pull_request is deliberately present. flatpak.yml shipped with push-only triggers and manifest
|
||||
# breakage reached main invisibly for weeks — do not "simplify" this workflow by dropping it.
|
||||
@@ -107,8 +121,15 @@ jobs:
|
||||
# real node (so actions/checkout works with no pre-checkout install dance), and audit.yml
|
||||
# already pulls it on this fleet, so it is proven to resolve here. Nix is installed below.
|
||||
image: node:22-bookworm
|
||||
timeout-minutes: 90
|
||||
# 180, not 90: the `cache` tier on a main push compiles the whole Rust workspace AND gamescope
|
||||
# from source inside the nix sandbox, where the sccache every other Rust job leans on cannot
|
||||
# reach (no network in a derivation, and RUSTC_WRAPPER is not set inside one).
|
||||
timeout-minutes: 180
|
||||
env:
|
||||
# Where the published cache lives on unom-1, and the URL users substitute from. Kept next to
|
||||
# the flatpak repo (3230) and winget source (3240) — see packaging/nix/server/.
|
||||
DEPLOY_DIR: unom-nix-cache
|
||||
CACHE_URL: https://nix.unom.io
|
||||
# The flake needs both experimental features. Also baked into the installer's --extra-conf
|
||||
# below; this covers any step that shells out before that config is read.
|
||||
NIX_CONFIG: "experimental-features = nix-command flakes"
|
||||
@@ -126,11 +147,12 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# The Determinate installer needs curl + xz; git so nix can read the flake from the checkout.
|
||||
# (node:22-bookworm is the full image and already has all three — this is belt-and-braces
|
||||
# against a future slim-image swap, and costs one cached apt call.)
|
||||
# The Determinate installer needs curl + xz; git so nix can read the flake from the checkout;
|
||||
# rsync + ssh to ship the built cache to unom-1. (node:22-bookworm is the full image and
|
||||
# already has all but rsync — this is belt-and-braces against a future slim-image swap, and
|
||||
# costs one cached apt call.)
|
||||
- name: Installer prerequisites
|
||||
run: apt-get update && apt-get install -y --no-install-recommends ca-certificates curl xz-utils git
|
||||
run: apt-get update && apt-get install -y --no-install-recommends ca-certificates curl xz-utils git rsync openssh-client
|
||||
|
||||
# `--init none` is the container mode: no systemd, no daemon. Running as root, nix then talks
|
||||
# to the store directly. Determinate Nix is also what the Nix box (.21) runs, so CI and the
|
||||
@@ -183,6 +205,133 @@ jobs:
|
||||
|| { echo "installed console is not a bun bundle" >&2; exit 1; }
|
||||
echo "bun packages OK: $web $scripting"
|
||||
|
||||
# ── binary cache (push to main only) ───────────────────────────────────────────────────────
|
||||
#
|
||||
# Decided against a bucket on storage.unom.io even though sccache already uses it and the
|
||||
# credentials already exist: it is local RustFS on the home uplink with no CDN in front, so
|
||||
# every NixOS user's download would come off the same pipe every CI runner uses — and S3
|
||||
# answers 403, not 404, for a missing key, which nix treats as a hard error rather than a
|
||||
# cache miss (see packaging/nix/server/Caddyfile). unom-1 already serves the flatpak repo
|
||||
# this way from a cloud IP; a Nix cache is the same static-files-behind-caddy shape.
|
||||
#
|
||||
# Gitea itself cannot host this at all: it has 23 package registry types and none is Nix, and
|
||||
# the binary cache protocol wants fixed anonymous paths at a URL root (/nix-cache-info,
|
||||
# /<hash>.narinfo, /nar/<hash>.nar.xz) that /api/packages/{owner}/generic/… cannot express.
|
||||
- name: Cache publish preflight
|
||||
id: cachecfg
|
||||
if: ${{ github.event_name == 'push' }}
|
||||
env:
|
||||
NIX_CACHE_SIGNING_KEY: ${{ secrets.NIX_CACHE_SIGNING_KEY }}
|
||||
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
|
||||
# Guard BEFORE the build, not before the upload: an unconfigured cache must not cost an
|
||||
# hour of rustc first. No-ops cleanly until the secret exists, exactly as flatpak.yml's
|
||||
# repo deploy does, so this workflow stays green through setup.
|
||||
run: |
|
||||
set -eu
|
||||
if [ -n "${NIX_CACHE_SIGNING_KEY:-}" ] && [ -n "${DEPLOY_HOST:-}" ]; then
|
||||
echo "go=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "go=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::warning::NIX_CACHE_SIGNING_KEY/DEPLOY_HOST not set — skipping the binary cache publish (see packaging/nix/README.md)."
|
||||
fi
|
||||
|
||||
- name: Build the publishable packages
|
||||
if: ${{ steps.cachecfg.outputs.go == 'true' }}
|
||||
# Everything a user can install. punktfunk-gamescope earns its place here more than any
|
||||
# other: host.gamescopeHdr DEFAULTS TRUE, so without it in the cache every
|
||||
# `services.punktfunk.host.enable = true` still compiles a compositor from source.
|
||||
run: |
|
||||
"$NIX" build --print-build-logs \
|
||||
.#punktfunk-host .#punktfunk-client .#punktfunk-tray \
|
||||
.#punktfunk-web .#punktfunk-scripting .#punktfunk-gamescope
|
||||
# This is now the heaviest job on the fleet — a full workspace build plus gamescope fills
|
||||
# the store with tens of GB, and this fleet ran a runner out of disk on 2026-08-06. Record
|
||||
# the headroom AFTER the build too, or a future "no space left on device" is a guess.
|
||||
df -h / /nix /tmp || true
|
||||
|
||||
- name: Sign + publish to nix.unom.io
|
||||
if: ${{ steps.cachecfg.outputs.go == 'true' }}
|
||||
env:
|
||||
NIX_CACHE_SIGNING_KEY: ${{ secrets.NIX_CACHE_SIGNING_KEY }}
|
||||
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
|
||||
DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
|
||||
DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }}
|
||||
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
|
||||
run: |
|
||||
# `set -eu`, NOT `set -euo pipefail`: act_runner may execute a step's `run:` under dash in
|
||||
# these containers (see scripts/ci/ensure-sccache.sh), and dash dies on `-o pipefail` with
|
||||
# "Illegal option". The two places below where a pipeline's LEFT side must be able to fail
|
||||
# the step are written as redirects instead, so nothing depends on pipefail.
|
||||
set -eu
|
||||
PKGS=".#punktfunk-host .#punktfunk-client .#punktfunk-tray .#punktfunk-web .#punktfunk-scripting .#punktfunk-gamescope"
|
||||
|
||||
# 1) Pick what to publish. PUBLISH ONLY OUR OWN PATHS — this is the difference between
|
||||
# ~300 MB and several GB per run, and it is not a corner cut: a runtime closure here
|
||||
# is our binaries plus stock nixpkgs (ffmpeg, gtk4, glibc, …), and every stock path is
|
||||
# already on cache.nixos.org, served by a real CDN. Mirroring them onto unom-1 would
|
||||
# cost disk and home-to-cloud bandwidth to serve a WORSE copy of what users already
|
||||
# have. Nothing in nixpkgs is named punktfunk, so the name filter is exact.
|
||||
paths="$("$NIX" path-info -r $PKGS | grep -- '-punktfunk' || true)"
|
||||
[ -n "$paths" ] || { echo "::error::no punktfunk store paths in the closure — the name filter is broken"; exit 1; }
|
||||
echo "$paths"
|
||||
# The filter is a string match, so it would fail SILENTLY if a pname ever changed — and
|
||||
# the package most likely to drift is gamescope, the most expensive one to lose. Assert
|
||||
# every built output is actually covered rather than discovering it as a user rebuild.
|
||||
for out in $("$NIX" build --print-out-paths $PKGS); do
|
||||
printf '%s\n' "$paths" | grep -qxF "$out" \
|
||||
|| { echo "::error::$out is not matched by the '-punktfunk' filter — publish would silently omit it"; exit 1; }
|
||||
done
|
||||
|
||||
# 2) Sign into a local binary cache. The secret is the whole `name:base64` line from
|
||||
# `nix key generate-secret`; the matching public key is what users pin (README).
|
||||
KEYDIR="$(mktemp -d)"; chmod 700 "$KEYDIR"
|
||||
printf '%s' "$NIX_CACHE_SIGNING_KEY" > "$KEYDIR/key"; chmod 600 "$KEYDIR/key"
|
||||
printf '%s\n' "$paths" | xargs "$NIX" copy --to "file://$PWD/nix-cache?secret-key=$KEYDIR/key"
|
||||
# Publish the PUBLIC half beside the cache and echo it here. Users must pin this key, so
|
||||
# it needs to be fetchable from the cache itself rather than only from a doc that can
|
||||
# drift — and on the first run this log line is where the value for README.md comes from.
|
||||
# Redirect, not `| tee`: without pipefail a failing nix would be masked by tee's success
|
||||
# and publish an EMPTY public key, which every user would then pin.
|
||||
"$NIX" key convert-secret-to-public < "$KEYDIR/key" > nix-cache/punktfunk-cache.pub
|
||||
cat nix-cache/punktfunk-cache.pub
|
||||
rm -rf "$KEYDIR"
|
||||
echo "publishing $(find nix-cache -name '*.narinfo' | wc -l) paths, $(du -sh nix-cache | cut -f1)"
|
||||
|
||||
# 3) Ship it. Same deploy key and retry discipline as flatpak.yml — this runner's link to
|
||||
# unom-1 drops TCP dials under load.
|
||||
install -d -m700 ~/.ssh
|
||||
printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy; chmod 600 ~/.ssh/deploy
|
||||
SSH="ssh -i $HOME/.ssh/deploy -p ${DEPLOY_PORT:-22} -o StrictHostKeyChecking=accept-new"
|
||||
DEST="${DEPLOY_USER}@${DEPLOY_HOST}"
|
||||
bash scripts/ci/retry.sh 5 $SSH "$DEST" "mkdir -p ~/$DEPLOY_DIR/site/nar"
|
||||
# ⚠ ORDER IS LOAD-BEARING: NARs first, narinfos second. A narinfo whose NAR has not landed
|
||||
# yet is a HARD download failure for whoever fetches it in that window; a NAR nothing
|
||||
# points at yet is simply invisible. rsync renames each file into place atomically, so a
|
||||
# cancelled run (this workflow has cancel-in-progress) can only ever under-publish.
|
||||
# No --delete: superseded paths are aged out by prune.sh below instead, so a client
|
||||
# mid-download is never pulled out from under.
|
||||
bash scripts/ci/retry.sh 5 rsync -az --info=stats1 -e "$SSH" nix-cache/nar/ "$DEST:$DEPLOY_DIR/site/nar/"
|
||||
bash scripts/ci/retry.sh 5 rsync -az -e "$SSH" nix-cache/nix-cache-info nix-cache/punktfunk-cache.pub nix-cache/*.narinfo "$DEST:$DEPLOY_DIR/site/"
|
||||
bash scripts/ci/retry.sh 5 rsync -az -e "$SSH" \
|
||||
packaging/nix/server/compose.production.yml packaging/nix/server/Caddyfile packaging/nix/server/prune.sh \
|
||||
"$DEST:$DEPLOY_DIR/"
|
||||
bash scripts/ci/retry.sh 5 $SSH "$DEST" "cd ~/$DEPLOY_DIR && docker compose -f compose.production.yml up -d"
|
||||
|
||||
# 4) Bound it. The flatpak repo next door reached 3.84 GB publishing this same way with
|
||||
# no sweep, on a box that has run out of disk before; this one gets the sweep from the
|
||||
# first publish. Never allowed to fail the job — the cache is already live by now, and
|
||||
# a growing disk is a slower problem than a red main.
|
||||
bash scripts/ci/retry.sh 3 $SSH "$DEST" "sh ~/$DEPLOY_DIR/prune.sh ~/$DEPLOY_DIR/site 180" \
|
||||
|| echo "::warning::cache prune failed — published cache may be growing unbounded"
|
||||
|
||||
# 5) Prove the published cache actually answers, rather than assuming the rsync landed.
|
||||
# A substituter that 200s on nix-cache-info but 403s on a miss is the failure mode that
|
||||
# breaks users' builds, so check both.
|
||||
bash scripts/ci/retry.sh 5 curl -fsS "$CACHE_URL/nix-cache-info"
|
||||
miss="$(curl -sS -o /dev/null -w '%{http_code}' "$CACHE_URL/0000000000000000000000000000000000.narinfo")"
|
||||
[ "$miss" = 404 ] || { echo "::error::cache returns $miss for an absent path; nix needs 404 or every user build fails"; exit 1; }
|
||||
echo "published → $CACHE_URL"
|
||||
|
||||
# Opt-in only: the full Rust workspace through crane, which is the hour-long leg.
|
||||
# `github.event.inputs.*` (string) rather than `inputs.*` — the portable spelling.
|
||||
- name: Build the Rust packages (dispatch opt-in)
|
||||
|
||||
@@ -32,8 +32,9 @@ On **Windows**, the host ships as a signed installer instead — see [Windows](#
|
||||
|
||||
Each registry is public — no auth, you just trust the repo's signing key. Adding the repo is a
|
||||
one-time step covered in the linked guide; after that, normal `apt upgrade` / `dnf upgrade` /
|
||||
`pacman -Syu` (or `sudo punktfunk-sysext update` on Bazzite) tracks new builds. On **NixOS** there
|
||||
is no repo to add — you add the flake as an input and enable its module, see [NixOS](#nixos).
|
||||
`pacman -Syu` (or `sudo punktfunk-sysext update` on Bazzite) tracks new builds. On **NixOS** you add
|
||||
the flake as an input and enable its module rather than adding a package repo — but do add the
|
||||
[binary cache](#nixos), or every build compiles from source.
|
||||
|
||||
> **Stable vs canary.** The repos in the per-distro guides are the **stable** channel — it only
|
||||
> moves when a `vX.Y.Z` release is cut. For the latest `main` build (fast, possibly broken), point
|
||||
@@ -93,6 +94,22 @@ The repo's `flake.nix` is a supported install path: it builds `punktfunk-host`,
|
||||
`punktfunk-web` and `punktfunk-scripting`, and ships a NixOS module. **`x86_64-linux` only**, and
|
||||
NixOS **24.11 or newer**.
|
||||
|
||||
**Add the binary cache first.** Without it, a build compiles the whole Rust workspace *and*
|
||||
gamescope from source — about an hour. With it you get prebuilt binaries:
|
||||
|
||||
```nix
|
||||
nix.settings = {
|
||||
substituters = [ "https://nix.unom.io" ];
|
||||
trusted-public-keys = [ "punktfunk-cache-1:<key>" ]; # curl https://nix.unom.io/punktfunk-cache.pub
|
||||
};
|
||||
```
|
||||
|
||||
Off NixOS, put the same two values in `/etc/nix/nix.conf` as `extra-substituters` /
|
||||
`extra-trusted-public-keys`. One caveat worth knowing before you copy a flake snippet from
|
||||
elsewhere: setting `inputs.punktfunk.inputs.nixpkgs.follows = "nixpkgs"` changes every store path
|
||||
and so misses the cache entirely — details in
|
||||
[packaging/nix](https://git.unom.io/unom/punktfunk/src/branch/main/packaging/nix/README.md#binary-cache-do-this-before-your-first-build).
|
||||
|
||||
You can run it straight from the flake without NixOS (on other distros, wrap it in
|
||||
[nixGL](https://github.com/nix-community/nixGL) so the GPU drivers resolve):
|
||||
|
||||
|
||||
+128
-5
@@ -45,6 +45,69 @@ GPU drivers are resolved at runtime from `/run/opengl-driver/lib`. On non-NixOS
|
||||
[nixGL](https://github.com/nix-community/nixGL) so that path is populated (`nixGL nix run …`); on
|
||||
NixOS the module (below) sets `hardware.graphics.enable = true` for you.
|
||||
|
||||
> **Do this first, or the commands above compile the world:** without the binary cache, `nix build`
|
||||
> here means the whole Rust workspace *and* a gamescope build from source — roughly an hour on a
|
||||
> fast machine. See below.
|
||||
|
||||
---
|
||||
|
||||
## Binary cache (do this before your first build)
|
||||
|
||||
CI publishes every punktfunk package to **`https://nix.unom.io`** on each push to `main` that moves
|
||||
the flake, so you get prebuilt binaries instead of an hour of `rustc`. It covers
|
||||
`punktfunk-host`, `-client`, `-tray`, `-web`, `-scripting` and `-gamescope` — everything the flake
|
||||
builds from source. Everything else in the closure is stock nixpkgs and comes from `cache.nixos.org`
|
||||
as usual, so the cache is deliberately small and adding it costs you nothing on unrelated builds.
|
||||
|
||||
**NixOS** — in your system configuration:
|
||||
|
||||
```nix
|
||||
nix.settings = {
|
||||
substituters = [ "https://nix.unom.io" ];
|
||||
trusted-public-keys = [ "punktfunk-cache-1:<PUBLIC KEY — see below>" ];
|
||||
};
|
||||
```
|
||||
|
||||
**Anywhere else** — in `/etc/nix/nix.conf` (or `~/.config/nix/nix.conf` if you are a trusted user):
|
||||
|
||||
```conf
|
||||
extra-substituters = https://nix.unom.io
|
||||
extra-trusted-public-keys = punktfunk-cache-1:<PUBLIC KEY — see below>
|
||||
```
|
||||
|
||||
The current public key is served by the cache itself, so you can always check it against the source
|
||||
of truth:
|
||||
|
||||
```sh
|
||||
curl https://nix.unom.io/punktfunk-cache.pub
|
||||
```
|
||||
|
||||
Verify the cache is being used — this should print the store paths without compiling anything:
|
||||
|
||||
```sh
|
||||
nix build --dry-run git+https://git.unom.io/unom/punktfunk#punktfunk-host
|
||||
```
|
||||
|
||||
### ⚠ `nixpkgs.follows` turns the cache off
|
||||
|
||||
Every store path is keyed by the exact inputs it was built from. Pointing punktfunk's nixpkgs at
|
||||
yours changes those inputs, so **every** path misses and you compile the workspace anyway:
|
||||
|
||||
```nix
|
||||
# Convenient, but it costs you the entire binary cache:
|
||||
inputs.punktfunk.inputs.nixpkgs.follows = "nixpkgs";
|
||||
```
|
||||
|
||||
That is a real trade, not a bug — `follows` buys you one shared nixpkgs in the closure instead of
|
||||
two. Take it if closure size matters more to you than build time; leave it out to get binaries.
|
||||
|
||||
### Why not `cachix`?
|
||||
|
||||
Nothing against it — punktfunk simply self-hosts every other channel (flatpak, deb, rpm, Arch,
|
||||
docker, winget), and a Nix cache is static files behind a web server, so it rides the same unom-1
|
||||
box and the same deploy key as the rest. Nothing about the cache is punktfunk-specific: it speaks
|
||||
plain HTTP binary-cache protocol, so any nix client works with it.
|
||||
|
||||
---
|
||||
|
||||
## NixOS module
|
||||
@@ -55,6 +118,8 @@ Add the flake and enable the host and/or client:
|
||||
{
|
||||
inputs.punktfunk.url = "git+https://git.unom.io/unom/punktfunk";
|
||||
# (optional) share your nixpkgs: inputs.punktfunk.inputs.nixpkgs.follows = "nixpkgs";
|
||||
# ⚠ this DISABLES the binary cache — different inputs, different store paths, so every
|
||||
# package is rebuilt from source (~1h). See "Binary cache" above.
|
||||
|
||||
outputs = { self, nixpkgs, punktfunk, ... }: {
|
||||
nixosConfigurations.myhost = nixpkgs.lib.nixosSystem {
|
||||
@@ -369,8 +434,66 @@ RUNPATH (`/run/opengl-driver/lib`) and the GTK GApps wrapper (GSettings schemas
|
||||
are present. Fixes discovered during that bring-up: `CMAKE_POLICY_VERSION_MINIMUM=3.5` (CMake ≥ 4),
|
||||
system `libopus` (audiopus_sys), and the session Skia note above.
|
||||
|
||||
In CI (`.gitea/workflows/nix.yml`): `nix flake check --no-build` evaluates every output *including*
|
||||
the module check above, and `punktfunk-web` + `punktfunk-scripting` are built for real. The Rust
|
||||
packages and `punktfunk-gamescope` are `workflow_dispatch` opt-ins (`build-rust`,
|
||||
`build-gamescope`) — run the latter after a `flake.lock` bump, since it patches whatever gamescope
|
||||
the pinned nixpkgs carries.
|
||||
In CI (`.gitea/workflows/nix.yml`), three tiers: `nix flake check --no-build` evaluates every output
|
||||
*including* the module check above; `punktfunk-web` + `punktfunk-scripting` are built for real on
|
||||
every PR; and on a push to `main` the Rust packages and `punktfunk-gamescope` are built and
|
||||
published to the binary cache. A `flake.lock` bump that breaks the gamescope patches therefore goes
|
||||
red on main rather than in an operator's rebuild. The `build-rust` / `build-gamescope`
|
||||
`workflow_dispatch` inputs remain, for checking those on a branch before merging.
|
||||
|
||||
---
|
||||
|
||||
## Cache infrastructure (maintainers)
|
||||
|
||||
`https://nix.unom.io` is a `caddy:2-alpine` container on unom-1 serving a static directory —
|
||||
`packaging/nix/server/` — exactly like the flatpak repo (3230) and the winget source (3240). A Nix
|
||||
binary cache *is* just `nix-cache-info` + `<hash>.narinfo` + `nar/<hash>.nar.xz` behind a web
|
||||
server; there is no cache daemon to run.
|
||||
|
||||
**Why not Gitea, and why not `storage.unom.io`:**
|
||||
|
||||
- Gitea has 23 package registry types and none is Nix. It is not a missing label — the protocol
|
||||
needs fixed anonymous paths at a URL *root* (`/nix-cache-info`, `/<hash>.narinfo`,
|
||||
`/nar/…`), and `/api/packages/{owner}/generic/{name}/{version}/{file}` cannot express them.
|
||||
- The RustFS S3 at `storage.unom.io` *would* work mechanically (nix speaks `s3://…?endpoint=`, and
|
||||
the sccache credentials already exist), but it is a local box on the home uplink with no CDN, so
|
||||
every user download competes with CI. It also answers **403** for a missing key unless the bucket
|
||||
policy grants anonymous `ListBucket` — and nix treats anything other than **404** as a hard error
|
||||
rather than a cache miss, which would break users' builds for packages the cache never held.
|
||||
|
||||
**One-time setup — in this order.** The publish step ends by fetching `nix.unom.io` to prove the
|
||||
cache really answers (and answers **404**, not 403, for a path it does not hold), so stand the
|
||||
service up *before* you set the secret that switches publishing on. The secret is the last step for
|
||||
exactly that reason: until it exists the publish no-ops with a warning and `main` stays green,
|
||||
the same way flatpak.yml's repo deploy does.
|
||||
|
||||
1. **Edge proxy:** `nix.unom.io { reverse_proxy 192.168.50.50:3250 }` on home-reverse-proxy-1.
|
||||
2. **Port allowlist:** add `3250` to `caddy_target_ports` in `unom/infra` (proxmox/unom-1) +
|
||||
terraform apply.
|
||||
3. **DNS:** ensure `nix.unom.io` resolves to the edge proxy.
|
||||
4. Dispatch `deploy-services.yml` (or `unom/infra`'s `deploy-all`) to bring the container up. It
|
||||
serves an empty cache — every path 404s, which is exactly what a healthy empty cache does.
|
||||
5. Generate the signing key on a Nix box and store the secret half as the repo Actions secret
|
||||
`NIX_CACHE_SIGNING_KEY` (the whole `name:base64` line):
|
||||
```sh
|
||||
nix key generate-secret --key-name punktfunk-cache-1
|
||||
```
|
||||
6. Push to `main` touching the flake. The publish step prints the **public** key — paste it into the
|
||||
"Binary cache" section above (and `docs-site/content/docs/install.md`) and commit.
|
||||
|
||||
**Operational notes:**
|
||||
|
||||
- Only punktfunk's own store paths are published (`nix path-info -r … | grep -- '-punktfunk'`).
|
||||
Everything else in a closure is stock nixpkgs, already on `cache.nixos.org` behind a real CDN;
|
||||
mirroring it would cost disk and home-to-cloud bandwidth to serve a worse copy. The publish step
|
||||
asserts every built output is matched by that filter, so a future `pname` change fails the build
|
||||
instead of silently dropping a package from the cache.
|
||||
- `rsync` runs **without** `--delete` (a client mid-download is never pulled out from under), and
|
||||
NARs are uploaded *before* narinfos — a narinfo whose NAR has not landed is a hard download
|
||||
failure for whoever fetches it in that window, while an unreferenced NAR is merely invisible.
|
||||
- Growth is bounded by `packaging/nix/server/prune.sh` (evicts narinfos untouched for 180 days,
|
||||
then sweeps NARs nothing references). The flatpak repo next door reached 3.84 GB publishing the
|
||||
same way with no sweep, on a box that has run out of disk before — hence the sweep from the first
|
||||
publish. Run its self-check with `sh packaging/nix/server/prune.sh --self-test`.
|
||||
- A user on a pinned rev older than the eviction window falls back to building from source, which
|
||||
is the pre-cache status quo.
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
# Inner Caddy (plain HTTP on :3250); the edge proxy on home-reverse-proxy-1 does TLS for
|
||||
# nix.unom.io. Serves the punktfunk Nix binary cache — which is nothing but a static tree:
|
||||
#
|
||||
# /nix-cache-info store dir + priority, fetched once per substituter
|
||||
# /<32-char-hash>.narinfo one per store path
|
||||
# /nar/<hash>.nar.xz the archives themselves
|
||||
#
|
||||
# ⚠ A MISSING PATH MUST 404, NOT 403. Nix reads 404 as "not in this cache, try the next
|
||||
# substituter" and treats anything else as a hard error that fails the build — so a cache
|
||||
# answering 403 for unknown hashes breaks every user who adds it, including for packages it
|
||||
# was never meant to serve. `file_server` 404s correctly; do not put an auth wrapper in front
|
||||
# of this without preserving that. This is also the concrete reason the cache is NOT a bucket
|
||||
# on storage.unom.io: S3 answers 403 for a missing key unless the bucket policy grants
|
||||
# anonymous ListBucket, and that box is on the home uplink besides.
|
||||
:3250 {
|
||||
root * /srv
|
||||
file_server browse
|
||||
|
||||
# Everything except nix-cache-info is content-addressed by the store hash and can never
|
||||
# change meaning — a narinfo for a given hash is as immutable as the NAR it points at.
|
||||
@immutable path /nar/* *.narinfo
|
||||
header @immutable Cache-Control "public, max-age=31536000, immutable"
|
||||
|
||||
# The only mutable file, and cheap to revalidate: nix reads it once per substituter per run.
|
||||
@info path /nix-cache-info
|
||||
header @info Cache-Control "public, max-age=300"
|
||||
|
||||
# nix does not care about Content-Type, but a browser poking at the cache should not be
|
||||
# offered a download for what is a two-line text file.
|
||||
@text path *.narinfo /nix-cache-info
|
||||
header @text Content-Type "text/plain; charset=utf-8"
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
# Static file server for the punktfunk Nix binary cache, on unom-1 (the DMZ services VM) —
|
||||
# the same shape as packaging/flatpak/server, because a Nix cache and an OSTree repo are both
|
||||
# just immutable files behind a web server. Caddy on home-reverse-proxy-1 terminates TLS for
|
||||
# nix.unom.io and reverse_proxies to 192.168.50.50:3250 (the port must be in unom/infra
|
||||
# caddy_target_ports). This inner Caddy serves the bind-mounted ./site tree over plain HTTP.
|
||||
# nix.yml rsyncs into ./site and runs `docker compose up -d` (idempotent).
|
||||
#
|
||||
# Port 3250: docs is 3220, flatpak 3230, winget 3240 — keep the run going.
|
||||
services:
|
||||
nix-cache:
|
||||
image: caddy:2-alpine
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "3250:3250"
|
||||
volumes:
|
||||
- ./Caddyfile:/etc/caddy/Caddyfile:ro
|
||||
- ./site:/srv:ro
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
#!/bin/sh
|
||||
# Bound the published Nix binary cache on unom-1.
|
||||
#
|
||||
# WHY THIS EXISTS UP FRONT, rather than being added after the box fills: the flatpak repo next
|
||||
# door taught this exact lesson the expensive way. It publishes with rsync WITHOUT --delete (so
|
||||
# a client mid-download is never broken), nothing ever removed the superseded objects, and it
|
||||
# reached 3.84 GB on a box that had already run out of disk once. This cache has the same
|
||||
# publish model and the same growth shape — every build whose inputs moved adds a fresh set of
|
||||
# store paths and keeps the old ones — so it gets the sweep from day one.
|
||||
#
|
||||
# The order below is the whole correctness argument:
|
||||
#
|
||||
# 1. Delete narinfos older than $DAYS. rsync -a carries the CI-side mtime over, and `nix copy`
|
||||
# rewrites every narinfo it publishes on every run, so a path that is still being published
|
||||
# keeps getting a fresh mtime. Age therefore means "no publish has referenced this in
|
||||
# $DAYS", which is exactly the eviction signal wanted.
|
||||
# 2. THEN delete NARs no surviving narinfo points at.
|
||||
#
|
||||
# Doing it the other way round — or aging the NARs independently — can strand a live narinfo
|
||||
# pointing at a deleted NAR, and that is strictly worse than a cache miss: nix reports a missing
|
||||
# NAR as a hard download failure, not as "not cached, build it yourself".
|
||||
#
|
||||
# POSIX sh: this runs over ssh on unom-1 (Debian), invoked by .gitea/workflows/nix.yml.
|
||||
#
|
||||
# Usage: sh prune.sh <cache-dir> [max-age-days] (default 180)
|
||||
# sh prune.sh --self-test
|
||||
set -eu
|
||||
|
||||
self_test() {
|
||||
# Smallest thing that fails if the ordering or the reference sweep breaks.
|
||||
t="$(mktemp -d)"
|
||||
trap 'rm -rf "$t"' EXIT
|
||||
mkdir -p "$t/nar"
|
||||
printf 'StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 41\n' >"$t/nix-cache-info"
|
||||
|
||||
# A live path, a stale one, and a NAR nothing ever pointed at.
|
||||
printf 'StorePath: /nix/store/aaa-live\nURL: nar/live.nar.xz\n' >"$t/aaa.narinfo"
|
||||
printf 'StorePath: /nix/store/bbb-stale\nURL: nar/stale.nar.xz\n' >"$t/bbb.narinfo"
|
||||
: >"$t/nar/live.nar.xz"
|
||||
: >"$t/nar/stale.nar.xz"
|
||||
: >"$t/nar/orphan.nar.xz"
|
||||
# Two paths that SHARE a NAR, one stale and one live: the shared NAR must survive. A sweep
|
||||
# that deleted NARs per-evicted-narinfo instead of by surviving references would drop it.
|
||||
printf 'StorePath: /nix/store/ccc-live\nURL: nar/shared.nar.xz\n' >"$t/ccc.narinfo"
|
||||
printf 'StorePath: /nix/store/ddd-stale\nURL: nar/shared.nar.xz\n' >"$t/ddd.narinfo"
|
||||
: >"$t/nar/shared.nar.xz"
|
||||
|
||||
# Age the stale ones well past any plausible threshold (portable -t form: YYYYMMDDhhmm).
|
||||
touch -t 200001010000 "$t/bbb.narinfo" "$t/ddd.narinfo" "$t/nar/stale.nar.xz"
|
||||
|
||||
prune "$t" 180
|
||||
|
||||
fail() { echo "SELF-TEST FAILED: $1" >&2; exit 1; }
|
||||
[ -f "$t/aaa.narinfo" ] || fail "evicted a fresh narinfo"
|
||||
[ -f "$t/nar/live.nar.xz" ] || fail "evicted a referenced NAR"
|
||||
[ -f "$t/nix-cache-info" ] || fail "deleted nix-cache-info"
|
||||
[ ! -f "$t/bbb.narinfo" ] || fail "kept a stale narinfo"
|
||||
[ ! -f "$t/nar/stale.nar.xz" ] || fail "kept a NAR nothing references any more"
|
||||
[ ! -f "$t/nar/orphan.nar.xz" ] || fail "kept an orphan NAR"
|
||||
[ -f "$t/nar/shared.nar.xz" ] || fail "deleted a NAR a surviving narinfo still references"
|
||||
echo "prune.sh self-test OK"
|
||||
}
|
||||
|
||||
prune() {
|
||||
root="$1"
|
||||
days="$2"
|
||||
cd "$root"
|
||||
|
||||
before="$(du -sh . 2>/dev/null | cut -f1)"
|
||||
|
||||
# 1. Age out the narinfos.
|
||||
find . -maxdepth 1 -name '*.narinfo' -mtime "+$days" -delete
|
||||
|
||||
# 2. Sweep NARs nothing points at any more. Both lists are relative to $root and spelled the
|
||||
# same way ("nar/<file>") so `comm` can diff them.
|
||||
keep="$(mktemp)"
|
||||
have="$(mktemp)"
|
||||
# `|| true`: an empty cache (or one whose narinfos were all just evicted) makes the glob
|
||||
# match nothing, and an empty keep-list is the correct answer there, not an error.
|
||||
cat ./*.narinfo 2>/dev/null | sed -n 's|^URL: ||p' | sort -u >"$keep" || true
|
||||
find nar -type f 2>/dev/null | sed 's|^\./||' | sort >"$have" || true
|
||||
comm -13 "$keep" "$have" | tr '\n' '\0' | xargs -0 -r rm -f
|
||||
rm -f "$keep" "$have"
|
||||
|
||||
# Leave the numbers in the deploy log — this is the only place the published size is visible.
|
||||
echo "cache pruned (narinfos older than ${days}d): ${before:-?} -> $(du -sh . 2>/dev/null | cut -f1) in $(pwd)"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
--self-test) self_test ;;
|
||||
"") echo "usage: prune.sh <cache-dir> [max-age-days] | --self-test" >&2; exit 2 ;;
|
||||
*) prune "$1" "${2:-180}" ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user