From c9d5ed575ce08e33e8e84fc12e79337d01e2d01c Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Thu, 1 Oct 2026 13:05:31 +0200 Subject: [PATCH] gitea-image-prune.sh: keep the newest 20 and every version in use; refuse when the in-use list is unreadable (Felhom 09 decision 62, R-750) Protected whatever --keep says: the controller floor, the vouched golden, the vouched agent and min_agent (the hub's Configuration page), every image of ours the vouched golden baked (its bake.log), the hub manifest's image. The dry-run prints each kept version and why. tests/test-prune-plan.sh pins it without network (red-proofed). No --apply was run. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- CHANGELOG.md | 14 ++++ README.md | 23 +++--- REPORT.md | 116 +++--------------------------- gitea-image-prune.sh | 147 ++++++++++++++++++++++++++++++++++----- tests/test-prune-plan.sh | 42 +++++++++++ 5 files changed, 209 insertions(+), 133 deletions(-) create mode 100755 tests/test-prune-plan.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 39190ab..741d94a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ All notable changes to the operator helper scripts. Newest on top. +## 2026-10-01 + +### Changed — `gitea-image-prune.sh`: the retention rule (Felhom `09` §3 decision 62, R-750) +- A prune keeps the newest **20** versions (`--keep` defaults to 20) **plus every version in use**: the controller + floor, the vouched golden, the vouched agent and its `min_agent` (the hub's operator Configuration page, `HUB_PW`), + every image of ours the vouched golden baked (its `bake.log` in `felhom.eu`), and the hub image `manifests/hub.yaml` + runs. The dry-run prints each kept version and why. **An unreadable in-use list refuses the prune (exit 3).** +- `tests/test-prune-plan.sh` (no network: test seams `PRUNE_TEST_VERSIONS`, `PRUNE_TEST_PROTECT`): an in-use version + older than the newest 20 stays; red-proof: without the in-use list it is deleted, and with the check removed from + `is_protected` the test fails. +- The "cron-friendly" and "set-and-forget Gitea rule" advice removed: nothing schedules a prune; `--apply` is a person's act. +- Recorded: the 2026-08-22 16:02 UTC run (`--all --keep 7 --apply`, HM-024) predates this rule — it is why the oldest + `felhom-controller` left is 0.213.0. + ## 2026-06-17 (later) ### Changed — `gitea-image-prune.sh` diff --git a/README.md b/README.md index c3eedb6..5f7ad0a 100644 --- a/README.md +++ b/README.md @@ -96,21 +96,28 @@ GITEA_TOKEN=… ./gitea-image-prune.sh ./gitea-image-prune.sh --all list ./gitea-image-prune.sh --all --no-sizes list # fast, skip size resolution -# Dry-run prune (DEFAULT — shows what would go, mutates nothing): -./gitea-image-prune.sh --repo felhom-hub --keep 10 +# Dry-run prune (DEFAULT — shows what would go, mutates nothing). --keep defaults to 20: +./gitea-image-prune.sh --all prune # containers +./gitea-image-prune.sh --type generic --all prune # agent + golden ./gitea-image-prune.sh --repo felhom-controller --older-than 90 -# Apply for real (typed confirmation unless --yes): -./gitea-image-prune.sh --repo felhom-hub --keep 10 --apply -./gitea-image-prune.sh --repo felhom-hub --keep 10 --apply --reclaim --measure +# Apply for real — a PERSON's act, after reading the dry-run (typed confirmation unless --yes): +./gitea-image-prune.sh --all prune --apply --reclaim --measure # Reclaim only (delete orphaned manifests + trigger/await GC): ./gitea-image-prune.sh --repo felhom-hub reclaim --apply - -# Cron-friendly, non-interactive, all packages: -./gitea-image-prune.sh --all --keep 15 --apply --yes --reclaim ``` +### The retention rule (Felhom `09` §3 decision 62, operator ruling 2026-10-01) + +A prune keeps the **newest 20** versions of every package **plus every version in use**, whatever `--keep` or +`--older-than` says: the controller floor, the vouched golden, the vouched agent and its `min_agent` (read from the +hub's operator Configuration page with `HUB_PW` — env, or `~/.config/credentials` through +`felhom.eu/scripts/read_credential.py`), every `gitea.dooplex.hu/admin/:` the vouched golden baked (its +`bake.log` under `felhom.eu/documentation/tests/`), and the hub image `felhom.eu/manifests/hub.yaml` runs. The dry-run +prints each kept version with its reason. **If any of these cannot be read, the prune refuses (exit 3).** Nothing runs +this script on a schedule; `--apply` is a person's act. `tests/test-prune-plan.sh` pins the rule without network. + ### Flags | Flag | Meaning | diff --git a/REPORT.md b/REPORT.md index fa06aeb..46ca7b4 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,108 +1,12 @@ -# REPORT — `gitea-image-prune.sh` (2026-06-17) +# REPORT — gitea-image-prune.sh gets the retention rule (2026-10-01) -New operator CLI to inspect/prune old container images in the Gitea registry and -reclaim disk, with a load-bearing live spike to **prove** the reclaim mechanism. +Felhom `09` §3 decision 62 (operator ruling, R-750). Full report: `felhom.eu/REPORT-calibre-name-and-prune-2026-10-01.md`. -## Confirmed baselines (live) - -| Thing | Value | -|---|---| -| Gitea version | **1.26.2** (`GET /api/v1/version`) | -| Owner namespace | `admin` (standard per-owner packages API; "admin" in the path is the owner) | -| Container packages | `felhom-controller` **96 tags / 247 digests**, `felhom-hub` **42 tags / 96 digests**, plus `recipe-importer` (42), `revfulop-calendar` (12), `jarr` (2), `wan-probe` (1) | -| Packages cron | `cleanup_packages`, default `@midnight`, `OLDER_THAN = 24h` | -| Packages dir | `/data/gitea/packages` (gitea-system pod, container `gitea`), **baseline 5,122,143,723 B ≈ 4.77 GiB** | -| Tooling | `jq` 1.7 present on build server; `shellcheck` **absent** (not run — script written carefully, `bash -n` clean) | - -## Minimal token scope set (empirically confirmed via 403 bodies) - -| Operation | Required scope | -|---|---| -| list packages / versions / files | `read:package` | -| delete a tag / manifest version | `write:package` | -| list cron tasks | `read:admin` | -| trigger `cleanup_packages` cron | **`write:admin`** | - -The build server token (`~/.gitea-token`) has `read:admin` + `write:package` but -**not** `write:admin` (its 403 body named exactly `required=[write:admin]`), so it -can list/prune/delete-orphans but cannot trigger the GC cron on demand. Token must -belong to a site-admin user. (Scopes were *not* minimized by minting reduced -tokens — that needs `write:user`, which this token also lacks — but each required -scope was confirmed by a successful call and the cron requirement by its 403.) - -## §3 spike — the reclaim mechanism (PROVEN, not assumed) - -Gitea stores a tag as a tiny OCI **index** pointer; the real bytes are in untagged -`sha256:` **manifest versions** (config + layer blobs), whose layers are shared -across tags. The mechanism turned out to be **three steps**, not two: - -| Step (single-version spike, `felhom-hub`) | `du` (bytes) | freed | -|---|---|---| -| baseline | 5,122,143,723 | — | -| DELETE tag `0.1.1` (via the script, HTTP 204) | 5,122,143,723 | **0** | -| run `cleanup_packages` (tag-only) | 5,122,138,771 | ~5 KB (index pointer only) | -| DELETE tag `0.1.2` + its **2 orphaned manifests** (204×3) | 5,122,138,771 | **0** | -| run `cleanup_packages` GC | 5,116,799,814 | **5,338,957 B ≈ 5.1 MiB** | - -**Conclusions:** -1. Deleting a tag frees ~nothing (only the index pointer). -2. **Default `cleanup_packages` does NOT remove untagged manifest versions** — - only unreferenced *blobs*. So the orphaned `sha256:` manifests must be deleted - explicitly (the script's reclaim does this); otherwise their blobs stay - referenced forever. (Confirmed: a tag-only delete + cron left `felhom-hub` - digests at 96.) -3. Once the orphaned manifests are deleted, `cleanup_packages` GCs their *unique* - blobs (created > `OLDER_THAN`); shared base layers stay. 5.1 MiB freed for one - 9.4 MB-apparent image — the difference is shared layers, correctly retained. - -Because the token lacks `write:admin`, the GC cron was triggered by adding -`[cron.cleanup_packages] RUN_AT_START = true` to `app.ini` (on the data PVC, -backup `app.ini.bak.prune-spike`) and rolling-restarting Gitea — left in place per -operator request (the daily `@midnight` run also performs the GC). - -## §9 verification results - -1. **List (read-only):** `--repo felhom-hub list` → 42 tags, newest-first, per-tag - sizes resolved via OCI (24 MB recent, ~9 MB older), `latest` flagged PROTECTED, - shared-layer caveat printed. `--all` lists all 6 packages; `--no-sizes` fast path - works. No mutation. -2. **Dry-run prune:** `--repo felhom-hub --keep 5 --dry-run` → would delete 36, - keep 5 + 1 protected, oldest first, totals shown, **nothing changed**. -3. **One-version live proof (spike):** see table above — delete-alone = 0 bytes; - delete + orphan-manifest delete + GC = 5.1 MiB. -4. **Full reclaim path validated** on `felhom-hub` only: `reclaim --apply` deleted - the **16 accumulated orphan manifests** (untagged, referenced by no tag — dead - weight from re-pointed `latest` + buildx attestations), then GC freed - **5,116,799,814 → 5,026,431,222 = 90,368,592 B ≈ 86 MiB**. -5. **Safety:** after reclaim, surviving tags still resolve and **`docker pull` - cleanly** (`latest`, `0.1.3` — the immediate neighbor of the deleted tags). - Orphan detection is fail-closed (skips a package if any surviving tag won't - resolve). -6. **Audit log** captured every RUN / DRY-RUN / APPLIED / RECLAIM line; **token - scan of the log = 0 hits**. Edge cases: `--keep`+`--older-than` → error; - `--keep 999` → "Nothing to prune (40 tags: 39 kept, 1 protected)". - -## State left on the registry - -- `felhom-hub`: tags `0.1.1` and `0.1.2` deleted (spike); 16 orphan manifests - cleaned; now **40 tags / 78 digests**; ~91 MiB reclaimed total. All remaining - tags pull cleanly. -- **`felhom-controller` (96 tags) and all other packages: UNTOUCHED.** -- `app.ini`: `RUN_AT_START = true` added for `cleanup_packages` (kept). - -## NOT yet run - -**The real bulk cleanup — left to the operator (interactive).** This run proved -the mechanism on one disposable version and validated the full reclaim path on -`felhom-hub`'s dead orphans only. Pruning the ~90 `felhom-controller` tags (and -the bulk of `felhom-hub`/`recipe-importer` history) is the operator's call via -`gitea-image-prune.sh --repo … --keep N --apply --reclaim`. - -## Backlog / notes - -- A `write:admin` token (or the native cleanup rule in the UI) would let `reclaim` - trigger the GC immediately instead of relying on the `@midnight`/restart run. -- Per-tag "apparent" sizes overlap (shared base layers counted once per tag); - `--measure` (`du`) is the honest real-reclaim signal. Documented in the tool. -- `shellcheck` was unavailable on the build server, so the script was not - statically linted (only `bash -n` syntax-checked + extensively run live). +- **Rule:** keep the newest 20 + every version in use (floor, vouched golden, vouched agent, `min_agent`, the golden's + baked images, the running hub); refuse (exit 3) when the in-use list cannot be read; `--apply` only by a person. +- **Test:** `tests/test-prune-plan.sh` — 7 checks pass; red-proof: the in-use check removed from `is_protected` → 3 fail + (the plan deletes 5 and lists 0.262.0). +- **Live dry-run, 2026-10-01 (nothing deleted):** in use — controller 0.285.0, golden 0.285.0, agent 0.138.0 and 0.131.0, + hub 0.126.0, felhom-samba 1.1.0. Would delete: felhom-controller 70 tags, felhom-hub 8; every other package nothing. + Evidence: `felhom.eu/documentation/audits/calibre-name-and-prune-2026-10-01/B/`. +- **No `--apply` was run.** diff --git a/gitea-image-prune.sh b/gitea-image-prune.sh index a3514fe..7ff1787 100755 --- a/gitea-image-prune.sh +++ b/gitea-image-prune.sh @@ -15,6 +15,18 @@ # so felhom-controller (~96 tags) / felhom-hub (~42 tags) accumulate one image # per build and the Gitea Longhorn PVC keeps filling. # +# RETENTION RULE — Felhom `09-update-architecture.md` §3 decision 62 (operator ruling 2026-10-01, R-750) +# A prune keeps the newest 20 versions of each package (--keep defaults to 20 when +# pruning) PLUS every version in use, whatever --keep or --older-than says: +# - the vouched golden, the controller floor, the vouched agent and its min_agent +# (read from the hub's operator Configuration page — Basic auth, HUB_PW); +# - every gitea.dooplex.hu/admin/: the vouched golden BAKED (its bake.log +# in felhom.eu/documentation/tests/golden--*/); +# - the hub image the GitOps manifest runs (felhom.eu/manifests/hub.yaml). +# If any of those cannot be read, a prune REFUSES (exit 3) — never "protect nothing". +# --apply is a person's act: nothing schedules this script (no cron, no timer). +# The August 2026 run (`--all --keep 7 --apply`, 2026-08-22, HM-024) predates this rule. +# # PACKAGE TYPES — --type, default "container" (added 2026-08-23) # Gitea namespaces packages by TYPE, and every API path embeds it. This script # handled only type=container, so --all meant "all *container* packages" and @@ -22,9 +34,9 @@ # admin/felhom-golden (22 versions x ~627 MB = 13 GB) grew to 88% of the Gitea # PVC while --all --keep 7 --apply reported success: it pruned ~1.3 GB of # container images and never saw the 13 GB sitting next to them. -# One type per run. Sweep both: -# ./gitea-image-prune.sh --all --keep 7 --apply --yes --reclaim # containers -# ./gitea-image-prune.sh --type generic --all --keep 3 --apply --yes --reclaim +# One type per run. Sweep both (dry-run first; --apply is a person's act): +# ./gitea-image-prune.sh --all prune # containers, keep 20 + in use +# ./gitea-image-prune.sh --type generic --all prune # generic, keep 20 + in use # # HOW GITEA STORES GENERIC PACKAGES (contrast with containers, below) # No index, no manifest indirection, no shared layers: a version IS its files, @@ -83,15 +95,14 @@ # ./gitea-image-prune.sh --repo felhom-hub --keep 10 # dry-run prune (default) # ./gitea-image-prune.sh --repo felhom-hub --keep 10 --apply # really delete # ./gitea-image-prune.sh --repo felhom-controller --older-than 90 --apply --reclaim -# ./gitea-image-prune.sh --all --keep 15 --apply --yes --reclaim --measure # cron-friendly +# ./gitea-image-prune.sh --all prune # dry-run: keep 20 + in use # ./gitea-image-prune.sh --repo felhom-hub reclaim # run cleanup cron only # ./gitea-image-prune.sh --type generic --all list # the OTHER half # ./gitea-image-prune.sh --type generic --repo felhom-golden --keep 3 --apply --reclaim # -# SET-AND-FORGET COMPLEMENT: configure a native Gitea cleanup rule per owner -# (package settings -> Cleanup Rules: keep most-recent N, exclude ^latest$); -# the daily cleanup_packages cron then enforces it. See README. This script -# does NOT auto-create rules — it is the on-demand tool. +# NO SET-AND-FORGET: a native Gitea cleanup rule cannot know which versions are IN USE (the +# vouched golden, the floor, the vouched agent), so decision 62 rules it out — this on-demand +# script, run by a person, is the only pruner. (None exists today: package_cleanup_rule is empty.) # ============================================================================= set -euo pipefail @@ -125,6 +136,14 @@ NO_SIZES=false TOKEN_FILE="" LOG_FILE="" GITEA_TOKEN="${GITEA_TOKEN:-}" +DEFAULT_KEEP=20 # decision 62: the newest 20 of each package +FELHOM_EU="${FELHOM_EU:-/mnt/5_hdd/felhom.eu/git/felhom.eu}" # the vouch records' checkout +HUB_URL="${HUB_URL:-}" # the hub (default: its ClusterIP via kubectl) +declare -A VOUCH_PROTECT=() # ":" -> why it is in use +# Test seams (tests/test-prune-plan.sh): a versions fixture instead of the API, a protect +# list instead of the hub. Never set in normal use. +PRUNE_TEST_VERSIONS="${PRUNE_TEST_VERSIONS:-}" +PRUNE_TEST_PROTECT="${PRUNE_TEST_PROTECT:-}" # --- Temp workspace ------------------------------------------------------- TMP="$(mktemp -d)" @@ -222,6 +241,9 @@ discover_git_credential() { return 1 } +if [[ -n "$PRUNE_TEST_VERSIONS" ]]; then + GITEA_TOKEN="test-no-network"; CRED_SRC="test fixture (no network)" +fi if [[ -z "$GITEA_TOKEN" ]]; then discover_git_credential || true fi @@ -310,6 +332,11 @@ oci_get() { VERSIONS_JSON="$TMP/versions.json" # JSON-lines, one version object per line load_versions() { + if [[ -n "$PRUNE_TEST_VERSIONS" ]]; then + jq -c '.[]' "$PRUNE_TEST_VERSIONS" > "$VERSIONS_JSON" + info "Loaded $(grep -c . "$VERSIONS_JSON") version records from the test fixture." + return + fi step "Fetching ${PKG_TYPE} packages for owner '${OWNER}' from ${GITEA_URL} ..." : > "$VERSIONS_JSON" local page=1 limit=50 body n total=0 @@ -388,12 +415,84 @@ human() { # bytes -> human readable else echo "${b}B"; fi } -is_protected() { # -> 0 if protected - local tag="$1" rx +is_protected() { # -> 0 if protected (a --protect regex, or a version in use) + local name="$1" tag="$2" rx for rx in "${PROTECT[@]}"; do [[ "$tag" =~ $rx ]] && return 0; done + [[ -n "${VOUCH_PROTECT[${name}:${tag}]:-}" ]] && return 0 return 1 } +# protect_reason -> why it is kept ("" if not protected) +protect_reason() { + local name="$1" tag="$2" rx + for rx in "${PROTECT[@]}"; do [[ "$tag" =~ $rx ]] && { echo "matches --protect ${rx}"; return; }; done + echo "${VOUCH_PROTECT[${name}:${tag}]:-}" +} + +# ============================================================================= +# The versions IN USE (decision 62). Fills VOUCH_PROTECT or returns non-zero — the caller +# then refuses to prune. Every value is checked to look like a version before it counts. +# ============================================================================= +vouch_add() { # + VOUCH_PROTECT["$1:$2"]="${VOUCH_PROTECT["$1:$2"]:+${VOUCH_PROTECT["$1:$2"]}; }$3" +} +load_vouch_protect() { + if [[ -n "$PRUNE_TEST_PROTECT" ]]; then + local pkg tag why + while read -r pkg tag why; do [[ -n "$pkg" ]] && vouch_add "$pkg" "$tag" "$why"; done < "$PRUNE_TEST_PROTECT" + info "In-use list from the test fixture: ${#VOUCH_PROTECT[@]} version(s)." + return 0 + fi + local hubpw="${HUB_PW:-}" page floor golden agent minagent bake hubtag + if [[ -z "$hubpw" ]]; then + local cred="${FELHOM_EU}/scripts/read_credential.py" f + f="$(mktemp)" + if [[ -r "$cred" ]] && python3 "$cred" HUB_PW "$f" >/dev/null 2>&1; then hubpw="$(cat "$f")"; fi + rm -f "$f" + fi + [[ -n "$hubpw" ]] || { error "In-use list: no HUB_PW (env or ~/.config/credentials) — cannot read the vouch."; return 1; } + if [[ -z "$HUB_URL" ]]; then + local ip; ip="$(sudo -n kubectl -n felhom-system get svc hub -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)" + [[ -n "$ip" ]] && HUB_URL="http://${ip}:8080" + fi + [[ -n "$HUB_URL" ]] || { error "In-use list: the hub's address is unknown (set HUB_URL)."; return 1; } + page="$(mktemp)" + # Basic auth through -u; NEVER -w '%{redirect_url}' (it prints the password, R-580). + curl -fsS --max-time 20 -u ":${hubpw}" -o "$page" "${HUB_URL}/configuration" 2>/dev/null \ + || { rm -f "$page"; error "In-use list: GET ${HUB_URL}/configuration failed."; return 1; } + hubpw="" + floor="$(grep -o 'name="min_controller_version" value="[^"]*"' "$page" | head -1 | sed 's/.*value="//; s/"$//')" + golden="$(tr '\n' ' ' < "$page" | grep -o '