gitea-image-prune.sh: keep the newest 20 and every version in use; refuse when the in-use list is unreadable (Felhom 09 decision 62, R-750)
Protected whatever --keep says: the controller floor, the vouched golden, the vouched agent and min_agent (the hub's Configuration page), every image of ours the vouched golden baked (its bake.log), the hub manifest's image. The dry-run prints each kept version and why. tests/test-prune-plan.sh pins it without network (red-proofed). No --apply was run. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
+128
-19
@@ -15,6 +15,18 @@
|
||||
# so felhom-controller (~96 tags) / felhom-hub (~42 tags) accumulate one image
|
||||
# per build and the Gitea Longhorn PVC keeps filling.
|
||||
#
|
||||
# RETENTION RULE — Felhom `09-update-architecture.md` §3 decision 62 (operator ruling 2026-10-01, R-750)
|
||||
# A prune keeps the newest 20 versions of each package (--keep defaults to 20 when
|
||||
# pruning) PLUS every version in use, whatever --keep or --older-than says:
|
||||
# - the vouched golden, the controller floor, the vouched agent and its min_agent
|
||||
# (read from the hub's operator Configuration page — Basic auth, HUB_PW);
|
||||
# - every gitea.dooplex.hu/admin/<pkg>:<tag> the vouched golden BAKED (its bake.log
|
||||
# in felhom.eu/documentation/tests/golden-<version>-*/);
|
||||
# - the hub image the GitOps manifest runs (felhom.eu/manifests/hub.yaml).
|
||||
# If any of those cannot be read, a prune REFUSES (exit 3) — never "protect nothing".
|
||||
# --apply is a person's act: nothing schedules this script (no cron, no timer).
|
||||
# The August 2026 run (`--all --keep 7 --apply`, 2026-08-22, HM-024) predates this rule.
|
||||
#
|
||||
# PACKAGE TYPES — --type, default "container" (added 2026-08-23)
|
||||
# Gitea namespaces packages by TYPE, and every API path embeds it. This script
|
||||
# handled only type=container, so --all meant "all *container* packages" and
|
||||
@@ -22,9 +34,9 @@
|
||||
# admin/felhom-golden (22 versions x ~627 MB = 13 GB) grew to 88% of the Gitea
|
||||
# PVC while --all --keep 7 --apply reported success: it pruned ~1.3 GB of
|
||||
# container images and never saw the 13 GB sitting next to them.
|
||||
# One type per run. Sweep both:
|
||||
# ./gitea-image-prune.sh --all --keep 7 --apply --yes --reclaim # containers
|
||||
# ./gitea-image-prune.sh --type generic --all --keep 3 --apply --yes --reclaim
|
||||
# One type per run. Sweep both (dry-run first; --apply is a person's act):
|
||||
# ./gitea-image-prune.sh --all prune # containers, keep 20 + in use
|
||||
# ./gitea-image-prune.sh --type generic --all prune # generic, keep 20 + in use
|
||||
#
|
||||
# HOW GITEA STORES GENERIC PACKAGES (contrast with containers, below)
|
||||
# No index, no manifest indirection, no shared layers: a version IS its files,
|
||||
@@ -83,15 +95,14 @@
|
||||
# ./gitea-image-prune.sh --repo felhom-hub --keep 10 # dry-run prune (default)
|
||||
# ./gitea-image-prune.sh --repo felhom-hub --keep 10 --apply # really delete
|
||||
# ./gitea-image-prune.sh --repo felhom-controller --older-than 90 --apply --reclaim
|
||||
# ./gitea-image-prune.sh --all --keep 15 --apply --yes --reclaim --measure # cron-friendly
|
||||
# ./gitea-image-prune.sh --all prune # dry-run: keep 20 + in use
|
||||
# ./gitea-image-prune.sh --repo felhom-hub reclaim # run cleanup cron only
|
||||
# ./gitea-image-prune.sh --type generic --all list # the OTHER half
|
||||
# ./gitea-image-prune.sh --type generic --repo felhom-golden --keep 3 --apply --reclaim
|
||||
#
|
||||
# SET-AND-FORGET COMPLEMENT: configure a native Gitea cleanup rule per owner
|
||||
# (package settings -> Cleanup Rules: keep most-recent N, exclude ^latest$);
|
||||
# the daily cleanup_packages cron then enforces it. See README. This script
|
||||
# does NOT auto-create rules — it is the on-demand tool.
|
||||
# NO SET-AND-FORGET: a native Gitea cleanup rule cannot know which versions are IN USE (the
|
||||
# vouched golden, the floor, the vouched agent), so decision 62 rules it out — this on-demand
|
||||
# script, run by a person, is the only pruner. (None exists today: package_cleanup_rule is empty.)
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
@@ -125,6 +136,14 @@ NO_SIZES=false
|
||||
TOKEN_FILE=""
|
||||
LOG_FILE=""
|
||||
GITEA_TOKEN="${GITEA_TOKEN:-}"
|
||||
DEFAULT_KEEP=20 # decision 62: the newest 20 of each package
|
||||
FELHOM_EU="${FELHOM_EU:-/mnt/5_hdd/felhom.eu/git/felhom.eu}" # the vouch records' checkout
|
||||
HUB_URL="${HUB_URL:-}" # the hub (default: its ClusterIP via kubectl)
|
||||
declare -A VOUCH_PROTECT=() # "<package>:<tag>" -> why it is in use
|
||||
# Test seams (tests/test-prune-plan.sh): a versions fixture instead of the API, a protect
|
||||
# list instead of the hub. Never set in normal use.
|
||||
PRUNE_TEST_VERSIONS="${PRUNE_TEST_VERSIONS:-}"
|
||||
PRUNE_TEST_PROTECT="${PRUNE_TEST_PROTECT:-}"
|
||||
|
||||
# --- Temp workspace -------------------------------------------------------
|
||||
TMP="$(mktemp -d)"
|
||||
@@ -222,6 +241,9 @@ discover_git_credential() {
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ -n "$PRUNE_TEST_VERSIONS" ]]; then
|
||||
GITEA_TOKEN="test-no-network"; CRED_SRC="test fixture (no network)"
|
||||
fi
|
||||
if [[ -z "$GITEA_TOKEN" ]]; then
|
||||
discover_git_credential || true
|
||||
fi
|
||||
@@ -310,6 +332,11 @@ oci_get() {
|
||||
VERSIONS_JSON="$TMP/versions.json" # JSON-lines, one version object per line
|
||||
|
||||
load_versions() {
|
||||
if [[ -n "$PRUNE_TEST_VERSIONS" ]]; then
|
||||
jq -c '.[]' "$PRUNE_TEST_VERSIONS" > "$VERSIONS_JSON"
|
||||
info "Loaded $(grep -c . "$VERSIONS_JSON") version records from the test fixture."
|
||||
return
|
||||
fi
|
||||
step "Fetching ${PKG_TYPE} packages for owner '${OWNER}' from ${GITEA_URL} ..."
|
||||
: > "$VERSIONS_JSON"
|
||||
local page=1 limit=50 body n total=0
|
||||
@@ -388,12 +415,84 @@ human() { # bytes -> human readable
|
||||
else echo "${b}B"; fi
|
||||
}
|
||||
|
||||
is_protected() { # <tag> -> 0 if protected
|
||||
local tag="$1" rx
|
||||
is_protected() { # <package> <tag> -> 0 if protected (a --protect regex, or a version in use)
|
||||
local name="$1" tag="$2" rx
|
||||
for rx in "${PROTECT[@]}"; do [[ "$tag" =~ $rx ]] && return 0; done
|
||||
[[ -n "${VOUCH_PROTECT[${name}:${tag}]:-}" ]] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
# protect_reason <package> <tag> -> why it is kept ("" if not protected)
|
||||
protect_reason() {
|
||||
local name="$1" tag="$2" rx
|
||||
for rx in "${PROTECT[@]}"; do [[ "$tag" =~ $rx ]] && { echo "matches --protect ${rx}"; return; }; done
|
||||
echo "${VOUCH_PROTECT[${name}:${tag}]:-}"
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# The versions IN USE (decision 62). Fills VOUCH_PROTECT or returns non-zero — the caller
|
||||
# then refuses to prune. Every value is checked to look like a version before it counts.
|
||||
# =============================================================================
|
||||
vouch_add() { # <package> <tag> <why>
|
||||
VOUCH_PROTECT["$1:$2"]="${VOUCH_PROTECT["$1:$2"]:+${VOUCH_PROTECT["$1:$2"]}; }$3"
|
||||
}
|
||||
load_vouch_protect() {
|
||||
if [[ -n "$PRUNE_TEST_PROTECT" ]]; then
|
||||
local pkg tag why
|
||||
while read -r pkg tag why; do [[ -n "$pkg" ]] && vouch_add "$pkg" "$tag" "$why"; done < "$PRUNE_TEST_PROTECT"
|
||||
info "In-use list from the test fixture: ${#VOUCH_PROTECT[@]} version(s)."
|
||||
return 0
|
||||
fi
|
||||
local hubpw="${HUB_PW:-}" page floor golden agent minagent bake hubtag
|
||||
if [[ -z "$hubpw" ]]; then
|
||||
local cred="${FELHOM_EU}/scripts/read_credential.py" f
|
||||
f="$(mktemp)"
|
||||
if [[ -r "$cred" ]] && python3 "$cred" HUB_PW "$f" >/dev/null 2>&1; then hubpw="$(cat "$f")"; fi
|
||||
rm -f "$f"
|
||||
fi
|
||||
[[ -n "$hubpw" ]] || { error "In-use list: no HUB_PW (env or ~/.config/credentials) — cannot read the vouch."; return 1; }
|
||||
if [[ -z "$HUB_URL" ]]; then
|
||||
local ip; ip="$(sudo -n kubectl -n felhom-system get svc hub -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)"
|
||||
[[ -n "$ip" ]] && HUB_URL="http://${ip}:8080"
|
||||
fi
|
||||
[[ -n "$HUB_URL" ]] || { error "In-use list: the hub's address is unknown (set HUB_URL)."; return 1; }
|
||||
page="$(mktemp)"
|
||||
# Basic auth through -u; NEVER -w '%{redirect_url}' (it prints the password, R-580).
|
||||
curl -fsS --max-time 20 -u ":${hubpw}" -o "$page" "${HUB_URL}/configuration" 2>/dev/null \
|
||||
|| { rm -f "$page"; error "In-use list: GET ${HUB_URL}/configuration failed."; return 1; }
|
||||
hubpw=""
|
||||
floor="$(grep -o 'name="min_controller_version" value="[^"]*"' "$page" | head -1 | sed 's/.*value="//; s/"$//')"
|
||||
golden="$(tr '\n' ' ' < "$page" | grep -o '<select name="golden_version".*</select>' | grep -o '<option value="[^"]*"[^>]*selected' | head -1 | sed 's/<option value="//; s/".*//')"
|
||||
agent="$(tr '\n' ' ' < "$page" | grep -o '<select name="agent_version".*' | grep -o '<option value="[^"]*"[^>]*selected' | head -1 | sed 's/<option value="//; s/".*//')"
|
||||
minagent="$(grep -o 'name="min_agent" value="[^"]*"' "$page" | sed 's/.*value="//; s/"$//' | sort -u)"
|
||||
rm -f "$page"
|
||||
local vre='^[0-9]+\.[0-9]+\.[0-9]+$' v
|
||||
for v in "$floor" "$golden" "$agent"; do
|
||||
[[ "$v" =~ $vre ]] || { error "In-use list: the hub page did not give a version (floor='${floor}' golden='${golden}' agent='${agent}')."; return 1; }
|
||||
done
|
||||
vouch_add felhom-controller "$floor" "the controller floor"
|
||||
vouch_add felhom-controller "$golden" "the vouched golden's controller"
|
||||
vouch_add felhom-golden "$golden" "the vouched golden"
|
||||
vouch_add felhom-agent "$agent" "the vouched agent"
|
||||
for v in $minagent; do
|
||||
[[ "$v" =~ $vre ]] || { error "In-use list: min_agent '${v}' is not a version."; return 1; }
|
||||
vouch_add felhom-agent "$v" "min_agent"
|
||||
done
|
||||
bake="$(ls -d "${FELHOM_EU}"/documentation/tests/golden-"${golden}"-*/bake.log 2>/dev/null | head -1)"
|
||||
[[ -r "$bake" ]] || { error "In-use list: no bake.log for golden ${golden} under ${FELHOM_EU}/documentation/tests/."; return 1; }
|
||||
local ref pkg tag n=0
|
||||
while read -r ref; do
|
||||
pkg="${ref#gitea.dooplex.hu/${OWNER}/}"; tag="${pkg##*:}"; pkg="${pkg%%:*}"
|
||||
vouch_add "$pkg" "$tag" "baked into golden ${golden}"; n=$((n + 1))
|
||||
done < <(grep -o "gitea\.dooplex\.hu/${OWNER}/[a-z0-9-]*:[A-Za-z0-9._-]*" "$bake" | sort -u)
|
||||
[[ "$n" -gt 0 ]] || { error "In-use list: golden ${golden}'s bake.log names no image of ours — refusing."; return 1; }
|
||||
hubtag="$(grep -o "gitea\.dooplex\.hu/${OWNER}/felhom-hub:[A-Za-z0-9._-]*" "${FELHOM_EU}/manifests/hub.yaml" 2>/dev/null | head -1)"
|
||||
[[ -n "$hubtag" ]] || { error "In-use list: no hub image in ${FELHOM_EU}/manifests/hub.yaml."; return 1; }
|
||||
vouch_add felhom-hub "${hubtag##*:}" "the hub the GitOps manifest runs"
|
||||
info "In-use list (decision 62): ${#VOUCH_PROTECT[@]} version(s) from the hub's vouch, golden ${golden}'s bake.log and the hub manifest."
|
||||
return 0
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# Orphan-manifest detection (drives reclaim).
|
||||
# A "sha256:" manifest version is an ORPHAN if no SURVIVING tag's index
|
||||
@@ -491,7 +590,7 @@ do_list() {
|
||||
count=$((count + 1))
|
||||
if $NO_SIZES; then bytes=0; else bytes="$(resolve_tag_bytes "$name" "$tag")"; fi
|
||||
total_bytes=$((total_bytes + bytes))
|
||||
prot=""; is_protected "$tag" && prot="PROTECTED"
|
||||
prot=""; is_protected "$name" "$tag" && prot="PROTECTED ($(protect_reason "$name" "$tag"))"
|
||||
printf ' %-28s %-22s %-12s %s\n' "$tag" "${created:0:19}" "$( $NO_SIZES && echo '-' || human "$bytes")" "$prot"
|
||||
done < <(tagged_versions "$name")
|
||||
echo " ----"
|
||||
@@ -511,18 +610,20 @@ do_list() {
|
||||
|
||||
# ---- Compute prune plan: prints "DELETE\t<tag>\t<created>" / "KEEP..." ----
|
||||
# Sets globals: PLAN_DELETE (array of tags), PLAN_KEEP_N, PLAN_PROT_N
|
||||
declare -a PLAN_DELETE=()
|
||||
declare -a PLAN_DELETE=() PLAN_PROTECTED=()
|
||||
PLAN_KEEP_N=0; PLAN_PROT_N=0; PLAN_TOTAL=0
|
||||
compute_plan() {
|
||||
local name="$1" tag created epoch now cutoff idx=0
|
||||
PLAN_DELETE=(); PLAN_KEEP_N=0; PLAN_PROT_N=0; PLAN_TOTAL=0
|
||||
PLAN_DELETE=(); PLAN_KEEP_N=0; PLAN_PROT_N=0; PLAN_TOTAL=0; PLAN_PROTECTED=()
|
||||
now="$(date +%s)"
|
||||
[[ -n "$OLDER_THAN" ]] && cutoff=$(( now - OLDER_THAN * 86400 ))
|
||||
while IFS=$'\t' read -r tag created; do
|
||||
[[ -z "$tag" ]] && continue
|
||||
PLAN_TOTAL=$((PLAN_TOTAL + 1))
|
||||
# Protected always wins.
|
||||
if is_protected "$tag"; then PLAN_PROT_N=$((PLAN_PROT_N + 1)); continue; fi
|
||||
if is_protected "$name" "$tag"; then
|
||||
PLAN_PROT_N=$((PLAN_PROT_N + 1)); PLAN_PROTECTED+=("${tag} — $(protect_reason "$name" "$tag")"); continue
|
||||
fi
|
||||
if [[ -n "$KEEP" ]]; then
|
||||
# tags arrive newest-first; keep the first KEEP non-protected... but
|
||||
# protected tags don't consume a keep slot — count index over all tags.
|
||||
@@ -544,6 +645,8 @@ do_prune_repo() {
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}== prune ${name} ==${NC} mode: $( [[ -n "$KEEP" ]] && echo "keep-last ${KEEP}" || echo "older-than ${OLDER_THAN}d" )"
|
||||
local p
|
||||
for p in "${PLAN_PROTECTED[@]}"; do printf ' PROTECTED %s\n' "$p"; done
|
||||
if [[ "${#PLAN_DELETE[@]}" -eq 0 ]]; then
|
||||
info " Nothing to prune (${PLAN_TOTAL} tags: ${PLAN_KEEP_N} kept, ${PLAN_PROT_N} protected)."
|
||||
return
|
||||
@@ -564,7 +667,7 @@ do_prune_repo() {
|
||||
fi
|
||||
|
||||
if ! $APPLY; then
|
||||
warn " DRY-RUN — nothing deleted. Re-run with --apply to delete."
|
||||
warn " DRY-RUN — nothing deleted. --apply is a person's act (decision 62); nothing schedules it."
|
||||
for tag in "${PLAN_DELETE[@]}"; do audit "DRY-RUN would-delete ${OWNER}/${name}:${tag}"; done
|
||||
return
|
||||
fi
|
||||
@@ -727,7 +830,7 @@ interactive_menu() {
|
||||
case "$act" in
|
||||
1) ACTION="list" ;;
|
||||
2) ACTION="prune"
|
||||
read -r -p "Keep how many most-recent tags? [10]: " KEEP; KEEP="${KEEP:-10}"
|
||||
read -r -p "Keep how many most-recent tags? [${DEFAULT_KEEP}]: " KEEP; KEEP="${KEEP:-$DEFAULT_KEEP}"
|
||||
[[ "$KEEP" =~ ^[0-9]+$ ]] || { error "invalid number"; exit 2; }
|
||||
read -r -p "Apply for real now? (dry-run otherwise) [y/N]: " ap
|
||||
[[ "$ap" =~ ^[Yy]$ ]] && APPLY=true
|
||||
@@ -750,7 +853,8 @@ info "Server: ${GITEA_URL} Owner: ${OWNER} Type: ${PKG_TYPE} Log: ${LOG_FI
|
||||
info "Auth: ${CRED_SRC}$( [[ -n "$GITEA_USER" ]] && echo " (user: ${GITEA_USER})")"
|
||||
|
||||
# Sanity: Gitea version (public, no auth)
|
||||
GV="$(curl -fsS "${GITEA_URL}/api/v1/version" 2>/dev/null | jq -r '.version' 2>/dev/null || echo '?')"
|
||||
if [[ -n "$PRUNE_TEST_VERSIONS" ]]; then GV="1.26.test"; else
|
||||
GV="$(curl -fsS "${GITEA_URL}/api/v1/version" 2>/dev/null | jq -r '.version' 2>/dev/null || echo '?')"; fi
|
||||
info "Gitea version: ${GV}"
|
||||
[[ "$GV" == 1.26.* ]] || warn "Tested against Gitea 1.26.2 — server reports '${GV}'. Verify API shapes."
|
||||
|
||||
@@ -788,9 +892,14 @@ fi
|
||||
if [[ -z "$ACTION" ]]; then
|
||||
if [[ -n "$KEEP" || -n "$OLDER_THAN" ]]; then ACTION="prune"; else ACTION="list"; fi
|
||||
fi
|
||||
# prune needs a mode
|
||||
# prune: --keep defaults to 20 (decision 62)
|
||||
if [[ "$ACTION" == "prune" && -z "$KEEP" && -z "$OLDER_THAN" ]]; then
|
||||
error "prune needs --keep N or --older-than DAYS."; exit 2
|
||||
KEEP="$DEFAULT_KEEP"; info "prune: --keep defaults to ${DEFAULT_KEEP} (decision 62)"
|
||||
fi
|
||||
# prune: the versions in use are read FIRST, or nothing is pruned (dry-run included — its plan would lie)
|
||||
if [[ "$ACTION" == "prune" ]]; then
|
||||
load_vouch_protect || { error "REFUSING to prune: the versions in use could not be read (decision 62 — never 'protect nothing')."; audit "REFUSED prune: in-use list unreadable"; exit 3; }
|
||||
for k in $(printf '%s\n' "${!VOUCH_PROTECT[@]}" | sort); do note " in use: ${k} — ${VOUCH_PROTECT[$k]}"; done
|
||||
fi
|
||||
|
||||
info "Action: ${ACTION} Type: ${PKG_TYPE} Targets: ${TARGETS[*]}"
|
||||
|
||||
Reference in New Issue
Block a user