#!/usr/bin/env bash # ============================================================================= # gitea-image-prune.sh — inspect & prune old container images and generic # artifacts in the self-hosted Gitea registry, then # reclaim disk. # ============================================================================= # Target server : gitea.dooplex.hu (Gitea 1.26.2, owner namespace "admin") # Best run on : build server 192.168.0.180 (has kubectl for --measure), # but the core (list/prune/reclaim) is pure curl + jq and runs # from any host that can reach Gitea and holds an admin token. # # ----------------------------------------------------------------------------- # WHY THIS EXISTS # build-felhom-{hub,controller}.sh push : AND :latest on every build, # so felhom-controller (~96 tags) / felhom-hub (~42 tags) accumulate one image # per build and the Gitea Longhorn PVC keeps filling. # # RETENTION RULE — Felhom `09-update-architecture.md` §3 decision 62 (operator ruling 2026-10-01, R-750) # A prune keeps the newest 20 versions of each package (--keep defaults to 20 when # pruning) PLUS every version in use, whatever --keep or --older-than says: # - the vouched golden, the controller floor, the vouched agent and its min_agent # (read from the hub's operator Configuration page — Basic auth, HUB_PW); # - every gitea.dooplex.hu/admin/: the vouched golden BAKED (its bake.log # in felhom.eu/documentation/tests/golden--*/); # - the hub image the GitOps manifest runs (felhom.eu/manifests/hub.yaml). # If any of those cannot be read, a prune REFUSES (exit 3) — never "protect nothing". # --apply is a person's act: nothing schedules this script (no cron, no timer). # The August 2026 run (`--all --keep 7 --apply`, 2026-08-22, HM-024) predates this rule. # # PACKAGE TYPES — --type, default "container" (added 2026-08-23) # Gitea namespaces packages by TYPE, and every API path embeds it. This script # handled only type=container, so --all meant "all *container* packages" and # type=generic artifacts were invisible to every mode. That is how # admin/felhom-golden (22 versions x ~627 MB = 13 GB) grew to 88% of the Gitea # PVC while --all --keep 7 --apply reported success: it pruned ~1.3 GB of # container images and never saw the 13 GB sitting next to them. # One type per run. Sweep both (dry-run first; --apply is a person's act): # ./gitea-image-prune.sh --all prune # containers, keep 20 + in use # ./gitea-image-prune.sh --type generic --all prune # generic, keep 20 + in use # # HOW GITEA STORES GENERIC PACKAGES (contrast with containers, below) # No index, no manifest indirection, no shared layers: a version IS its files, # and its blobs are unique to it. So reclaim has only TWO steps — delete the # version, then let cleanup_packages GC the blobs. There are never orphaned # manifests to hunt, and "apparent" size equals real reclaimed size (the # shared-layer caveat that applies to containers does not apply here). # # HOW GITEA STORES CONTAINER IMAGES (load-bearing — drives the reclaim design) # A pushed tag is an OCI image *index* (a ~850 B pointer). The real bytes live # in untagged "sha256:..." manifest *versions* (config + layer blobs, ~9-25 MB # each), whose layer blobs are content-addressed and SHARED across tags. # Deleting a tag removes only the tiny index pointer — the manifest versions it # referenced linger as untagged versions, and their blobs stay referenced. # # PROVEN RECLAIM MECHANISM (verified live on this instance, 2026-06-17 — see §3 # of the task / REPORT.md). It is THREE steps, not two: # 1. DELETE the tag(s) -> frees ~nothing (only the index ptr) # 2. DELETE the now-ORPHANED "sha256:" manifest versions (referenced by no # surviving tag) -> still frees nothing on its own, BUT # makes their unique blobs unreferenced. (Default cleanup_packages does # NOT remove untagged manifests — only a cleanup *rule* would — so the # script must delete the orphaned manifests itself. write:package scope.) # 3. cleanup_packages cron runs -> GCs unreferenced blobs created # >OLDER_THAN (24h default) ago -> THIS frees disk. Shared base layers # still referenced by surviving tags are correctly retained. # Live proof: deleting one 9.4 MB-apparent tag + its 2 manifests + GC freed # 5.1 MiB (the rest was shared base layers, correctly kept). # => "prune" deletes tags; "reclaim" deletes the orphaned manifests and then # triggers (or defers to) the cleanup_packages cron. # # TRIGGERING THE GC CRON: POST /api/v1/admin/cron/cleanup_packages needs # write:admin. If the token lacks it, the orphaned manifests are still deleted # and their blobs are freed by the daily "@midnight" run (or on the next Gitea # restart — RUN_AT_START was enabled in app.ini on 2026-06-17). # # CREDENTIALS (in order: GITEA_TOKEN env -> --token-file -> git's stored # credential for the Gitea host: the remote URL's embedded token, else a # configured credential helper). Auto-discovery lets you just run the script # inside a clone with no token fuss — but a *git* credential may only have repo # scope; if so, a 403 will name the missing package/admin scope. # # REQUIRED TOKEN (env GITEA_TOKEN, or --token-file). Must belong to a Gitea # site-admin user. Minimal fine-grained scopes (Gitea 1.26): # read:package — list packages / versions / files (list, prune planning) # write:package — delete a version (prune --apply) # read:admin — list cron tasks # write:admin — run the cleanup_packages cron (reclaim) # The project read-only token is insufficient (no package scope). # The token is NEVER echoed, logged, or committed. # # USAGE # GITEA_TOKEN=... ./gitea-image-prune.sh # interactive menu # ./gitea-image-prune.sh --repo felhom-hub list # list one package # ./gitea-image-prune.sh --all list # list all packages # ./gitea-image-prune.sh --repo felhom-hub --keep 10 # dry-run prune (default) # ./gitea-image-prune.sh --repo felhom-hub --keep 10 --apply # really delete # ./gitea-image-prune.sh --repo felhom-controller --older-than 90 --apply --reclaim # ./gitea-image-prune.sh --all prune # dry-run: keep 20 + in use # ./gitea-image-prune.sh --repo felhom-hub reclaim # run cleanup cron only # ./gitea-image-prune.sh --type generic --all list # the OTHER half # ./gitea-image-prune.sh --type generic --repo felhom-golden --keep 3 --apply --reclaim # # NO SET-AND-FORGET: a native Gitea cleanup rule cannot know which versions are IN USE (the # vouched golden, the floor, the vouched agent), so decision 62 rules it out — this on-demand # script, run by a person, is the only pruner. (None exists today: package_cleanup_rule is empty.) # ============================================================================= set -euo pipefail # --- Configuration -------------------------------------------------------- GITEA_URL="${GITEA_URL:-https://gitea.dooplex.hu}" OWNER="${GITEA_OWNER:-admin}" CLEANUP_CRON="cleanup_packages" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" # --- Colors / log helpers (match build-felhom-hub.sh) --------------------- RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; BOLD='\033[1m'; NC='\033[0m' info() { echo -e "${GREEN}[INFO]${NC} $*"; } warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; } error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } step() { echo -e "${CYAN}[STEP]${NC} $*"; } note() { echo -e "${CYAN} $*${NC}"; } # --- Defaults / arg state ------------------------------------------------- declare -a REPOS=() declare -a PROTECT=('^latest$') # always-protected tags; --protect appends ALL=false PKG_TYPE="container" # Gitea package type; --type. Every API path embeds it. ACTION="" # list | prune | reclaim (positional or inferred) KEEP="" OLDER_THAN="" APPLY=false # dry-run is the default ASSUME_YES=false DO_RECLAIM=false # run cleanup cron after an --apply prune MEASURE=false NO_SIZES=false TOKEN_FILE="" LOG_FILE="" GITEA_TOKEN="${GITEA_TOKEN:-}" DEFAULT_KEEP=20 # decision 62: the newest 20 of each package FELHOM_EU="${FELHOM_EU:-/mnt/5_hdd/felhom.eu/git/felhom.eu}" # the vouch records' checkout HUB_URL="${HUB_URL:-}" # the hub (default: its ClusterIP via kubectl) declare -A VOUCH_PROTECT=() # ":" -> why it is in use # Test seams (tests/test-prune-plan.sh): a versions fixture instead of the API, a protect # list instead of the hub. Never set in normal use. PRUNE_TEST_VERSIONS="${PRUNE_TEST_VERSIONS:-}" PRUNE_TEST_PROTECT="${PRUNE_TEST_PROTECT:-}" # --- Temp workspace ------------------------------------------------------- TMP="$(mktemp -d)" cleanup() { rm -rf "$TMP"; } trap cleanup EXIT usage() { sed -n '2,/^set -euo/p' "$0" | sed 's/^# \{0,1\}//; s/^#//' | sed '$d' exit "${1:-0}" } # ============================================================================= # Argument parsing # ============================================================================= while [[ $# -gt 0 ]]; do case "$1" in list|prune|reclaim) ACTION="$1"; shift ;; --repo) REPOS+=("$2"); shift 2 ;; --all) ALL=true; shift ;; --type) PKG_TYPE="$2"; shift 2 ;; --keep) KEEP="$2"; shift 2 ;; --older-than) OLDER_THAN="$2"; shift 2 ;; --dry-run) APPLY=false; shift ;; --apply) APPLY=true; shift ;; --yes|-y) ASSUME_YES=true; shift ;; --reclaim) DO_RECLAIM=true; shift ;; --measure) MEASURE=true; shift ;; --no-sizes) NO_SIZES=true; shift ;; --protect) PROTECT+=("$2"); shift 2 ;; --token-file) TOKEN_FILE="$2"; shift 2 ;; --owner) OWNER="$2"; shift 2 ;; --log) LOG_FILE="$2"; shift 2 ;; -h|--help) usage 0 ;; *) error "Unknown argument: $1"; usage 1 ;; esac done # --- Validate flag combinations ------------------------------------------- if [[ -n "$KEEP" && -n "$OLDER_THAN" ]]; then error "--keep and --older-than are mutually exclusive (pick one prune mode)." exit 2 fi if [[ -n "$KEEP" && ! "$KEEP" =~ ^[0-9]+$ ]]; then error "--keep must be a non-negative integer"; exit 2; fi if [[ -n "$OLDER_THAN" && ! "$OLDER_THAN" =~ ^[0-9]+$ ]]; then error "--older-than must be a non-negative integer (days)"; exit 2; fi if [[ ! "$PKG_TYPE" =~ ^[a-z_]+$ ]]; then error "--type must be a Gitea package type slug (e.g. container, generic)"; exit 2; fi # Only container carries the OCI index -> manifest -> blob indirection. Every # other Gitea type stores a version's files directly, so it takes the simple # path: delete the version, GC the blobs. Both are implemented; anything beyond # container/generic is untested here but structurally identical to generic. IS_CONTAINER=false [[ "$PKG_TYPE" == "container" ]] && IS_CONTAINER=true case "$PKG_TYPE" in container|generic) ;; *) warn "--type '${PKG_TYPE}' is untested; treating it like 'generic' (no manifest indirection)." ;; esac # ============================================================================= # Preflight # ============================================================================= for bin in curl jq; do command -v "$bin" &>/dev/null || { error "Required tool not found: $bin"; exit 1; } done # Token: --token-file beats env. Never printed. GITEA_USER="${GITEA_USER:-}" # set when credentials come with a username (-> Basic auth) CRED_SRC="GITEA_TOKEN env" if [[ -n "$TOKEN_FILE" ]]; then [[ -r "$TOKEN_FILE" ]] || { error "--token-file not readable: $TOKEN_FILE"; exit 1; } GITEA_TOKEN="$(tr -d ' \t\r\n' < "$TOKEN_FILE")" CRED_SRC="--token-file" fi # Auto-discover from git when no explicit token: reuse the credential git already # has for the Gitea host (embedded remote URL, or a configured credential helper). # Convenient, but a *git* token may lack package/admin scopes — a 403 will say so. discover_git_credential() { command -v git &>/dev/null || return 1 git rev-parse --is-inside-work-tree &>/dev/null || return 1 local host="${GITEA_URL#*://}"; host="${host%%/*}" # (1) credentials embedded in the remote URL (https://user:token@host/...) local url; url="$(git remote get-url origin 2>/dev/null || true)" if [[ "$url" == *"$host"* && "$url" =~ ^https?://([^:/@]+):([^@/]+)@ ]]; then GITEA_USER="${BASH_REMATCH[1]}"; GITEA_TOKEN="${BASH_REMATCH[2]}"; CRED_SRC="git remote URL"; return 0 fi # (2) a configured credential helper (store / cache / manager). Never prompt. if git config --get credential.helper &>/dev/null; then local out user pass out="$(printf 'protocol=https\nhost=%s\n\n' "$host" | GIT_TERMINAL_PROMPT=0 git credential fill 2>/dev/null || true)" pass="$(printf '%s\n' "$out" | sed -n 's/^password=//p' | head -1)" user="$(printf '%s\n' "$out" | sed -n 's/^username=//p' | head -1)" if [[ -n "$pass" ]]; then GITEA_TOKEN="$pass"; GITEA_USER="${user:-$OWNER}"; CRED_SRC="git credential helper"; return 0; fi fi return 1 } if [[ -n "$PRUNE_TEST_VERSIONS" ]]; then GITEA_TOKEN="test-no-network"; CRED_SRC="test fixture (no network)" fi if [[ -z "$GITEA_TOKEN" ]]; then discover_git_credential || true fi if [[ -z "$GITEA_TOKEN" ]]; then error "No credentials. Set GITEA_TOKEN env, pass --token-file , or run inside" error "a clone of a ${GITEA_URL#*://} repo whose git credentials are configured." error "Needs a site-admin token with read/write:package + read/write:admin scope." exit 1 fi # Auth method: Basic (user:token) when a username is known — works for both API # tokens and passwords; bare 'token' header otherwise. OCI /v2/ always uses Basic. declare -a AUTH if [[ -n "$GITEA_USER" ]]; then AUTH=(-u "${GITEA_USER}:${GITEA_TOKEN}"); else AUTH=(-H "Authorization: token ${GITEA_TOKEN}"); fi OCI_USER="${GITEA_USER:-$OWNER}" # Default audit log if [[ -z "$LOG_FILE" ]]; then LOG_FILE="${SCRIPT_DIR}/logs/gitea-prune-$(date +%Y-%m-%d).log" fi mkdir -p "$(dirname "$LOG_FILE")" # Redact the token from anything we print (belt-and-suspenders vs set -x etc.) redact() { sed "s|${GITEA_TOKEN}|***TOKEN***|g"; } audit() { # free-form line -> audit log (token-free by construction) printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >> "$LOG_FILE" } # ============================================================================= # HTTP helpers — every call checks status; non-2xx is never silent. # ============================================================================= # api_get -> body on stdout; dies on non-2xx # api_delete -> echoes HTTP code; returns non-zero on non-204 # api_post -> echoes HTTP code; dies on non-2xx # oci_get -> OCI manifest JSON (Basic auth); empty + rc1 on fail api_get() { local path="$1" out code out="$(mktemp)" code="$(curl -sS "${AUTH[@]}" \ -o "$out" -w '%{http_code}' "${GITEA_URL}${path}" || true)" if [[ ! "$code" =~ ^2[0-9][0-9]$ ]]; then error "GET ${path} -> HTTP ${code}" head -c 400 "$out" | redact >&2; echo >&2 rm -f "$out"; return 1 fi cat "$out"; rm -f "$out" } api_delete() { local path="$1" code code="$(curl -sS -X DELETE "${AUTH[@]}" \ -o "$TMP/del.body" -w '%{http_code}' "${GITEA_URL}${path}" || true)" echo "$code" [[ "$code" == "204" ]] } api_post() { local path="$1" code code="$(curl -sS -X POST "${AUTH[@]}" \ -o "$TMP/post.body" -w '%{http_code}' "${GITEA_URL}${path}" || true)" if [[ ! "$code" =~ ^2[0-9][0-9]$ ]]; then error "POST ${path} -> HTTP ${code}" head -c 400 "$TMP/post.body" | redact >&2; echo >&2 return 1 fi echo "$code" } oci_get() { local name="$1" ref="$2" out code out="$(mktemp)" code="$(curl -sS -u "${OCI_USER}:${GITEA_TOKEN}" \ -H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json' \ -o "$out" -w '%{http_code}' "${GITEA_URL}/v2/${OWNER}/${name}/manifests/${ref}" || true)" if [[ ! "$code" =~ ^2[0-9][0-9]$ ]]; then rm -f "$out"; return 1; fi cat "$out"; rm -f "$out" } # ============================================================================= # Data load — fetch ALL versions of PKG_TYPE for OWNER once (paginated), cache it. # NOTE: the type filter is the whole reason --all used to miss 13 GB. --all # means "every package OF THIS TYPE", never "every package". # ============================================================================= VERSIONS_JSON="$TMP/versions.json" # JSON-lines, one version object per line load_versions() { if [[ -n "$PRUNE_TEST_VERSIONS" ]]; then jq -c '.[]' "$PRUNE_TEST_VERSIONS" > "$VERSIONS_JSON" info "Loaded $(grep -c . "$VERSIONS_JSON") version records from the test fixture." return fi step "Fetching ${PKG_TYPE} packages for owner '${OWNER}' from ${GITEA_URL} ..." : > "$VERSIONS_JSON" local page=1 limit=50 body n total=0 while :; do body="$(api_get "/api/v1/packages/${OWNER}?type=${PKG_TYPE}&page=${page}&limit=${limit}")" || exit 1 n="$(echo "$body" | jq 'length')" echo "$body" | jq -c '.[]' >> "$VERSIONS_JSON" total=$((total + n)) [[ "$n" -lt "$limit" ]] && break page=$((page + 1)) done info "Loaded ${total} version records across $(jq -rs '[.[].name]|unique|length' "$VERSIONS_JSON") package(s)." } # Distinct package names (sorted) package_names() { jq -rs '[.[].name]|unique|.[]' "$VERSIONS_JSON"; } # Tagged versions of a package, newest first: "tagcreated_at" tagged_versions() { local name="$1" jq -rs --arg n "$name" ' [ .[] | select(.name==$n and (.version|startswith("sha256:")|not)) ] | sort_by(.created_at) | reverse | .[] | "\(.version)\t\(.created_at)"' "$VERSIONS_JSON" } # Count of digest (sha256:) versions of a package digest_count() { local name="$1" jq -rs --arg n "$name" '[ .[] | select(.name==$n and (.version|startswith("sha256:"))) ] | length' "$VERSIONS_JSON" } # ============================================================================= # Size resolution (best-effort). Per-version "apparent" size in bytes. # container: tag -> OCI index -> referenced manifest digests -> files-API sums. # Caches digest->bytes so shared digests aren't refetched. Apparent # size OVERSTATES reclaim, because base layers are shared. # other: version -> files-API sum directly. Blobs are unique to the # version, so apparent size IS the reclaimable size. # ============================================================================= declare -A DIGEST_BYTES_CACHE=() digest_files_bytes() { # -> bytes (files-API sum) local name="$1" digest="$2" local key="${name}@${digest}" if [[ -n "${DIGEST_BYTES_CACHE[$key]:-}" ]]; then echo "${DIGEST_BYTES_CACHE[$key]}"; return; fi local body sum body="$(api_get "/api/v1/packages/${OWNER}/${PKG_TYPE}/${name}/${digest}/files" 2>/dev/null || echo '[]')" sum="$(echo "$body" | jq '[.[].size] | add // 0' 2>/dev/null || echo 0)" DIGEST_BYTES_CACHE[$key]="$sum" echo "$sum" } resolve_tag_bytes() { # -> apparent bytes (0 if unresolved) local name="$1" tag="$2" man total=0 d # Non-container types have no manifest indirection: the version's files ARE # its bytes. URL-encode nothing — Gitea version strings here are path-safe. if ! $IS_CONTAINER; then digest_files_bytes "$name" "$tag"; return; fi man="$(oci_get "$name" "$tag")" || { echo 0; return; } if echo "$man" | jq -e '.manifests' >/dev/null 2>&1; then # OCI index / manifest list: sum referenced manifest digest versions while IFS= read -r d; do [[ -z "$d" ]] && continue total=$(( total + $(digest_files_bytes "$name" "$d") )) done < <(echo "$man" | jq -r '.manifests[].digest') else # Single image manifest: config + layers from the manifest itself total="$(echo "$man" | jq '((.config.size // 0) + ([.layers[].size] | add // 0))')" fi echo "$total" } human() { # bytes -> human readable local b="${1:-0}" if command -v numfmt &>/dev/null; then numfmt --to=iec --suffix=B "$b" 2>/dev/null || echo "${b}B"; else echo "${b}B"; fi } is_protected() { # -> 0 if protected (a --protect regex, or a version in use) local name="$1" tag="$2" rx for rx in "${PROTECT[@]}"; do [[ "$tag" =~ $rx ]] && return 0; done [[ -n "${VOUCH_PROTECT[${name}:${tag}]:-}" ]] && return 0 return 1 } # protect_reason -> why it is kept ("" if not protected) protect_reason() { local name="$1" tag="$2" rx for rx in "${PROTECT[@]}"; do [[ "$tag" =~ $rx ]] && { echo "matches --protect ${rx}"; return; }; done echo "${VOUCH_PROTECT[${name}:${tag}]:-}" } # ============================================================================= # The versions IN USE (decision 62). Fills VOUCH_PROTECT or returns non-zero — the caller # then refuses to prune. Every value is checked to look like a version before it counts. # ============================================================================= vouch_add() { # VOUCH_PROTECT["$1:$2"]="${VOUCH_PROTECT["$1:$2"]:+${VOUCH_PROTECT["$1:$2"]}; }$3" } load_vouch_protect() { if [[ -n "$PRUNE_TEST_PROTECT" ]]; then local pkg tag why while read -r pkg tag why; do [[ -n "$pkg" ]] && vouch_add "$pkg" "$tag" "$why"; done < "$PRUNE_TEST_PROTECT" info "In-use list from the test fixture: ${#VOUCH_PROTECT[@]} version(s)." return 0 fi local hubpw="${HUB_PW:-}" page floor golden agent minagent bake hubtag if [[ -z "$hubpw" ]]; then local cred="${FELHOM_EU}/scripts/read_credential.py" f f="$(mktemp)" if [[ -r "$cred" ]] && python3 "$cred" HUB_PW "$f" >/dev/null 2>&1; then hubpw="$(cat "$f")"; fi rm -f "$f" fi [[ -n "$hubpw" ]] || { error "In-use list: no HUB_PW (env or ~/.config/credentials) — cannot read the vouch."; return 1; } if [[ -z "$HUB_URL" ]]; then local ip; ip="$(sudo -n kubectl -n felhom-system get svc hub -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)" [[ -n "$ip" ]] && HUB_URL="http://${ip}:8080" fi [[ -n "$HUB_URL" ]] || { error "In-use list: the hub's address is unknown (set HUB_URL)."; return 1; } page="$(mktemp)" # Basic auth through -u; NEVER -w '%{redirect_url}' (it prints the password, R-580). curl -fsS --max-time 20 -u ":${hubpw}" -o "$page" "${HUB_URL}/configuration" 2>/dev/null \ || { rm -f "$page"; error "In-use list: GET ${HUB_URL}/configuration failed."; return 1; } hubpw="" floor="$(grep -o 'name="min_controller_version" value="[^"]*"' "$page" | head -1 | sed 's/.*value="//; s/"$//')" golden="$(tr '\n' ' ' < "$page" | grep -o '