diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6d23035 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# Audit logs written by gitea-image-prune.sh (--log overrides the location). +# Token-free by construction, but operational output — not repo content. +logs/ diff --git a/gitea-image-prune.sh b/gitea-image-prune.sh index 3061eca..a3514fe 100755 --- a/gitea-image-prune.sh +++ b/gitea-image-prune.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # ============================================================================= -# gitea-image-prune.sh — inspect & prune old container images in the -# self-hosted Gitea registry, then reclaim disk. +# gitea-image-prune.sh — inspect & prune old container images and generic +# artifacts in the self-hosted Gitea registry, then +# reclaim disk. # ============================================================================= # Target server : gitea.dooplex.hu (Gitea 1.26.2, owner namespace "admin") # Best run on : build server 192.168.0.180 (has kubectl for --measure), @@ -14,6 +15,24 @@ # so felhom-controller (~96 tags) / felhom-hub (~42 tags) accumulate one image # per build and the Gitea Longhorn PVC keeps filling. # +# PACKAGE TYPES — --type, default "container" (added 2026-08-23) +# Gitea namespaces packages by TYPE, and every API path embeds it. This script +# handled only type=container, so --all meant "all *container* packages" and +# type=generic artifacts were invisible to every mode. That is how +# admin/felhom-golden (22 versions x ~627 MB = 13 GB) grew to 88% of the Gitea +# PVC while --all --keep 7 --apply reported success: it pruned ~1.3 GB of +# container images and never saw the 13 GB sitting next to them. +# One type per run. Sweep both: +# ./gitea-image-prune.sh --all --keep 7 --apply --yes --reclaim # containers +# ./gitea-image-prune.sh --type generic --all --keep 3 --apply --yes --reclaim +# +# HOW GITEA STORES GENERIC PACKAGES (contrast with containers, below) +# No index, no manifest indirection, no shared layers: a version IS its files, +# and its blobs are unique to it. So reclaim has only TWO steps — delete the +# version, then let cleanup_packages GC the blobs. There are never orphaned +# manifests to hunt, and "apparent" size equals real reclaimed size (the +# shared-layer caveat that applies to containers does not apply here). +# # HOW GITEA STORES CONTAINER IMAGES (load-bearing — drives the reclaim design) # A pushed tag is an OCI image *index* (a ~850 B pointer). The real bytes live # in untagged "sha256:..." manifest *versions* (config + layer blobs, ~9-25 MB @@ -66,6 +85,8 @@ # ./gitea-image-prune.sh --repo felhom-controller --older-than 90 --apply --reclaim # ./gitea-image-prune.sh --all --keep 15 --apply --yes --reclaim --measure # cron-friendly # ./gitea-image-prune.sh --repo felhom-hub reclaim # run cleanup cron only +# ./gitea-image-prune.sh --type generic --all list # the OTHER half +# ./gitea-image-prune.sh --type generic --repo felhom-golden --keep 3 --apply --reclaim # # SET-AND-FORGET COMPLEMENT: configure a native Gitea cleanup rule per owner # (package settings -> Cleanup Rules: keep most-recent N, exclude ^latest$); @@ -92,6 +113,7 @@ note() { echo -e "${CYAN} $*${NC}"; } declare -a REPOS=() declare -a PROTECT=('^latest$') # always-protected tags; --protect appends ALL=false +PKG_TYPE="container" # Gitea package type; --type. Every API path embeds it. ACTION="" # list | prune | reclaim (positional or inferred) KEEP="" OLDER_THAN="" @@ -122,6 +144,7 @@ while [[ $# -gt 0 ]]; do list|prune|reclaim) ACTION="$1"; shift ;; --repo) REPOS+=("$2"); shift 2 ;; --all) ALL=true; shift ;; + --type) PKG_TYPE="$2"; shift 2 ;; --keep) KEEP="$2"; shift 2 ;; --older-than) OLDER_THAN="$2"; shift 2 ;; --dry-run) APPLY=false; shift ;; @@ -146,6 +169,18 @@ if [[ -n "$KEEP" && -n "$OLDER_THAN" ]]; then fi if [[ -n "$KEEP" && ! "$KEEP" =~ ^[0-9]+$ ]]; then error "--keep must be a non-negative integer"; exit 2; fi if [[ -n "$OLDER_THAN" && ! "$OLDER_THAN" =~ ^[0-9]+$ ]]; then error "--older-than must be a non-negative integer (days)"; exit 2; fi +if [[ ! "$PKG_TYPE" =~ ^[a-z_]+$ ]]; then error "--type must be a Gitea package type slug (e.g. container, generic)"; exit 2; fi + +# Only container carries the OCI index -> manifest -> blob indirection. Every +# other Gitea type stores a version's files directly, so it takes the simple +# path: delete the version, GC the blobs. Both are implemented; anything beyond +# container/generic is untested here but structurally identical to generic. +IS_CONTAINER=false +[[ "$PKG_TYPE" == "container" ]] && IS_CONTAINER=true +case "$PKG_TYPE" in + container|generic) ;; + *) warn "--type '${PKG_TYPE}' is untested; treating it like 'generic' (no manifest indirection)." ;; +esac # ============================================================================= # Preflight @@ -268,16 +303,18 @@ oci_get() { } # ============================================================================= -# Data load — fetch ALL container versions for OWNER once (paginated), cache it. +# Data load — fetch ALL versions of PKG_TYPE for OWNER once (paginated), cache it. +# NOTE: the type filter is the whole reason --all used to miss 13 GB. --all +# means "every package OF THIS TYPE", never "every package". # ============================================================================= VERSIONS_JSON="$TMP/versions.json" # JSON-lines, one version object per line load_versions() { - step "Fetching container packages for owner '${OWNER}' from ${GITEA_URL} ..." + step "Fetching ${PKG_TYPE} packages for owner '${OWNER}' from ${GITEA_URL} ..." : > "$VERSIONS_JSON" local page=1 limit=50 body n total=0 while :; do - body="$(api_get "/api/v1/packages/${OWNER}?type=container&page=${page}&limit=${limit}")" || exit 1 + body="$(api_get "/api/v1/packages/${OWNER}?type=${PKG_TYPE}&page=${page}&limit=${limit}")" || exit 1 n="$(echo "$body" | jq 'length')" echo "$body" | jq -c '.[]' >> "$VERSIONS_JSON" total=$((total + n)) @@ -306,9 +343,12 @@ digest_count() { } # ============================================================================= -# Size resolution (best-effort, OCI). Per-tag "apparent" image size in bytes. -# tag -> OCI index -> referenced manifest digests -> files-API byte sums. -# Caches digest->bytes so repeated/shared digests aren't refetched. +# Size resolution (best-effort). Per-version "apparent" size in bytes. +# container: tag -> OCI index -> referenced manifest digests -> files-API sums. +# Caches digest->bytes so shared digests aren't refetched. Apparent +# size OVERSTATES reclaim, because base layers are shared. +# other: version -> files-API sum directly. Blobs are unique to the +# version, so apparent size IS the reclaimable size. # ============================================================================= declare -A DIGEST_BYTES_CACHE=() @@ -317,14 +357,17 @@ digest_files_bytes() { # -> bytes (files-API sum) local key="${name}@${digest}" if [[ -n "${DIGEST_BYTES_CACHE[$key]:-}" ]]; then echo "${DIGEST_BYTES_CACHE[$key]}"; return; fi local body sum - body="$(api_get "/api/v1/packages/${OWNER}/container/${name}/${digest}/files" 2>/dev/null || echo '[]')" + body="$(api_get "/api/v1/packages/${OWNER}/${PKG_TYPE}/${name}/${digest}/files" 2>/dev/null || echo '[]')" sum="$(echo "$body" | jq '[.[].size] | add // 0' 2>/dev/null || echo 0)" DIGEST_BYTES_CACHE[$key]="$sum" echo "$sum" } -resolve_tag_bytes() { # -> apparent image bytes (0 if unresolved) +resolve_tag_bytes() { # -> apparent bytes (0 if unresolved) local name="$1" tag="$2" man total=0 d + # Non-container types have no manifest indirection: the version's files ARE + # its bytes. URL-encode nothing — Gitea version strings here are path-safe. + if ! $IS_CONTAINER; then digest_files_bytes "$name" "$tag"; return; fi man="$(oci_get "$name" "$tag")" || { echo 0; return; } if echo "$man" | jq -e '.manifests' >/dev/null 2>&1; then # OCI index / manifest list: sum referenced manifest digest versions @@ -381,6 +424,9 @@ referenced_digests() { # find_orphans -> prints orphan sha256: versions (one per line); rc1 if unsafe find_orphans() { local name="$1" ref="$TMP/ref.${name//\//_}" sha="$TMP/sha.${name//\//_}" + # Only container has untagged manifest versions to orphan. Everything else + # frees its blobs the moment the version is deleted — nothing to hunt. + $IS_CONTAINER || return 0 referenced_digests "$name" "$ref" || return 1 LC_ALL=C sort -u "$ref" -o "$ref" jq -rs --arg n "$name" \ @@ -429,10 +475,16 @@ measure_bytes() { # echoes byte count of packages dir, or empty if unavailable # ---- LIST ---------------------------------------------------------------- do_list() { - local name="$1" line tag created bytes total_bytes=0 count=0 prot + local name="$1" line tag created bytes total_bytes=0 count=0 prot hdr echo "" - echo -e "${BOLD}== ${name} ==${NC} (owner ${OWNER}; $(digest_count "$name") digest manifests)" - printf ' %-28s %-22s %-12s %s\n' "TAG" "UPLOADED" "APPARENT" "FLAG" + if $IS_CONTAINER; then + echo -e "${BOLD}== ${name} ==${NC} (owner ${OWNER}; $(digest_count "$name") digest manifests)" + hdr="TAG" + else + echo -e "${BOLD}== ${name} ==${NC} (owner ${OWNER}; type ${PKG_TYPE})" + hdr="VERSION" + fi + printf ' %-28s %-22s %-12s %s\n' "$hdr" "UPLOADED" "APPARENT" "FLAG" printf ' %-28s %-22s %-12s %s\n' "---" "--------" "--------" "----" while IFS=$'\t' read -r tag created; do [[ -z "$tag" ]] && continue @@ -447,8 +499,13 @@ do_list() { info " ${count} tagged version(s). (sizes skipped: --no-sizes)" else info " ${count} tagged version(s); apparent total ≈ $(human "$total_bytes")" - note " CAVEAT: apparent sizes count shared base layers once PER TAG, so they" - note " overlap heavily. Real reclaimed space is much less — measure with --measure." + if $IS_CONTAINER; then + note " CAVEAT: apparent sizes count shared base layers once PER TAG, so they" + note " overlap heavily. Real reclaimed space is much less — measure with --measure." + else + note " ${PKG_TYPE} blobs are unique per version (no shared layers), so these" + note " sizes are the real reclaimable bytes." + fi fi } @@ -498,7 +555,13 @@ do_prune_repo() { plan_bytes=$((plan_bytes + bytes)) printf ' %-28s %s\n' "$tag" "$szlabel" done - $NO_SIZES || note " apparent total to delete ≈ $(human "$plan_bytes") (overlaps shared layers — real reclaim less)" + if ! $NO_SIZES; then + if $IS_CONTAINER; then + note " apparent total to delete ≈ $(human "$plan_bytes") (overlaps shared layers — real reclaim less)" + else + note " total to delete ≈ $(human "$plan_bytes") (blobs unique per version — reclaim is real)" + fi + fi if ! $APPLY; then warn " DRY-RUN — nothing deleted. Re-run with --apply to delete." @@ -516,7 +579,7 @@ do_prune_repo() { for tag in "${PLAN_DELETE[@]}"; do if $NO_SIZES; then bytes=0; else bytes="$(resolve_tag_bytes "$name" "$tag")"; fi - code="$(api_delete "/api/v1/packages/${OWNER}/container/${name}/${tag}")" && { + code="$(api_delete "/api/v1/packages/${OWNER}/${PKG_TYPE}/${name}/${tag}")" && { info " deleted ${name}:${tag} (HTTP ${code})" audit "APPLIED deleted ${OWNER}/${name}:${tag} apparent=${bytes}B http=${code}" PRUNE_DELETED=$((PRUNE_DELETED + 1)) @@ -533,7 +596,11 @@ do_prune_repo() { # Step 2 + 3 of the proven mechanism: delete orphaned manifest versions, then # trigger (or defer to) the cleanup_packages GC cron. Honors --dry-run/--apply. do_reclaim() { - step "Reclaim: find & delete orphaned (untagged, unreferenced) manifests, then GC blobs." + if $IS_CONTAINER; then + step "Reclaim: find & delete orphaned (untagged, unreferenced) manifests, then GC blobs." + else + step "Reclaim: ${PKG_TYPE} has no manifest indirection — GC the deleted versions' blobs." + fi # Reload state — a preceding prune deleted tags, so the cached listing is stale. load_versions @@ -546,7 +613,10 @@ do_reclaim() { unsafe=$((unsafe + 1)); continue # warning already emitted; skip this package fi norph="$(printf '%s\n' "$orphans" | grep -c . || true)" - if [[ "$norph" -eq 0 ]]; then info " ${name}: no orphaned manifests."; continue; fi + if [[ "$norph" -eq 0 ]]; then + if $IS_CONTAINER; then info " ${name}: no orphaned manifests."; fi + continue + fi total_orphans=$((total_orphans + norph)) if ! $APPLY; then info " ${name}: ${norph} orphaned manifest version(s) WOULD be deleted (dry-run):" @@ -556,7 +626,7 @@ do_reclaim() { info " ${name}: deleting ${norph} orphaned manifest version(s)..." while IFS= read -r d; do [[ -z "$d" ]] && continue - code="$(api_delete "/api/v1/packages/${OWNER}/container/${name}/${d}")" && { + code="$(api_delete "/api/v1/packages/${OWNER}/${PKG_TYPE}/${name}/${d}")" && { deleted=$((deleted + 1)); audit "RECLAIM deleted-orphan ${OWNER}/${name}/${d} http=${code}" } || { failed=$((failed + 1)); error " FAILED orphan ${name}/${d} (HTTP ${code})" @@ -567,12 +637,18 @@ do_reclaim() { if ! $APPLY; then echo "" - warn " DRY-RUN — ${total_orphans} orphaned manifest(s) shown, none deleted, cron not triggered." - warn " Re-run reclaim with --apply to delete them and free disk." + if $IS_CONTAINER; then + warn " DRY-RUN — ${total_orphans} orphaned manifest(s) shown, none deleted, cron not triggered." + else + warn " DRY-RUN — GC cron not triggered." + fi + warn " Re-run reclaim with --apply to free disk." return 0 fi - info " Orphaned manifests: deleted=${deleted} failed=${failed}$( [[ $unsafe -gt 0 ]] && echo " (skipped ${unsafe} pkg for safety)")" + if $IS_CONTAINER; then + info " Orphaned manifests: deleted=${deleted} failed=${failed}$( [[ $unsafe -gt 0 ]] && echo " (skipped ${unsafe} pkg for safety)")" + fi # --- Step 3: GC the now-unreferenced blobs --- local cron_ran=false res @@ -580,6 +656,8 @@ do_reclaim() { res="$(trigger_cleanup_cron)" && cron_ran=true || true case "$res" in ok\ *) info " cron '${CLEANUP_CRON}' triggered (HTTP ${res#ok }) — GC running." + note " Blobs UPLOADED within [cron.cleanup_packages] OLDER_THAN (24h default) are" + note " NOT collected by this pass — their space returns on a later run." audit "RECLAIM triggered cron ${CLEANUP_CRON} ${res}" ;; denied) warn " Token lacks write:admin — cannot trigger the GC cron directly." note " The orphaned manifests are deleted; their unique blobs are now unreferenced and" @@ -612,18 +690,25 @@ interactive_menu() { load_versions local -a names=(); local n while IFS= read -r n; do names+=("$n"); done < <(package_names) - if [[ "${#names[@]}" -eq 0 ]]; then warn "No container packages found for owner '${OWNER}'."; exit 0; fi + if [[ "${#names[@]}" -eq 0 ]]; then warn "No ${PKG_TYPE} packages found for owner '${OWNER}'."; exit 0; fi echo "" - echo -e "${BOLD}Container packages under '${OWNER}':${NC}" + echo -e "${BOLD}Packages of type '${PKG_TYPE}' under '${OWNER}':${NC}" local i for i in "${!names[@]}"; do - printf ' %2d) %-22s %s tags, %s digest manifests\n' \ - "$((i+1))" "${names[$i]}" \ - "$(tagged_versions "${names[$i]}" | grep -c . || true)" \ - "$(digest_count "${names[$i]}")" + if $IS_CONTAINER; then + printf ' %2d) %-22s %s tags, %s digest manifests\n' \ + "$((i+1))" "${names[$i]}" \ + "$(tagged_versions "${names[$i]}" | grep -c . || true)" \ + "$(digest_count "${names[$i]}")" + else + printf ' %2d) %-22s %s versions\n' \ + "$((i+1))" "${names[$i]}" \ + "$(tagged_versions "${names[$i]}" | grep -c . || true)" + fi done - echo " a) all packages" + echo " a) all packages of this type" + note " (other types are NOT listed — re-run with --type to see them)" echo "" read -r -p "Select package(s) [numbers/space-sep, or 'a' for all]: " sel local -a chosen=() @@ -659,9 +744,9 @@ interactive_menu() { # ============================================================================= echo "" info "╔══════════════════════════════════════════╗" -info "║ Gitea container-image prune ║" +info "║ Gitea package prune ║" info "╚══════════════════════════════════════════╝" -info "Server: ${GITEA_URL} Owner: ${OWNER} Log: ${LOG_FILE}" +info "Server: ${GITEA_URL} Owner: ${OWNER} Type: ${PKG_TYPE} Log: ${LOG_FILE}" info "Auth: ${CRED_SRC}$( [[ -n "$GITEA_USER" ]] && echo " (user: ${GITEA_USER})")" # Sanity: Gitea version (public, no auth) @@ -687,6 +772,7 @@ fi declare -a TARGETS=() if $ALL; then while IFS= read -r n; do TARGETS+=("$n"); done < <(package_names) + warn "--all covers type '${PKG_TYPE}' ONLY. Other types are untouched by this run." else for r in "${REPOS[@]}"; do if jq -rs --arg n "$r" 'any(.[]; .name==$n)' "$VERSIONS_JSON" | grep -q true; then @@ -707,8 +793,8 @@ if [[ "$ACTION" == "prune" && -z "$KEEP" && -z "$OLDER_THAN" ]]; then error "prune needs --keep N or --older-than DAYS."; exit 2 fi -info "Action: ${ACTION} Targets: ${TARGETS[*]}" -audit "RUN action=${ACTION} owner=${OWNER} targets='${TARGETS[*]}' apply=${APPLY} keep='${KEEP}' older_than='${OLDER_THAN}' protect='${PROTECT[*]}'" +info "Action: ${ACTION} Type: ${PKG_TYPE} Targets: ${TARGETS[*]}" +audit "RUN action=${ACTION} owner=${OWNER} type=${PKG_TYPE} targets='${TARGETS[*]}' apply=${APPLY} keep='${KEEP}' older_than='${OLDER_THAN}' protect='${PROTECT[*]}'" case "$ACTION" in list)