v0.351.0 (unreleased): old -rc controller images and old infra images are deleted (decision 56 amended, decision 285)
gates / gates (push) Successful in 31s

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
adminandClaude Opus 5.5 committed 2026-11-09 10:23:06 +01:00
1 parent 64b99b74e3
commit 27360d6abe
10 files changed
+471 -31

No files matched your search

@@ -28,7 +28,11 @@ import (
// of the running version; the PREVIOUS — the self-update's own record (update-state.json previous_image of a swap
// that ended on the running version), or, when there is no such record or its image is gone, the highest version
// below the running one (logged as "by version order"); every version ABOVE the running one (a pulled target the
// swap has not taken yet); every tag that is not a plain X.Y.Z (latest, -rc) — left alone, logged. A candidate is an
// swap has not taken yet); every tag that is not a version at all (latest) — left alone, logged. A test build's tag
// (X.Y.Z-rc.N) IS a version since v0.351.0 (decision 56, amended 2026-11-09): util.ParseVersionPre orders it below its release, so an old rc is a
// candidate like an old release — 25 of them (2.2 GB) sat on demo-hp on 2026-11-09 because the rule left every -rc
// alone. The previous by version order prefers a release; a test build is the previous only when no release is below
// the running one. A box that RUNS a test build is judged too (it was skipped). A candidate is an
// untagged (<none>) controller image or one whose every tag is a version below the previous. Deleted by exact
// name/ID, never forced, never by prune; an ID that also carries another repository's name is left alone. NOTHING is
// deleted while the controller is swapping itself (selfUpdatingNow). Registry tags are never touched.
@@ -45,7 +49,7 @@ type ControllerImageRecord struct {
func (m *Manager) RetainControllerImages(rec ControllerImageRecord) ([]string, error) {
imageRetentionMu.Lock()
defer imageRetentionMu.Unlock()
running, err := util.ParseVersion(rec.Running)
running, err := util.ParseVersionPre(rec.Running)
if err != nil || rec.Repo == "" {
m.logger.Printf("[INFO] [stacks] controller image retention: skipped — running version %q is not a release (or no repository)", rec.Running)
return nil, nil
@@ -82,15 +86,27 @@ func (m *Manager) RetainControllerImages(rec ControllerImageRecord) ([]string, e
}
}
if prevTag == "" {
var best *util.Version
// A release first: a box on 0.350.0 holding 0.349.0 and 0.350.0-rc.3 keeps 0.349.0 for a hand roll-back.
var best, bestPre *util.Version
for _, im := range imgs {
if im.Repo != repo {
continue
}
if v, err := util.ParseVersion(im.Tag); err == nil && v.Compare(running) < 0 && (best == nil || v.Compare(*best) > 0) {
vv := v
best = &vv
v, err := util.ParseVersionPre(im.Tag)
if err != nil || v.Compare(running) >= 0 {
continue
}
vv := v
if v.Pre == "" {
if best == nil || v.Compare(*best) > 0 {
best = &vv
}
} else if bestPre == nil || v.Compare(*bestPre) > 0 {
bestPre = &vv
}
}
if best == nil {
best = bestPre
}
if best != nil {
prevTag, prevHow = best.Raw, "by version order (no swap record names one present)"
@@ -98,7 +114,7 @@ func (m *Manager) RetainControllerImages(rec ControllerImageRecord) ([]string, e
}
var prev *util.Version
if prevTag != "" {
if v, err := util.ParseVersion(prevTag); err == nil {
if v, err := util.ParseVersionPre(prevTag); err == nil {
prev = &v
}
}
@@ -140,10 +156,10 @@ func (m *Manager) RetainControllerImages(rec ControllerImageRecord) ([]string, e
continue
}
names = append(names, im.Repo+":"+im.Tag)
v, err := util.ParseVersion(im.Tag)
v, err := util.ParseVersionPre(im.Tag)
switch {
case err != nil:
deletable, odd = false, im.Tag // latest, -rc: not ours to judge
deletable, odd = false, im.Tag // latest: not a version, not ours to judge
case v.Compare(running) >= 0:
deletable = false // the running one, or a pulled target not swapped to yet
case prev == nil || v.Compare(*prev) >= 0:
@@ -48,16 +48,17 @@ func TestControllerRetention_KeepsRunningAndPreviousDeletesOlder(t *testing.T) {
if strings.Contains(s, ":0.284.2,") {
t.Fatalf("the previous controller (0.284.2) was deleted: %v", f.rmi)
}
for _, want := range []string{ctlRepo + ":0.284.1", ctlRepo + ":0.283.1", "sha256:CNONE"} {
// 0.273.0-rc1: a test build below the previous goes like an old release (decision 56 amended 2026-11-09).
for _, want := range []string{ctlRepo + ":0.284.1", ctlRepo + ":0.283.1", "sha256:CNONE", ctlRepo + ":0.273.0-rc1"} {
if !strings.Contains(s, ","+want+",") {
t.Fatalf("%s (older than the previous / untagged) was not deleted: rmi=%v", want, f.rmi)
}
}
if strings.Contains(s, "0.273.0-rc1") || strings.Contains(s, "acme/web") || strings.Contains(s, "W3") {
t.Fatalf("a non-version tag or another repository's image was touched: %v", f.rmi)
if strings.Contains(s, "acme/web") || strings.Contains(s, "W3") {
t.Fatalf("another repository's image was touched: %v", f.rmi)
}
if len(got) != 3 {
t.Fatalf("deleted %d, want 3: %v", len(got), got)
if len(got) != 4 {
t.Fatalf("deleted %d, want 4: %v", len(got), got)
}
}
@@ -183,3 +184,67 @@ func TestRetainImagesAfterUpdate_AppGoneDoesNotPanic(t *testing.T) {
t.Fatalf("deleted images for an app that is gone: %v", f.rmi)
}
}
// Decision 56 amended 2026-11-09: demo-hp's guest held 25 old test builds (X.Y.Z-rc.N, 2.2 GB) because every -rc tag was
// "left alone". A test build is now a version (util.ParseVersionPre: below its release): an old one goes; a test build
// ABOVE the running one (a pulled target) and one BETWEEN the previous and the running one stay; `latest` stays.
// COMPANION RED-PROOF: parse tags with util.ParseVersion again → "an old test build (0.330.0-rc.2) was not deleted".
func TestControllerRetention_OldTestBuildsGo(t *testing.T) {
m := retentionManager(t)
f := &fakeImages{
imgs: []localImage{
{ID: "sha256:R350", Repo: ctlRepo, Tag: "0.350.0", Size: "348MB"},
{ID: "sha256:R349", Repo: ctlRepo, Tag: "0.349.0", Size: "348MB"},
{ID: "sha256:RC350", Repo: ctlRepo, Tag: "0.350.0-rc.3", Size: "348MB"},
{ID: "sha256:RC330", Repo: ctlRepo, Tag: "0.330.0-rc.2", Size: "347MB"},
{ID: "sha256:RC351", Repo: ctlRepo, Tag: "0.351.0-rc.1", Size: "348MB"},
{ID: "sha256:LATEST", Repo: ctlRepo, Tag: "latest", Size: "348MB"},
},
containers: map[string]string{"c-ctl": "sha256:R350"},
}
withFakeImages(t, f)
// No swap record: the previous by version order is the RELEASE 0.349.0, not the newer test build 0.350.0-rc.3.
if _, err := m.RetainControllerImages(ControllerImageRecord{Repo: ctlRepo, Running: "0.350.0"}); err != nil {
t.Fatal(err)
}
s := rmiSet(f)
if !strings.Contains(s, ","+ctlRepo+":0.330.0-rc.2,") {
t.Fatalf("an old test build (0.330.0-rc.2) was not deleted: %v", f.rmi)
}
if strings.Contains(s, ":0.349.0,") {
t.Fatalf("the previous RELEASE (0.349.0) was deleted — version order must prefer a release over a test build: %v", f.rmi)
}
if strings.Contains(s, ":0.351.0-rc.1,") || strings.Contains(s, ":latest,") || strings.Contains(s, ":0.350.0,") {
t.Fatalf("a newer test build, latest, or the running one was deleted: %v", f.rmi)
}
// 0.350.0-rc.3 sits between the previous (0.349.0) and the running one: kept, like a release there would be.
if strings.Contains(s, ":0.350.0-rc.3,") {
t.Fatalf("a test build between the previous and the running one was deleted: %v", f.rmi)
}
}
// A box that RUNS a test build (Tester 1 during an rc round) is judged too — it was skipped before.
// COMPANION RED-PROOF: parse rec.Running with util.ParseVersion again → "a box running a test build deleted nothing".
func TestControllerRetention_BoxOnATestBuildIsJudged(t *testing.T) {
m := retentionManager(t)
f := &fakeImages{
imgs: []localImage{
{ID: "sha256:RC2", Repo: ctlRepo, Tag: "0.351.0-rc.2", Size: "348MB"},
{ID: "sha256:RC1", Repo: ctlRepo, Tag: "0.351.0-rc.1", Size: "348MB"},
{ID: "sha256:R350", Repo: ctlRepo, Tag: "0.350.0", Size: "348MB"},
{ID: "sha256:R349", Repo: ctlRepo, Tag: "0.349.0", Size: "348MB"},
},
containers: map[string]string{"c-ctl": "sha256:RC2"},
}
withFakeImages(t, f)
if _, err := m.RetainControllerImages(ControllerImageRecord{Repo: ctlRepo, Running: "0.351.0-rc.2", Previous: ctlRepo + ":0.350.0"}); err != nil {
t.Fatal(err)
}
s := rmiSet(f)
if !strings.Contains(s, ","+ctlRepo+":0.349.0,") {
t.Fatalf("a box running a test build deleted nothing: %v", f.rmi)
}
if strings.Contains(s, ":0.350.0,") || strings.Contains(s, ":0.351.0-rc.2,") || strings.Contains(s, ":0.351.0-rc.1,") {
t.Fatalf("the recorded previous, the running test build or one between them was deleted: %v", f.rmi)
}
}
@@ -0,0 +1,186 @@
package stacks
import (
"context"
"fmt"
"sort"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
// ── Infrastructure image retention (`09` §3 decision 285, operator 2026-11-09) ──────────────────────────
//
// Decision 53 cleans app images and decision 56 the controller's own; the pinned infrastructure images (traefik,
// cloudflared, FileBrowser, Samba — infra.Images()) were cleaned by nothing, so every pin a box ever ran stayed:
// on demo-hp's guest on 2026-11-09, 2 old traefik, 3 old cloudflared, 2 old FileBrowser and 1 old Samba image
// (about 1 GB). Every one of them rides the whole-guest backup (mp0 holds /var/lib/docker), on a system disk the
// local tier already struggles to fit (R-548).
//
// THE RULE, decision 56's shape applied per pinned repository: a box keeps the PINNED tag (what this controller
// runs), every version ABOVE it (a box rolled back to an older controller keeps the newer pin's image for the way
// forward), the highest version BELOW it (the previous pin — a hand roll-back), and every image a container uses
// (running or stopped). An untagged image of the repository that no container uses is a candidate; a tag that is
// not a version (latest) is left alone. Deleted by exact name/ID, never forced, never by prune; an ID that also
// carries another repository's name is left alone. No pass while an image-pulling compose command runs (R-863),
// and none while the controller swaps itself. Pinned by internal/stacks/infra_image_retention_test.go.
// RetainInfraImages applies the rule once to every pinned infrastructure image. Returns what it deleted, and
// errImageRetentionBusy when image work was in flight (nothing was judged; try again later).
func (m *Manager) RetainInfraImages(pins []string) ([]string, error) {
imageRetentionMu.Lock()
defer imageRetentionMu.Unlock()
if m.selfUpdatingNow() {
m.logger.Printf("[INFO] [stacks] infra image retention: skipped — the controller is swapping itself")
return nil, errImageRetentionBusy
}
endCleanup, ok := dockerexec.TryImageCleanup()
if !ok {
m.logger.Printf("[INFO] [stacks] infra image retention: skipped — an app install, update, restore or undo is pulling images now (R-863); tried again later")
return nil, errImageRetentionBusy
}
defer endCleanup()
imgs, err := listLocalImages()
if err != nil {
return nil, err
}
used, err := imagesUsedByContainers()
if err != nil {
m.logger.Printf("[WARN] [stacks] infra image retention: the containers could not be read (%v) — NOTHING is deleted", err)
return nil, err
}
var deleted []string
for _, pin := range pins {
deleted = append(deleted, m.retainOneInfraRepo(pin, imgs, used)...)
}
return deleted, nil
}
func (m *Manager) retainOneInfraRepo(pin string, imgs []localImage, used map[string]bool) []string {
r, tag, _ := splitRepoTag(pin)
repo := normRepo(r)
pinned, err := util.ParseVersionPre(tag)
if err != nil {
m.logger.Printf("[INFO] [stacks] infra image retention: %s — the pin %q is not a version; the repository is left alone", repo, tag)
return nil
}
byID := map[string][]localImage{}
var prev *util.Version
for _, im := range imgs {
byID[im.ID] = append(byID[im.ID], im)
if im.Repo != repo {
continue
}
if v, err := util.ParseVersionPre(im.Tag); err == nil && v.Compare(pinned) < 0 && (prev == nil || v.Compare(*prev) > 0) {
vv := v
prev = &vv
}
}
prevTag := ""
if prev != nil {
prevTag = prev.Raw
}
ids := make([]string, 0, len(byID))
for id := range byID {
ids = append(ids, id)
}
sort.Strings(ids)
var deleted []string
considered, candidates := 0, 0
defer func() {
// ONE line per repository per pass, whatever it did — the positive observable that the pass ran (R-96 rule 3).
m.logger.Printf("[INFO] [stacks] infra image retention: %s — pass over %d image(s), pinned %s, previous %q, %d candidate(s), %d deleted, the rest kept",
repo, considered, tag, prevTag, candidates, len(deleted))
}()
for _, id := range ids {
group := byID[id]
var mine []localImage
other := false
for _, im := range group {
if im.Repo == repo {
mine = append(mine, im)
} else {
other = true
}
}
if len(mine) == 0 {
continue
}
considered++
if used[id] {
continue
}
deletable, odd := true, ""
var names []string
for _, im := range mine {
if im.Tag == "<none>" || im.Tag == "" {
continue
}
names = append(names, im.Repo+":"+im.Tag)
v, err := util.ParseVersionPre(im.Tag)
switch {
case err != nil:
deletable, odd = false, im.Tag // latest: not a version, not ours to judge
case v.Compare(pinned) >= 0:
deletable = false // the pin, or a newer pin's image
case prev == nil || v.Compare(*prev) >= 0:
deletable = false // the previous pin
}
}
if !deletable {
if odd != "" {
m.logger.Printf("[INFO] [stacks] infra image retention: %s carries a tag that is not a version (%s) — left alone", shortID(id), odd)
}
continue
}
candidates++
if other {
m.logger.Printf("[INFO] [stacks] infra image retention: %s also carries another repository's name — left alone", shortID(id))
continue
}
targets := names
if len(targets) == 0 {
targets = []string{id}
}
ok := true
for _, t := range targets {
if out, err := imageDocker("rmi", t); err != nil {
m.logger.Printf("[WARN] [stacks] infra image retention: docker refused to delete %s (%s): %s", t, shortID(id), truncateStr(strings.TrimSpace(out), 160))
ok = false
break
}
}
if !ok {
continue
}
label := strings.Join(names, ",")
if label == "" {
label = repo + ":<none>"
}
m.logger.Printf("[INFO] [stacks] infra image retention: deleted %s (%s, %s) — older than the previous pin and no container uses it (decision 285)", label, shortID(id), mine[0].Size)
deleted = append(deleted, fmt.Sprintf("%s@%s", label, shortID(id)))
}
return deleted
}
// RunInfraImageRetention runs RetainInfraImages, and again every `every` while image work is in flight, at most
// `tries` times (R-863's retry shape: a household's first installs land in the minutes after a start).
func (m *Manager) RunInfraImageRetention(ctx context.Context, pins []string, every time.Duration, tries int) {
for i := 0; i < tries; i++ {
_, err := m.RetainInfraImages(pins)
if err != errImageRetentionBusy {
if err != nil {
m.logger.Printf("[WARN] [stacks] infra image retention: %v — tried again at the next start", err)
}
return
}
select {
case <-ctx.Done():
return
case <-time.After(every):
}
}
m.logger.Printf("[WARN] [stacks] infra image retention: not run after %d tries — tried again at the next start", tries)
}
@@ -0,0 +1,128 @@
package stacks
import (
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
)
// Decision 285 (2026-11-09): the pinned infrastructure images keep the pin, every newer version, the previous pin and
// every image a container uses; older ones go. The fixture is demo-hp's guest as measured on 2026-11-09 (before the
// hand clean-up): traefik v3.6.7/v3.7.13/v3.7.16 beside the pinned v3.7.17, FileBrowser 1.3.3/1.5.6 beside 1.5.8.
// Docker is the imageDocker seam (fakeImages) — nothing here reaches a daemon.
var infraPins = []string{"traefik:v3.7.17", "gtstef/filebrowser:1.5.8-stable", "gitea.dooplex.hu/admin/felhom-samba:1.2.0"}
func infraImages() *fakeImages {
return &fakeImages{
imgs: []localImage{
{ID: "sha256:T3717", Repo: "traefik", Tag: "v3.7.17", Size: "195MB"},
{ID: "sha256:T3716", Repo: "traefik", Tag: "v3.7.16", Size: "194MB"},
{ID: "sha256:T3713", Repo: "traefik", Tag: "v3.7.13", Size: "195MB"},
{ID: "sha256:T367", Repo: "traefik", Tag: "v3.6.7", Size: "186MB"},
{ID: "sha256:TNONE", Repo: "traefik", Tag: "<none>", Size: "190MB"},
{ID: "sha256:F158", Repo: "gtstef/filebrowser", Tag: "1.5.8-stable", Size: "196MB"},
{ID: "sha256:F156", Repo: "gtstef/filebrowser", Tag: "1.5.6-stable", Size: "195MB"},
{ID: "sha256:F133", Repo: "gtstef/filebrowser", Tag: "1.3.3-stable", Size: "215MB"},
// Samba is pinned but not enabled on this box (no container): the pin itself must stay.
{ID: "sha256:S120", Repo: "gitea.dooplex.hu/admin/felhom-samba", Tag: "1.2.0", Size: "107MB"},
{ID: "sha256:S110", Repo: "gitea.dooplex.hu/admin/felhom-samba", Tag: "1.1.0", Size: "103MB"},
{ID: "sha256:S100", Repo: "gitea.dooplex.hu/admin/felhom-samba", Tag: "1.0.0", Size: "101MB"},
// not an infra repository: never touched by this pass
{ID: "sha256:A1", Repo: "acme/web", Tag: "1", Size: "100MB"},
{ID: "sha256:ALP", Repo: "alpine", Tag: "latest", Size: "8MB"},
},
containers: map[string]string{"c-traefik": "sha256:T3717", "c-fb": "sha256:F158"},
}
}
// COMPANION RED-PROOF: drop the `prev` case from the keep switch → "the previous pin traefik:v3.7.16 was deleted";
// drop the `v >= pinned` case → "the pinned Samba image (no container) was deleted".
func TestInfraRetention_KeepsPinAndPreviousDeletesOlder(t *testing.T) {
m := retentionManager(t)
f := infraImages()
withFakeImages(t, f)
got, err := m.RetainInfraImages(infraPins)
if err != nil {
t.Fatal(err)
}
s := rmiSet(f)
for _, keep := range []string{"traefik:v3.7.17", "traefik:v3.7.16", "gtstef/filebrowser:1.5.8-stable", "gtstef/filebrowser:1.5.6-stable"} {
if strings.Contains(s, ","+keep+",") {
t.Fatalf("the previous pin %s was deleted: %v", keep, f.rmi)
}
}
if strings.Contains(s, "felhom-samba:1.2.0") || strings.Contains(s, "felhom-samba:1.1.0") {
t.Fatalf("the pinned Samba image (no container) or its previous pin was deleted: %v", f.rmi)
}
for _, want := range []string{"traefik:v3.7.13", "traefik:v3.6.7", "sha256:TNONE", "gtstef/filebrowser:1.3.3-stable", "gitea.dooplex.hu/admin/felhom-samba:1.0.0"} {
if !strings.Contains(s, ","+want+",") {
t.Fatalf("%s (older than the previous pin / untagged) was not deleted: rmi=%v", want, f.rmi)
}
}
if strings.Contains(s, "acme/web") || strings.Contains(s, "sha256:A1") || strings.Contains(s, "alpine") {
t.Fatalf("an image outside the pinned repositories was touched: %v", f.rmi)
}
if len(got) != 5 {
t.Fatalf("deleted %d, want 5: %v", len(got), got)
}
}
// A box rolled back to an older controller pins an OLDER traefik: the newer one stays (the way forward), and a
// container that still runs an old image keeps it.
// COMPANION RED-PROOF: drop the `used[id]` check → "an image a container uses was deleted".
func TestInfraRetention_NewerPinAndInUseKept(t *testing.T) {
m := retentionManager(t)
f := infraImages()
f.containers["c-old-fb"] = "sha256:F133"
withFakeImages(t, f)
if _, err := m.RetainInfraImages([]string{"traefik:v3.7.16", "gtstef/filebrowser:1.5.8-stable"}); err != nil {
t.Fatal(err)
}
s := rmiSet(f)
if strings.Contains(s, "traefik:v3.7.17") {
t.Fatalf("a version above the pin was deleted: %v", f.rmi)
}
if strings.Contains(s, "traefik:v3.7.13") || !strings.Contains(s, "traefik:v3.6.7") {
t.Fatalf("pinned v3.7.16: want v3.7.13 kept (previous) and v3.6.7 deleted: %v", f.rmi)
}
if strings.Contains(s, "1.3.3-stable") {
t.Fatalf("an image a container uses was deleted: %v", f.rmi)
}
}
// While an image-pulling compose command runs (R-863), or the controller swaps itself, NOTHING is deleted.
// COMPANION RED-PROOF: drop the TryImageCleanup check → "deleted while image work was in flight".
func TestInfraRetention_NothingWhileImageWorkOrSwap(t *testing.T) {
m := retentionManager(t)
f := infraImages()
withFakeImages(t, f)
end := dockerexec.HoldImageWork()
_, err := m.RetainInfraImages(infraPins)
end()
if err != errImageRetentionBusy || len(f.rmi) != 0 {
t.Fatalf("deleted while image work was in flight: err=%v rmi=%v", err, f.rmi)
}
m.SetSelfUpdatingCheck(func() bool { return true })
_, err = m.RetainInfraImages(infraPins)
m.SetSelfUpdatingCheck(func() bool { return false })
if err != errImageRetentionBusy || len(f.rmi) != 0 {
t.Fatalf("deleted while the controller swaps itself: err=%v rmi=%v", err, f.rmi)
}
}
// A pin that is not a version, and a tag that is not a version, are left alone.
func TestInfraRetention_NonVersionsLeftAlone(t *testing.T) {
m := retentionManager(t)
f := infraImages()
f.imgs = append(f.imgs, localImage{ID: "sha256:TLATEST", Repo: "traefik", Tag: "latest", Size: "190MB"})
withFakeImages(t, f)
if _, err := m.RetainInfraImages([]string{"traefik:v3.7.17", "gtstef/filebrowser:latest"}); err != nil {
t.Fatal(err)
}
s := rmiSet(f)
if strings.Contains(s, "traefik:latest") || strings.Contains(s, "filebrowser") {
t.Fatalf("a non-version tag or a repository with a non-version pin was touched: %v", f.rmi)
}
}