From 27360d6abec763848067980ef5ff9b6f0cfa7e6b Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Mon, 9 Nov 2026 10:23:06 +0100 Subject: [PATCH] v0.351.0 (unreleased): old -rc controller images and old infra images are deleted (decision 56 amended, decision 285) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- CHANGELOG.md | 27 +++ CONTEXT.md | 6 + REPORT.md | 31 +-- REUSE.md | 3 +- controller/README.md | 8 +- controller/cmd/controller/main.go | 4 + .../stacks/controller_image_retention.go | 34 +++- .../stacks/controller_image_retention_test.go | 75 ++++++- .../internal/stacks/infra_image_retention.go | 186 ++++++++++++++++++ .../stacks/infra_image_retention_test.go | 128 ++++++++++++ 10 files changed, 471 insertions(+), 31 deletions(-) create mode 100644 controller/internal/stacks/infra_image_retention.go create mode 100644 controller/internal/stacks/infra_image_retention_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 65ccbf1e..0282a143 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,30 @@ +## v0.351.0 (unreleased, 2026-11-09) — old test builds and old infrastructure images are deleted (decision 56 amended, decision 285) + +### 2026-11-09 + +**MinAgent: 0.158.1** (unchanged). + +**Why (measured 2026-11-09, demo-hp guest 9201, `docker system df -v`):** the whole-guest backup copies `/var/lib/docker` (mp0, +`backup=1`), and demo-hp's local tier was refused for space again (felhom.eu R-548). Of 27.5 GB of images, 13.2 GB ran in +containers, 9.8 GB were the apps' undo images (decision 53, kept on purpose), 2.2 GB were 28 old controller images and +1.35 GB were old infrastructure and helper images. Nothing deleted the last two kinds. + +**Controller images: a test build is a version (decision 56 amended).** `RetainControllerImages` left every `-rc` tag alone, +so 25 test builds stayed. Tags, the running version and the previous now parse with `util.ParseVersionPre` (an rc orders +below its release): an old rc goes like an old release; an rc above the running one or between the previous and the +running one stays; `latest` stays. A box that RUNS a test build is judged (it was skipped). The fallback previous prefers +a release over a newer test build. Tests `TestControllerRetention_OldTestBuildsGo`, `_BoxOnATestBuildIsJudged`; the +existing test's rc expectation changed with the rule. + +**Infrastructure images (decision 285, new).** `RetainInfraImages(infra.Images())`, 3 minutes after every start: per pinned +repository the pin, newer versions, the previous pin and every in-use image stay; older and untagged ones go by exact +name/ID; never forced, never pruned; not while an app pulls (R-863, retried every 2 minutes, 30 tries) or while the +controller swaps itself. Tests `TestInfraRetention_*` (4). Customer boxes get a new traefik or cloudflared pin every few +weeks (~65–195 MB each); this is the part a customer box needs. + +Red-proofs: six code mutants, each test red (the previous-pin keep, the in-use check, the R-863 lock, the tag parser, the +running-version parser, the release preference). Green gate: `go build`, `go vet`, `go test ./...` rc 0; `controller_gates.py --fast` OK. + ## v0.350.0 (2026-11-03) — Traefik v3.7.17 (GHSA-xv5x-242w-7hf7); a restore forgets the update-held flag (R-1347); a failed put-back at start-up mails (R-1348); three false start-up lines (R-1339, R-1351, R-1323) ### 2026-11-03 diff --git a/CONTEXT.md b/CONTEXT.md index 41620b5b..8c536c3d 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -1,5 +1,11 @@ # CONTEXT.md — Project Memory +> **2026-11-09 (attended) — v0.351.0 in code, not released.** Operator's word 2026-11-09 („go with A"): decision 56 amended +> (an old `-rc` controller image is deleted like an old release) and decision 285 (pinned infra images keep the pin + the +> previous pin). Measured cause: demo-hp's whole-guest local copy refused for space (felhom.eu R-548); 3.5 GB of its 27.5 GB +> of images were these kinds. The bigger lever — the whole-guest copy carries every Docker image — is a design question +> (felhom.eu register, filed the same day). + > **2026-11-03 (day, attended) — v0.350.0 RELEASED, the global floor since 08:18 CET (MinAgent 0.158.1).** Image `sha256:9ee53908…` from `5c336d8`. Traefik v3.7.17 (GHSA-xv5x), R-1347, R-1348, R-1339, R-1351, R-1323. Golden 0.350.0 vouched 10:18:10 CET. > > **2026-10-29 (evening, attended) — v0.349.0 RELEASED, the global floor since 19:24 CET (MinAgent 0.158.1).** Image diff --git a/REPORT.md b/REPORT.md index b5b2e589..1a4fadc0 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,19 +1,20 @@ -# REPORT — v0.350.0 released as the global floor: Traefik v3.7.17 first (2026-11-03, attended) +# REPORT — v0.351.0 in code: old test builds and old infrastructure images are deleted (2026-11-09, attended) -**Plain words first.** Every box runs 0.350.0 since 08:18 CET. It closes a real hole: a stranger with no password could fill a box's web front door until it stopped. It also carries five small fixes. +**Plain words first.** demo-hp's whole-box copy was too big for its system disk. Part of it was old images that +nothing deletes: 25 old controller test builds and old copies of the system images (traefik, cloudflared, FileBrowser, +Samba). The controller now deletes both kinds by itself, 3 minutes after each start. **Not released yet.** -| Commit | CI | What | -|---|---|---| -| `c4b6844` | 2551 green | Traefik v3.7.17 (GHSA-xv5x-242w-7hf7); R-1347, R-1348, R-1339, R-1351, R-1323 | -| `3764445` | 2560 green | R-1348: the mail names a restore | -| `5c336d8` | 2568 green | v0.350.0 heading — the release build's commit. MinAgent 0.158.1 | +| | | +|---|---| +| Version | 0.351.0 (CHANGELOG heading „unreleased"), MinAgent 0.158.1 unchanged | +| Rulings | `09` §3 decision 56 amended, decision 285 new — operator 2026-11-09 („go with A") | +| Tests | `TestControllerRetention_OldTestBuildsGo`, `_BoxOnATestBuildIsJudged`, `TestInfraRetention_*` (4); one existing rc expectation changed with the rule | +| Red-proof | six code mutants, each test red: previous-pin keep, in-use check, R-863 lock, tag parser, running-version parser, release preference | +| Green gate | `go build ./...`, `go vet ./...`, `go test -count=1 ./...` rc 0; `controller_gates.py --fast` OK | -- rc.1 (`13e34d4`, `sha256:cbf4d42f…`) on 9202: walk green, sign-in works. -- 0.350.0: `sha256:9ee53908e6d95f0340c41bf4eaba954280084a8e6927de59779440e3551df261`. -- Per-box floors 08:04:24 CET: all three moved within 45 s, walk green, 26 public names answer, `https_only_check.py` green, sign-in works on Tester 1 and demo-hp (not driven on the lived-in N100). -- One restore on Tester 1 on the release build: clean. -- Global floor 08:18:16 CET, its own save; the 30-minute watch stayed healthy, 0 ERROR or held lines at the hub. +**Measured cause** (felhom.eu `REPORT.md`): on demo-hp guest 9201, 2.2 GB of old controller images and 1.35 GB of old +infra and helper images, of 27.5 GB. The same images were removed there by hand the same day, so a live proof needs +a box that still has them (Tester 1: 37 images, 3.8 GB unused). -**The load test on 9202:** v3.7.16 18.7 → 79.0 MiB after 920 NTLM connections, not released; v3.7.17 flat near 25 MiB for 25,116. - -**Test proof only:** R-1351 and R-1323 (their conditions did not fire). R-1347 and R-1348 were proven on the test image of the same code, not on the release build. Evidence: felhom.eu `documentation/audits/release-2026-11-03/part1/`. +**Not done:** no image built, no box moved, no floor. The release path is the skill's: an `-rc` on the three test boxes, +the box walk, then the tag and the floor. diff --git a/REUSE.md b/REUSE.md index f2ec3463..7af3d489 100644 --- a/REUSE.md +++ b/REUSE.md @@ -35,7 +35,8 @@ | `dockerexec.BeginImageWork(args)` / `TryImageCleanup()` / `ImagePulling(args)` (v0.294.0, R-863) | controller/internal/dockerexec/imagework.go | `defer dockerexec.BeginImageWork(args)()` | **Every NEW place that runs `compose up/pull/create/run` must hold it** (the stacks compose helpers, appexport's restore and the FileBrowser sync already do) | An image pulled by compose is named by no container until compose creates one; a clean-up pass in that window deleted it (measured, R-863). The pass takes the lock exclusively without waiting; pulling commands wait for a running pass (seconds) | | `nightchain.Ledger` / `CatchUp` / `ResumeWatch` / `ComputeBanner` (v0.295.0, R-871) | controller/internal/nightchain/ | `Open(path, now)`, `MarkEnded(leg, t)`, `Missed(now, W, loc)`, `Evaluate(ctx, why)` | "was a night missed?" and making it up; the missed-backup banner's rule | The ledger is an ATTEMPT record (ran to its end) — never evidence a backup exists; a NEW nightly backup leg must be wrapped by `withLeg` in main.go and listed in `nightchain.Order`, or the catch-up never makes it up; never add the update leg to `catchUpLegs` | | `scheduler.DailyLateLimit` (v0.295.0) | controller/internal/scheduler/scheduler.go | — | a daily timer firing > 60 min late (host suspend) is skipped | do not "fix" a late fire by running it: the app-update leg would run at noon | -| `stacks.RetainControllerImages(ControllerImageRecord)` · `selfupdate.UpdateState.RecordedPrevious` (v0.285.0, decision 56) | controller/internal/stacks/controller_image_retention.go | `({Repo, Running, Previous})` | Deletes controller images older than the running + previous one | The previous comes from the SWAP RECORD (success onto the running version), version order only as the fallback; versions above the running one and non-version tags are kept; skipped while the controller swaps itself; same `imageDocker` seam | +| `stacks.RetainControllerImages(ControllerImageRecord)` · `selfupdate.UpdateState.RecordedPrevious` (v0.285.0, decision 56) | controller/internal/stacks/controller_image_retention.go | `({Repo, Running, Previous})` | Deletes controller images older than the running + previous one | The previous comes from the SWAP RECORD (success onto the running version), version order only as the fallback (a release preferred over a test build, v0.351.0); versions above the running one and non-version tags (`latest`) are kept — an `-rc` tag IS a version (`util.ParseVersionPre`) since v0.351.0; skipped while the controller swaps itself; same `imageDocker` seam | +| `stacks.RetainInfraImages(pins)` / `RunInfraImageRetention(ctx, pins, every, tries)` (v0.351.0, decision 285) | controller/internal/stacks/infra_image_retention.go | `(infra.Images())` | Deletes pinned infra images older than the pin + the previous pin | Per repository: the pin, newer versions, the previous pin and in-use images stay; untagged unused go; non-version tags stay; skipped (errImageRetentionBusy) while an app pulls (R-863) or the controller swaps; same `imageDocker` seam | | `stacks.CloseSignupNow` / `CloseSignupOffered` / `applyNativeLock` (v0.282.0, decisions 47/49) | controller/internal/stacks/after_setup.go | `(name)` | The app's own sign-up switch after the setup; "close sign-up now" for an app installed before the rule | **Check the installed compose reads the variable** (an old install carries the old compose until its next update) — never record a lock that is not there. One run per app at a time (`nativeLockBusy`) | | `backup.judgeCopy` / `HollowCopies` / `SetHollowCopyNotify` (Part D, v0.279.0) | controller/internal/backup/hollow_watch.go | `(app, tier, unitDir)` | A RUNNING app whose newest copy holds no data → operator digest once/day + page sentence | Uses `unitCarriesData` (the manifest, never size); a stopped held app is never flagged | | `web.nightChain` (R-705, v0.279.0) | controller/internal/web/night_chain.go | `POST /api/debug/backup/night-chain` | The night's four legs now, in order | Refuses while any op/update/chain runs; the leg uses `RunUpdateLegNow` | diff --git a/controller/README.md b/controller/README.md index dbad1cc4..0f1b615e 100644 --- a/controller/README.md +++ b/controller/README.md @@ -2244,10 +2244,16 @@ that folder is never a dead end, and an install never runs into it silently (R-6 - **Controller image retention (v0.285.0, decision 56)** — 3 minutes after every start the box keeps the controller image it runs and the one before it (the swap record `update-state.json` `previous_image` of a swap that ended on the running version; without one, the highest version below the running one), every version above the running one (a - pulled target) and every non-version tag (`latest`, `-rc`); older and untagged controller images are deleted by exact + pulled target) and every non-version tag (`latest`); since v0.351.0 a test build (`X.Y.Z-rc.N`) is a version below its + release, so an old one goes too, and the fallback previous prefers a release; older and untagged controller images are deleted by exact name/ID after the in-use check. Nothing while the controller swaps itself; registry tags are never touched. The agent's own roll-back needs only the running image (it restores what `/etc/felhom-controller-image` named when the swap began). See `internal/stacks/controller_image_retention.go`. +- **Infrastructure image retention (v0.351.0, decision 285)** — in the same start-up pass, for each pinned infra image + (`infra.Images()`: traefik, cloudflared, FileBrowser, Samba) the box keeps the pin, every newer version, the highest + version below the pin (the previous pin) and every image a container uses; older and untagged images of that + repository are deleted by exact name/ID. Non-version tags stay. Skipped while an app pulls (R-863, retried every 2 + minutes) or while the controller swaps itself. See `internal/stacks/infra_image_retention.go`. - **The setup gate (v0.280.0, decision 46)** — `.felhom.yml` `setup_gate: true` + optional `setup_done_probe: {url, field, done}`. A FRESH install is closed to everyone but the household: the traefik file `/traefik/dynamic/setup-gate-.yml` is written BEFORE the first start (a failed write refuses the install) and diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index 80c0e0c1..6440ec53 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -2062,6 +2062,10 @@ func main() { prevCtl = st.RecordedPrevious(Version) } _, _ = stackMgr.RetainControllerImages(stacks.ControllerImageRecord{Repo: cfg.SelfUpdate.Image, Running: Version, Previous: prevCtl}) + // decision 285 (2026-11-09): the pinned infrastructure images (traefik, cloudflared, FileBrowser, Samba) — keep + // the pin and the one before it. A pin moves only with a controller release, so once per start is enough; the + // infra stacks were reconciled long before these 3 minutes. Retried every 2 minutes while an app pulls (R-863). + stackMgr.RunInfraImageRetention(ctx, infra.Images(), 2*time.Minute, 30) }() // --- Initialize API router --- diff --git a/controller/internal/stacks/controller_image_retention.go b/controller/internal/stacks/controller_image_retention.go index cac412cf..839ec11b 100644 --- a/controller/internal/stacks/controller_image_retention.go +++ b/controller/internal/stacks/controller_image_retention.go @@ -28,7 +28,11 @@ import ( // of the running version; the PREVIOUS — the self-update's own record (update-state.json previous_image of a swap // that ended on the running version), or, when there is no such record or its image is gone, the highest version // below the running one (logged as "by version order"); every version ABOVE the running one (a pulled target the -// swap has not taken yet); every tag that is not a plain X.Y.Z (latest, -rc) — left alone, logged. A candidate is an +// swap has not taken yet); every tag that is not a version at all (latest) — left alone, logged. A test build's tag +// (X.Y.Z-rc.N) IS a version since v0.351.0 (decision 56, amended 2026-11-09): util.ParseVersionPre orders it below its release, so an old rc is a +// candidate like an old release — 25 of them (2.2 GB) sat on demo-hp on 2026-11-09 because the rule left every -rc +// alone. The previous by version order prefers a release; a test build is the previous only when no release is below +// the running one. A box that RUNS a test build is judged too (it was skipped). A candidate is an // untagged () controller image or one whose every tag is a version below the previous. Deleted by exact // name/ID, never forced, never by prune; an ID that also carries another repository's name is left alone. NOTHING is // deleted while the controller is swapping itself (selfUpdatingNow). Registry tags are never touched. @@ -45,7 +49,7 @@ type ControllerImageRecord struct { func (m *Manager) RetainControllerImages(rec ControllerImageRecord) ([]string, error) { imageRetentionMu.Lock() defer imageRetentionMu.Unlock() - running, err := util.ParseVersion(rec.Running) + running, err := util.ParseVersionPre(rec.Running) if err != nil || rec.Repo == "" { m.logger.Printf("[INFO] [stacks] controller image retention: skipped — running version %q is not a release (or no repository)", rec.Running) return nil, nil @@ -82,15 +86,27 @@ func (m *Manager) RetainControllerImages(rec ControllerImageRecord) ([]string, e } } if prevTag == "" { - var best *util.Version + // A release first: a box on 0.350.0 holding 0.349.0 and 0.350.0-rc.3 keeps 0.349.0 for a hand roll-back. + var best, bestPre *util.Version for _, im := range imgs { if im.Repo != repo { continue } - if v, err := util.ParseVersion(im.Tag); err == nil && v.Compare(running) < 0 && (best == nil || v.Compare(*best) > 0) { - vv := v - best = &vv + v, err := util.ParseVersionPre(im.Tag) + if err != nil || v.Compare(running) >= 0 { + continue } + vv := v + if v.Pre == "" { + if best == nil || v.Compare(*best) > 0 { + best = &vv + } + } else if bestPre == nil || v.Compare(*bestPre) > 0 { + bestPre = &vv + } + } + if best == nil { + best = bestPre } if best != nil { prevTag, prevHow = best.Raw, "by version order (no swap record names one present)" @@ -98,7 +114,7 @@ func (m *Manager) RetainControllerImages(rec ControllerImageRecord) ([]string, e } var prev *util.Version if prevTag != "" { - if v, err := util.ParseVersion(prevTag); err == nil { + if v, err := util.ParseVersionPre(prevTag); err == nil { prev = &v } } @@ -140,10 +156,10 @@ func (m *Manager) RetainControllerImages(rec ControllerImageRecord) ([]string, e continue } names = append(names, im.Repo+":"+im.Tag) - v, err := util.ParseVersion(im.Tag) + v, err := util.ParseVersionPre(im.Tag) switch { case err != nil: - deletable, odd = false, im.Tag // latest, -rc: not ours to judge + deletable, odd = false, im.Tag // latest: not a version, not ours to judge case v.Compare(running) >= 0: deletable = false // the running one, or a pulled target not swapped to yet case prev == nil || v.Compare(*prev) >= 0: diff --git a/controller/internal/stacks/controller_image_retention_test.go b/controller/internal/stacks/controller_image_retention_test.go index ea5b6f07..b2b029ec 100644 --- a/controller/internal/stacks/controller_image_retention_test.go +++ b/controller/internal/stacks/controller_image_retention_test.go @@ -48,16 +48,17 @@ func TestControllerRetention_KeepsRunningAndPreviousDeletesOlder(t *testing.T) { if strings.Contains(s, ":0.284.2,") { t.Fatalf("the previous controller (0.284.2) was deleted: %v", f.rmi) } - for _, want := range []string{ctlRepo + ":0.284.1", ctlRepo + ":0.283.1", "sha256:CNONE"} { + // 0.273.0-rc1: a test build below the previous goes like an old release (decision 56 amended 2026-11-09). + for _, want := range []string{ctlRepo + ":0.284.1", ctlRepo + ":0.283.1", "sha256:CNONE", ctlRepo + ":0.273.0-rc1"} { if !strings.Contains(s, ","+want+",") { t.Fatalf("%s (older than the previous / untagged) was not deleted: rmi=%v", want, f.rmi) } } - if strings.Contains(s, "0.273.0-rc1") || strings.Contains(s, "acme/web") || strings.Contains(s, "W3") { - t.Fatalf("a non-version tag or another repository's image was touched: %v", f.rmi) + if strings.Contains(s, "acme/web") || strings.Contains(s, "W3") { + t.Fatalf("another repository's image was touched: %v", f.rmi) } - if len(got) != 3 { - t.Fatalf("deleted %d, want 3: %v", len(got), got) + if len(got) != 4 { + t.Fatalf("deleted %d, want 4: %v", len(got), got) } } @@ -183,3 +184,67 @@ func TestRetainImagesAfterUpdate_AppGoneDoesNotPanic(t *testing.T) { t.Fatalf("deleted images for an app that is gone: %v", f.rmi) } } + +// Decision 56 amended 2026-11-09: demo-hp's guest held 25 old test builds (X.Y.Z-rc.N, 2.2 GB) because every -rc tag was +// "left alone". A test build is now a version (util.ParseVersionPre: below its release): an old one goes; a test build +// ABOVE the running one (a pulled target) and one BETWEEN the previous and the running one stay; `latest` stays. +// COMPANION RED-PROOF: parse tags with util.ParseVersion again → "an old test build (0.330.0-rc.2) was not deleted". +func TestControllerRetention_OldTestBuildsGo(t *testing.T) { + m := retentionManager(t) + f := &fakeImages{ + imgs: []localImage{ + {ID: "sha256:R350", Repo: ctlRepo, Tag: "0.350.0", Size: "348MB"}, + {ID: "sha256:R349", Repo: ctlRepo, Tag: "0.349.0", Size: "348MB"}, + {ID: "sha256:RC350", Repo: ctlRepo, Tag: "0.350.0-rc.3", Size: "348MB"}, + {ID: "sha256:RC330", Repo: ctlRepo, Tag: "0.330.0-rc.2", Size: "347MB"}, + {ID: "sha256:RC351", Repo: ctlRepo, Tag: "0.351.0-rc.1", Size: "348MB"}, + {ID: "sha256:LATEST", Repo: ctlRepo, Tag: "latest", Size: "348MB"}, + }, + containers: map[string]string{"c-ctl": "sha256:R350"}, + } + withFakeImages(t, f) + // No swap record: the previous by version order is the RELEASE 0.349.0, not the newer test build 0.350.0-rc.3. + if _, err := m.RetainControllerImages(ControllerImageRecord{Repo: ctlRepo, Running: "0.350.0"}); err != nil { + t.Fatal(err) + } + s := rmiSet(f) + if !strings.Contains(s, ","+ctlRepo+":0.330.0-rc.2,") { + t.Fatalf("an old test build (0.330.0-rc.2) was not deleted: %v", f.rmi) + } + if strings.Contains(s, ":0.349.0,") { + t.Fatalf("the previous RELEASE (0.349.0) was deleted — version order must prefer a release over a test build: %v", f.rmi) + } + if strings.Contains(s, ":0.351.0-rc.1,") || strings.Contains(s, ":latest,") || strings.Contains(s, ":0.350.0,") { + t.Fatalf("a newer test build, latest, or the running one was deleted: %v", f.rmi) + } + // 0.350.0-rc.3 sits between the previous (0.349.0) and the running one: kept, like a release there would be. + if strings.Contains(s, ":0.350.0-rc.3,") { + t.Fatalf("a test build between the previous and the running one was deleted: %v", f.rmi) + } +} + +// A box that RUNS a test build (Tester 1 during an rc round) is judged too — it was skipped before. +// COMPANION RED-PROOF: parse rec.Running with util.ParseVersion again → "a box running a test build deleted nothing". +func TestControllerRetention_BoxOnATestBuildIsJudged(t *testing.T) { + m := retentionManager(t) + f := &fakeImages{ + imgs: []localImage{ + {ID: "sha256:RC2", Repo: ctlRepo, Tag: "0.351.0-rc.2", Size: "348MB"}, + {ID: "sha256:RC1", Repo: ctlRepo, Tag: "0.351.0-rc.1", Size: "348MB"}, + {ID: "sha256:R350", Repo: ctlRepo, Tag: "0.350.0", Size: "348MB"}, + {ID: "sha256:R349", Repo: ctlRepo, Tag: "0.349.0", Size: "348MB"}, + }, + containers: map[string]string{"c-ctl": "sha256:RC2"}, + } + withFakeImages(t, f) + if _, err := m.RetainControllerImages(ControllerImageRecord{Repo: ctlRepo, Running: "0.351.0-rc.2", Previous: ctlRepo + ":0.350.0"}); err != nil { + t.Fatal(err) + } + s := rmiSet(f) + if !strings.Contains(s, ","+ctlRepo+":0.349.0,") { + t.Fatalf("a box running a test build deleted nothing: %v", f.rmi) + } + if strings.Contains(s, ":0.350.0,") || strings.Contains(s, ":0.351.0-rc.2,") || strings.Contains(s, ":0.351.0-rc.1,") { + t.Fatalf("the recorded previous, the running test build or one between them was deleted: %v", f.rmi) + } +} diff --git a/controller/internal/stacks/infra_image_retention.go b/controller/internal/stacks/infra_image_retention.go new file mode 100644 index 00000000..5e1aa253 --- /dev/null +++ b/controller/internal/stacks/infra_image_retention.go @@ -0,0 +1,186 @@ +package stacks + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec" + "gitea.dooplex.hu/admin/felhom-controller/internal/util" +) + +// ── Infrastructure image retention (`09` §3 decision 285, operator 2026-11-09) ────────────────────────── +// +// Decision 53 cleans app images and decision 56 the controller's own; the pinned infrastructure images (traefik, +// cloudflared, FileBrowser, Samba — infra.Images()) were cleaned by nothing, so every pin a box ever ran stayed: +// on demo-hp's guest on 2026-11-09, 2 old traefik, 3 old cloudflared, 2 old FileBrowser and 1 old Samba image +// (about 1 GB). Every one of them rides the whole-guest backup (mp0 holds /var/lib/docker), on a system disk the +// local tier already struggles to fit (R-548). +// +// THE RULE, decision 56's shape applied per pinned repository: a box keeps the PINNED tag (what this controller +// runs), every version ABOVE it (a box rolled back to an older controller keeps the newer pin's image for the way +// forward), the highest version BELOW it (the previous pin — a hand roll-back), and every image a container uses +// (running or stopped). An untagged image of the repository that no container uses is a candidate; a tag that is +// not a version (latest) is left alone. Deleted by exact name/ID, never forced, never by prune; an ID that also +// carries another repository's name is left alone. No pass while an image-pulling compose command runs (R-863), +// and none while the controller swaps itself. Pinned by internal/stacks/infra_image_retention_test.go. + +// RetainInfraImages applies the rule once to every pinned infrastructure image. Returns what it deleted, and +// errImageRetentionBusy when image work was in flight (nothing was judged; try again later). +func (m *Manager) RetainInfraImages(pins []string) ([]string, error) { + imageRetentionMu.Lock() + defer imageRetentionMu.Unlock() + if m.selfUpdatingNow() { + m.logger.Printf("[INFO] [stacks] infra image retention: skipped — the controller is swapping itself") + return nil, errImageRetentionBusy + } + endCleanup, ok := dockerexec.TryImageCleanup() + if !ok { + m.logger.Printf("[INFO] [stacks] infra image retention: skipped — an app install, update, restore or undo is pulling images now (R-863); tried again later") + return nil, errImageRetentionBusy + } + defer endCleanup() + imgs, err := listLocalImages() + if err != nil { + return nil, err + } + used, err := imagesUsedByContainers() + if err != nil { + m.logger.Printf("[WARN] [stacks] infra image retention: the containers could not be read (%v) — NOTHING is deleted", err) + return nil, err + } + var deleted []string + for _, pin := range pins { + deleted = append(deleted, m.retainOneInfraRepo(pin, imgs, used)...) + } + return deleted, nil +} + +func (m *Manager) retainOneInfraRepo(pin string, imgs []localImage, used map[string]bool) []string { + r, tag, _ := splitRepoTag(pin) + repo := normRepo(r) + pinned, err := util.ParseVersionPre(tag) + if err != nil { + m.logger.Printf("[INFO] [stacks] infra image retention: %s — the pin %q is not a version; the repository is left alone", repo, tag) + return nil + } + byID := map[string][]localImage{} + var prev *util.Version + for _, im := range imgs { + byID[im.ID] = append(byID[im.ID], im) + if im.Repo != repo { + continue + } + if v, err := util.ParseVersionPre(im.Tag); err == nil && v.Compare(pinned) < 0 && (prev == nil || v.Compare(*prev) > 0) { + vv := v + prev = &vv + } + } + prevTag := "" + if prev != nil { + prevTag = prev.Raw + } + ids := make([]string, 0, len(byID)) + for id := range byID { + ids = append(ids, id) + } + sort.Strings(ids) + var deleted []string + considered, candidates := 0, 0 + defer func() { + // ONE line per repository per pass, whatever it did — the positive observable that the pass ran (R-96 rule 3). + m.logger.Printf("[INFO] [stacks] infra image retention: %s — pass over %d image(s), pinned %s, previous %q, %d candidate(s), %d deleted, the rest kept", + repo, considered, tag, prevTag, candidates, len(deleted)) + }() + for _, id := range ids { + group := byID[id] + var mine []localImage + other := false + for _, im := range group { + if im.Repo == repo { + mine = append(mine, im) + } else { + other = true + } + } + if len(mine) == 0 { + continue + } + considered++ + if used[id] { + continue + } + deletable, odd := true, "" + var names []string + for _, im := range mine { + if im.Tag == "" || im.Tag == "" { + continue + } + names = append(names, im.Repo+":"+im.Tag) + v, err := util.ParseVersionPre(im.Tag) + switch { + case err != nil: + deletable, odd = false, im.Tag // latest: not a version, not ours to judge + case v.Compare(pinned) >= 0: + deletable = false // the pin, or a newer pin's image + case prev == nil || v.Compare(*prev) >= 0: + deletable = false // the previous pin + } + } + if !deletable { + if odd != "" { + m.logger.Printf("[INFO] [stacks] infra image retention: %s carries a tag that is not a version (%s) — left alone", shortID(id), odd) + } + continue + } + candidates++ + if other { + m.logger.Printf("[INFO] [stacks] infra image retention: %s also carries another repository's name — left alone", shortID(id)) + continue + } + targets := names + if len(targets) == 0 { + targets = []string{id} + } + ok := true + for _, t := range targets { + if out, err := imageDocker("rmi", t); err != nil { + m.logger.Printf("[WARN] [stacks] infra image retention: docker refused to delete %s (%s): %s", t, shortID(id), truncateStr(strings.TrimSpace(out), 160)) + ok = false + break + } + } + if !ok { + continue + } + label := strings.Join(names, ",") + if label == "" { + label = repo + ":" + } + m.logger.Printf("[INFO] [stacks] infra image retention: deleted %s (%s, %s) — older than the previous pin and no container uses it (decision 285)", label, shortID(id), mine[0].Size) + deleted = append(deleted, fmt.Sprintf("%s@%s", label, shortID(id))) + } + return deleted +} + +// RunInfraImageRetention runs RetainInfraImages, and again every `every` while image work is in flight, at most +// `tries` times (R-863's retry shape: a household's first installs land in the minutes after a start). +func (m *Manager) RunInfraImageRetention(ctx context.Context, pins []string, every time.Duration, tries int) { + for i := 0; i < tries; i++ { + _, err := m.RetainInfraImages(pins) + if err != errImageRetentionBusy { + if err != nil { + m.logger.Printf("[WARN] [stacks] infra image retention: %v — tried again at the next start", err) + } + return + } + select { + case <-ctx.Done(): + return + case <-time.After(every): + } + } + m.logger.Printf("[WARN] [stacks] infra image retention: not run after %d tries — tried again at the next start", tries) +} diff --git a/controller/internal/stacks/infra_image_retention_test.go b/controller/internal/stacks/infra_image_retention_test.go new file mode 100644 index 00000000..91f34a8d --- /dev/null +++ b/controller/internal/stacks/infra_image_retention_test.go @@ -0,0 +1,128 @@ +package stacks + +import ( + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec" +) + +// Decision 285 (2026-11-09): the pinned infrastructure images keep the pin, every newer version, the previous pin and +// every image a container uses; older ones go. The fixture is demo-hp's guest as measured on 2026-11-09 (before the +// hand clean-up): traefik v3.6.7/v3.7.13/v3.7.16 beside the pinned v3.7.17, FileBrowser 1.3.3/1.5.6 beside 1.5.8. +// Docker is the imageDocker seam (fakeImages) — nothing here reaches a daemon. + +var infraPins = []string{"traefik:v3.7.17", "gtstef/filebrowser:1.5.8-stable", "gitea.dooplex.hu/admin/felhom-samba:1.2.0"} + +func infraImages() *fakeImages { + return &fakeImages{ + imgs: []localImage{ + {ID: "sha256:T3717", Repo: "traefik", Tag: "v3.7.17", Size: "195MB"}, + {ID: "sha256:T3716", Repo: "traefik", Tag: "v3.7.16", Size: "194MB"}, + {ID: "sha256:T3713", Repo: "traefik", Tag: "v3.7.13", Size: "195MB"}, + {ID: "sha256:T367", Repo: "traefik", Tag: "v3.6.7", Size: "186MB"}, + {ID: "sha256:TNONE", Repo: "traefik", Tag: "", Size: "190MB"}, + {ID: "sha256:F158", Repo: "gtstef/filebrowser", Tag: "1.5.8-stable", Size: "196MB"}, + {ID: "sha256:F156", Repo: "gtstef/filebrowser", Tag: "1.5.6-stable", Size: "195MB"}, + {ID: "sha256:F133", Repo: "gtstef/filebrowser", Tag: "1.3.3-stable", Size: "215MB"}, + // Samba is pinned but not enabled on this box (no container): the pin itself must stay. + {ID: "sha256:S120", Repo: "gitea.dooplex.hu/admin/felhom-samba", Tag: "1.2.0", Size: "107MB"}, + {ID: "sha256:S110", Repo: "gitea.dooplex.hu/admin/felhom-samba", Tag: "1.1.0", Size: "103MB"}, + {ID: "sha256:S100", Repo: "gitea.dooplex.hu/admin/felhom-samba", Tag: "1.0.0", Size: "101MB"}, + // not an infra repository: never touched by this pass + {ID: "sha256:A1", Repo: "acme/web", Tag: "1", Size: "100MB"}, + {ID: "sha256:ALP", Repo: "alpine", Tag: "latest", Size: "8MB"}, + }, + containers: map[string]string{"c-traefik": "sha256:T3717", "c-fb": "sha256:F158"}, + } +} + +// COMPANION RED-PROOF: drop the `prev` case from the keep switch → "the previous pin traefik:v3.7.16 was deleted"; +// drop the `v >= pinned` case → "the pinned Samba image (no container) was deleted". +func TestInfraRetention_KeepsPinAndPreviousDeletesOlder(t *testing.T) { + m := retentionManager(t) + f := infraImages() + withFakeImages(t, f) + got, err := m.RetainInfraImages(infraPins) + if err != nil { + t.Fatal(err) + } + s := rmiSet(f) + for _, keep := range []string{"traefik:v3.7.17", "traefik:v3.7.16", "gtstef/filebrowser:1.5.8-stable", "gtstef/filebrowser:1.5.6-stable"} { + if strings.Contains(s, ","+keep+",") { + t.Fatalf("the previous pin %s was deleted: %v", keep, f.rmi) + } + } + if strings.Contains(s, "felhom-samba:1.2.0") || strings.Contains(s, "felhom-samba:1.1.0") { + t.Fatalf("the pinned Samba image (no container) or its previous pin was deleted: %v", f.rmi) + } + for _, want := range []string{"traefik:v3.7.13", "traefik:v3.6.7", "sha256:TNONE", "gtstef/filebrowser:1.3.3-stable", "gitea.dooplex.hu/admin/felhom-samba:1.0.0"} { + if !strings.Contains(s, ","+want+",") { + t.Fatalf("%s (older than the previous pin / untagged) was not deleted: rmi=%v", want, f.rmi) + } + } + if strings.Contains(s, "acme/web") || strings.Contains(s, "sha256:A1") || strings.Contains(s, "alpine") { + t.Fatalf("an image outside the pinned repositories was touched: %v", f.rmi) + } + if len(got) != 5 { + t.Fatalf("deleted %d, want 5: %v", len(got), got) + } +} + +// A box rolled back to an older controller pins an OLDER traefik: the newer one stays (the way forward), and a +// container that still runs an old image keeps it. +// COMPANION RED-PROOF: drop the `used[id]` check → "an image a container uses was deleted". +func TestInfraRetention_NewerPinAndInUseKept(t *testing.T) { + m := retentionManager(t) + f := infraImages() + f.containers["c-old-fb"] = "sha256:F133" + withFakeImages(t, f) + if _, err := m.RetainInfraImages([]string{"traefik:v3.7.16", "gtstef/filebrowser:1.5.8-stable"}); err != nil { + t.Fatal(err) + } + s := rmiSet(f) + if strings.Contains(s, "traefik:v3.7.17") { + t.Fatalf("a version above the pin was deleted: %v", f.rmi) + } + if strings.Contains(s, "traefik:v3.7.13") || !strings.Contains(s, "traefik:v3.6.7") { + t.Fatalf("pinned v3.7.16: want v3.7.13 kept (previous) and v3.6.7 deleted: %v", f.rmi) + } + if strings.Contains(s, "1.3.3-stable") { + t.Fatalf("an image a container uses was deleted: %v", f.rmi) + } +} + +// While an image-pulling compose command runs (R-863), or the controller swaps itself, NOTHING is deleted. +// COMPANION RED-PROOF: drop the TryImageCleanup check → "deleted while image work was in flight". +func TestInfraRetention_NothingWhileImageWorkOrSwap(t *testing.T) { + m := retentionManager(t) + f := infraImages() + withFakeImages(t, f) + end := dockerexec.HoldImageWork() + _, err := m.RetainInfraImages(infraPins) + end() + if err != errImageRetentionBusy || len(f.rmi) != 0 { + t.Fatalf("deleted while image work was in flight: err=%v rmi=%v", err, f.rmi) + } + m.SetSelfUpdatingCheck(func() bool { return true }) + _, err = m.RetainInfraImages(infraPins) + m.SetSelfUpdatingCheck(func() bool { return false }) + if err != errImageRetentionBusy || len(f.rmi) != 0 { + t.Fatalf("deleted while the controller swaps itself: err=%v rmi=%v", err, f.rmi) + } +} + +// A pin that is not a version, and a tag that is not a version, are left alone. +func TestInfraRetention_NonVersionsLeftAlone(t *testing.T) { + m := retentionManager(t) + f := infraImages() + f.imgs = append(f.imgs, localImage{ID: "sha256:TLATEST", Repo: "traefik", Tag: "latest", Size: "190MB"}) + withFakeImages(t, f) + if _, err := m.RetainInfraImages([]string{"traefik:v3.7.17", "gtstef/filebrowser:latest"}); err != nil { + t.Fatal(err) + } + s := rmiSet(f) + if strings.Contains(s, "traefik:latest") || strings.Contains(s, "filebrowser") { + t.Fatalf("a non-version tag or a repository with a non-version pin was touched: %v", f.rmi) + } +}