diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b6b7dfd..b236743c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ **MinAgent: 0.158.1** (unchanged — nothing here asks the agent for anything) +- **Review fix 5 (2026-11-15), the box's half — version skew.** A new box with an older hub sent its end records in + the window request; that hub reads 4 KiB of it (about 55 snapshot ids), then decided the window on a wrong count, + and the weekly clean-up stalled. Now the box sends `ended` ONLY to a hub whose last window answer said + `accepts_ended` (kept in `settings.json` → `offsite_hub_accepts_ended`, re-read from every answer, so a rolled-back + hub stops getting them after one answer); otherwise the request is the one every hub always got and the ordinary + clean-up runs. One request carries at most 5000 ids (`boundEndedClaims`). **Cost, said plainly:** after the hub is + updated the box learns it at its next nightly window request and sends from the one after. Tests + `TestD84_ANewBoxWithAnOlderHub_NothingEndedIsSentAndTheCleanUpStillRuns`, `TestD84_OneRequestCarriesABoundedNumberOfIds` + — seen red before the fix; 6 mutations seen red (felhom.eu `…/A/redproof-review-5.txt`). - **Review fix 3 (2026-11-15): EVERY install deletes the off-site end record, not only a deploy.** A `.fab` import (`SaveImportedAppConfig`) installed the app and the record survived; the restore that installs from a unit (`PersistUnitRedeployConfigGate`) had the same hole. The rule now sits at ONE point: `stacks.SaveAppConfig`, the diff --git a/controller/README.md b/controller/README.md index db251611..8f0b25b5 100644 --- a/controller/README.md +++ b/controller/README.md @@ -1133,7 +1133,10 @@ deletes the record); a removed app with copies and no record shows „Törlés 3 (`POST /backup/offsite-end/start`). An install of the same app deletes the record. The copies are removed only in the weekly clean-up window, by snapshot id, for ids the hub agreed to — never without a record, never a snapshot that also belongs to an installed app, never `_shares` or the box's own stacks. A box whose off-site target is the -household's own NAS keeps a removed app's copies as before. +household's own NAS keeps a removed app's copies as before. After the review of 2026-11-15: every install deletes +the record at one point (`stacks.SaveAppConfig`: a deploy, an import, a restore that installs); the window reads the +records again immediately before the forget, so a take-back or an install while it was being prepared keeps the copies; +and the box sends its end records only to a hub whose last window answer said `accepts_ended`. **Protected stacks** (traefik, cloudflared, felhom-controller) cannot be stopped, removed, or deleted from the UI. Restart is allowed. diff --git a/controller/internal/backup/offbox_ended.go b/controller/internal/backup/offbox_ended.go index 2dd45afb..f100cfd1 100644 --- a/controller/internal/backup/offbox_ended.go +++ b/controller/internal/backup/offbox_ended.go @@ -36,6 +36,9 @@ type OffsiteEndedClaim struct { SnapshotIDs []string `json:"snapshot_ids"` } +// maxEndedClaimIDs is the most snapshot ids ONE window request carries (the hub reads as many: its maxEndedClaimIDs). +const maxEndedClaimIDs = 5000 + // OffsiteEndedRefusal is one refusal the hub sent back, in its own words (logged). type OffsiteEndedRefusal struct { App string `json:"app"` @@ -319,3 +322,21 @@ func (m *Manager) PendingOffsiteEnds() []settings.OffsiteEnd { return m.settings func (m *Manager) OffsiteEndsHistory() []settings.OffsiteEndDone { return m.settings.GetOffsiteEndsDone() } + +// boundEndedClaims cuts a request to at most limit snapshot ids in all (the hub reads no more). What is cut is asked +// again in a later window: the record stays while a copy of the app is in the store. +func boundEndedClaims(claims []OffsiteEndedClaim, limit int) []OffsiteEndedClaim { + left := limit + var out []OffsiteEndedClaim + for _, c := range claims { + if left <= 0 { + break + } + if len(c.SnapshotIDs) > left { + c.SnapshotIDs = c.SnapshotIDs[:left] + } + left -= len(c.SnapshotIDs) + out = append(out, c) + } + return out +} diff --git a/controller/internal/backup/offbox_ended_skew_test.go b/controller/internal/backup/offbox_ended_skew_test.go new file mode 100644 index 00000000..75cbefb2 --- /dev/null +++ b/controller/internal/backup/offbox_ended_skew_test.go @@ -0,0 +1,86 @@ +package backup + +import ( + "context" + "fmt" + "strings" + "testing" + "time" +) + +// Review of 2026-11-15, finding 5 — VERSION SKEW. A new box with an older hub: the window request with `ended` can +// pass that hub's 4 KiB read limit (about 55 ids); the hub then reads no count, opens the window with the smallest +// cap, and the guard refuses the honest plan every week — the ordinary clean-up stalls. The box now sends its end +// records ONLY to a hub that said, in its last window answer, that it reads them. +// RED-PROOF: felhom.eu/documentation/audits/build-release-4-2026-11-15/A/redproof-review-5.txt. +func TestD84_ANewBoxWithAnOlderHub_NothingEndedIsSentAndTheCleanUpStillRuns(t *testing.T) { + m, sett, er, hub := endedBox(t, "immich") + if err := sett.SetOffsiteHubAcceptsEnded(false); err != nil { // a box that has never heard a hub say it + t.Fatal(err) + } + hub.oldHub = true + pressed(t, sett, "vikunja", 31) + old := time.Now().Add(-60 * 24 * time.Hour) + er.plan = []guardSnap{tagged("p1", old, "immich"), tagged("p2", old.Add(time.Minute), "immich")} // the week's honest plan + er.snaps = append(er.snaps, er.plan...) + for week := 1; week <= 2; week++ { + m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") + if len(hub.claims) != 0 { + t.Fatalf("week %d: the box sent %d end record(s) to a hub that never said it reads them — that hub reads 4 KiB and would stall", week, len(hub.claims)) + } + er.plan = nil // the policy's two are gone after the first window (the fake's plan does not follow by itself) + } + // the ordinary weekly clean-up ran: the policy's two went in the first window, and nothing of the removed app + if got := er.removedIDs(); strings.Join(got, ",") != strings.Join(ids("p1", "p2"), ",") { + t.Fatalf("removed %v, want the policy's own two and nothing of the removed app", got) + } + if hub.closed[0].Outcome != "pruned" || len(hub.closed[0].RemovedForEndedApps) != 0 { + t.Fatalf("the first window: %+v", hub.closed[0]) + } + if _, ok := sett.GetOffsiteEnd("vikunja"); !ok { + t.Fatal("the record was lost while the hub could not be asked") + } + + // The hub is updated. An ordinary night's answer (not granted) says accepts_ended; the next window carries the + // end records, and the removed app's copies go. + hub.oldHub, hub.notGranted = false, true + m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") + if len(hub.claims) != 0 || !sett.GetOffsiteHubAcceptsEnded() { + t.Fatalf("the night the box LEARNS it: claims=%d learned=%v — it must send none yet and remember the answer", len(hub.claims), sett.GetOffsiteHubAcceptsEnded()) + } + hub.notGranted = false + m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") + if len(hub.claims) != 1 || hub.claims[0].App != "vikunja" { + t.Fatalf("after the hub said it reads end records the box sent %+v", hub.claims) + } + if got := er.removedIDs(); len(got) != 5 { + t.Fatalf("removed %v, want the policy's two and vikunja's three", got) + } + // … and if the hub is rolled BACK, the box stops sending at once after the first answer without the word + pressed(t, sett, "kimai", 31) + hub.oldHub = true + m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") // this one still carries claims (the box had heard yes) + m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") + if len(hub.claims) != 0 || sett.GetOffsiteHubAcceptsEnded() { + t.Fatalf("after a rolled-back hub's answer the box still sends end records: claims=%d learned=%v", len(hub.claims), sett.GetOffsiteHubAcceptsEnded()) + } +} + +// One request never carries more ids than the hub reads (5000): the rest go in a later window. +func TestD84_OneRequestCarriesABoundedNumberOfIds(t *testing.T) { + m, sett, er, hub := endedBox(t, "immich") + pressed(t, sett, "bigapp", 31) + old := time.Now().Add(-60 * 24 * time.Hour) + for i := 0; i < maxEndedClaimIDs+200; i++ { + er.snaps = append(er.snaps, tagged(fmt.Sprintf("b%05d", i), old, "bigapp")) + } + hub.notGranted = true + m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") + n := 0 + for _, c := range hub.claims { + n += len(c.SnapshotIDs) + } + if n != maxEndedClaimIDs { + t.Fatalf("one request carried %d snapshot ids, want the limit %d", n, maxEndedClaimIDs) + } +} diff --git a/controller/internal/backup/offbox_ended_test.go b/controller/internal/backup/offbox_ended_test.go index 7359f3b0..e5c92928 100644 --- a/controller/internal/backup/offbox_ended_test.go +++ b/controller/internal/backup/offbox_ended_test.go @@ -86,12 +86,18 @@ type agreeingHub struct { allowAll func() []string refuse []OffsiteEndedRefusal maxRm int + // oldHub: a hub from before ended apps — it never says accepts_ended. notGranted: the answer of an ordinary night. + oldHub bool + notGranted bool } func (h *agreeingHub) Open(ctx context.Context, n int, ended []OffsiteEndedClaim) (OffsiteWindow, error) { h.opened++ h.claims = ended - w := OffsiteWindow{Granted: true, ID: 7, NewestAllowed: time.Now(), MaxRemove: h.maxRm, EndedAllowed: h.allow, EndedRefused: h.refuse} + w := OffsiteWindow{Granted: true, ID: 7, NewestAllowed: time.Now(), MaxRemove: h.maxRm, EndedAllowed: h.allow, EndedRefused: h.refuse, AcceptsEnded: !h.oldHub} + if h.notGranted { + w = OffsiteWindow{Reason: "not due", AcceptsEnded: !h.oldHub} + } if h.allowAll != nil { w.EndedAllowed = h.allowAll() } @@ -113,6 +119,9 @@ func endedBox(t *testing.T, living ...string) (*Manager, *settings.Settings, *en tagged("k1", old, "kimai"), }} m.SetOffboxRunner(er.run) + if err := sett.SetOffsiteHubAcceptsEnded(true); err != nil { // the box has heard this hub say it knows ended apps + t.Fatal(err) + } hub := &agreeingHub{maxRm: 5} hub.allowAll = func() []string { var all []string diff --git a/controller/internal/backup/offbox_window.go b/controller/internal/backup/offbox_window.go index b55f6e6a..8b73e5e8 100644 --- a/controller/internal/backup/offbox_window.go +++ b/controller/internal/backup/offbox_window.go @@ -60,6 +60,10 @@ type OffsiteWindow struct { // MaxRemove — and what it refused. Empty from a hub that does not know ended apps: then none is removed. EndedAllowed []string EndedRefused []OffsiteEndedRefusal + // AcceptsEnded: this hub said, in this answer, that it reads `ended` in a window request (the hub's + // `accepts_ended`; review of 2026-11-15, version skew). An older hub does not say it, and then the box sends it + // no end records: that hub reads only 4 KiB of the request and would decide the window on a wrong count. + AcceptsEnded bool } // OffsiteWindowResult is what the box reports when it is done (the hub closes the window on it). @@ -238,11 +242,23 @@ func (m *Manager) offsiteWindowRetention(ctx context.Context, base, env []string due := dueOffsiteEnds(m.settings.GetOffsiteEnds(), m.now(), m.offsiteEndLiving) m.dropEndsWithNoCopies(snaps, due) claims, candidate := endedCandidates(snaps, due) + // VERSION SKEW (review of 2026-11-15): the end records go ONLY to a hub that said, in its last window answer, + // that it reads them. An older hub reads 4 KiB of the request; a larger one made it open the window on a wrong + // count, and the weekly clean-up stalled. A hub that has not said it gets the request it always got. + if len(claims) > 0 && !m.settings.GetOffsiteHubAcceptsEnded() { + m.logger.Printf("[INFO] [offbox] %d removed app(s) are due, but the hub has not said that it reads end records (an older hub) — none is sent, nothing of them is removed; the ordinary clean-up goes on", len(claims)) + claims, candidate = nil, map[string]bool{} + } + claims = boundEndedClaims(claims, maxEndedClaimIDs) w, err := m.offsiteWindow.Open(ctx, len(snaps), claims) if err != nil { m.logger.Printf("[WARN] [offbox] retention skipped (%s): asking the hub for a window failed: %v", why, err) return } + // Remember what THIS answer said (granted or not): the next request follows it. + if serr := m.settings.SetOffsiteHubAcceptsEnded(w.AcceptsEnded); serr != nil { + m.logger.Printf("[WARN] [offbox] the hub's answer about end records could not be saved: %v", serr) + } if !w.Granted { m.logger.Printf("[INFO] [offbox] retention skipped (%s): no clean-up window now (%s) — nothing deleted (decision 68)", why, w.Reason) return diff --git a/controller/internal/offsiteapply/d84_window_client_test.go b/controller/internal/offsiteapply/d84_window_client_test.go index a1b7609f..e6f4fdb1 100644 --- a/controller/internal/offsiteapply/d84_window_client_test.go +++ b/controller/internal/offsiteapply/d84_window_client_test.go @@ -40,6 +40,9 @@ func TestD84_WindowClient_SendsTheClaimsAndReadsTheAgreement(t *testing.T) { if err := json.Unmarshal(gotBody["ended"], &sent); err != nil || len(sent) != 1 || sent[0].App != "vikunja" || strings.Join(sent[0].SnapshotIDs, ",") != "aa,bb,cc" || sent[0].RemovedAt != "2026-11-15T10:00:00Z" { t.Fatalf("the hub did not receive the claim: %s (%v)", gotBody["ended"], err) } + if w.AcceptsEnded { + t.Fatal("an answer WITHOUT accepts_ended (an older hub) reads as a hub that knows end records") + } if !w.Granted || w.ID != 9 || w.MaxRemove != 10 || strings.Join(w.EndedAllowed, ",") != "aa,bb" || len(w.EndedRefused) != 1 || w.EndedRefused[0].App != "immich" { t.Fatalf("window = %+v, want the agreement (aa,bb) and the refusal of immich", w) } @@ -52,9 +55,13 @@ func TestD84_WindowClient_SendsTheClaimsAndReadsTheAgreement(t *testing.T) { t.Fatalf("a request with no claim carries `ended`: %v", gotBody) } - // A window that is NOT granted carries no agreement, whatever the answer holds. - answer = `{"granted":false,"reason":"not due","ended_allowed":["aa","bb"]}` + // A window that is NOT granted carries no agreement, whatever the answer holds — and it does carry the hub's + // word that it reads end records (the box learns it on an ordinary night). + answer = `{"granted":false,"reason":"not due","ended_allowed":["aa","bb"],"accepts_ended":true}` w2, err := c.Open(context.Background(), 20, claims) + if err == nil && !w2.AcceptsEnded { + t.Fatal("accepts_ended of a not-granted answer was not read") + } if err != nil || w2.Granted || len(w2.EndedAllowed) != 0 { t.Fatalf("not granted: %+v %v — an agreement without a window must not reach the box", w2, err) } diff --git a/controller/internal/offsiteapply/seams.go b/controller/internal/offsiteapply/seams.go index 0125d8cb..b090c9ea 100644 --- a/controller/internal/offsiteapply/seams.go +++ b/controller/internal/offsiteapply/seams.go @@ -146,11 +146,13 @@ func (c HubWindowClient) Open(ctx context.Context, countBefore int, ended []back Reason string `json:"reason"` EndedAllowed []string `json:"ended_allowed"` EndedRefused []backup.OffsiteEndedRefusal `json:"ended_refused"` + // AcceptsEnded: this hub reads `ended` (in every answer of a hub that does, granted or not). + AcceptsEnded bool `json:"accepts_ended"` } if err := json.Unmarshal(raw, &r); err != nil { return backup.OffsiteWindow{}, fmt.Errorf("hub window-open: malformed response") } - w := backup.OffsiteWindow{Granted: r.Granted, ID: r.WindowID, MaxRemove: r.MaxRemove, Reason: r.Reason} + w := backup.OffsiteWindow{Granted: r.Granted, ID: r.WindowID, MaxRemove: r.MaxRemove, Reason: r.Reason, AcceptsEnded: r.AcceptsEnded} if r.Granted { t, perr := time.Parse(time.RFC3339, r.NewestAllowed) if perr != nil { diff --git a/controller/internal/settings/offsite_ends.go b/controller/internal/settings/offsite_ends.go index 6725aea4..51b7099a 100644 --- a/controller/internal/settings/offsite_ends.go +++ b/controller/internal/settings/offsite_ends.go @@ -169,3 +169,22 @@ func (s *Settings) GetOffsiteEndsDone() []OffsiteEndDone { defer s.mu.RUnlock() return append([]OffsiteEndDone(nil), s.OffsiteEndsDone...) } + +// OffsiteHubAcceptsEnded: did the hub's LAST window answer say that it reads end records (`accepts_ended`)? Kept in +// settings so a controller restart does not cost a week: the box sends its end records only to a hub that said so. +func (s *Settings) GetOffsiteHubAcceptsEnded() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.OffsiteHubAcceptsEnded +} + +// SetOffsiteHubAcceptsEnded records the hub's last answer; it writes only on a change. +func (s *Settings) SetOffsiteHubAcceptsEnded(v bool) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.OffsiteHubAcceptsEnded == v { + return nil + } + s.OffsiteHubAcceptsEnded = v + return s.save() +} diff --git a/controller/internal/settings/settings.go b/controller/internal/settings/settings.go index 19534a59..cfa11a93 100644 --- a/controller/internal/settings/settings.go +++ b/controller/internal/settings/settings.go @@ -233,6 +233,8 @@ type Settings struct { // OffsiteEndsDone is the short history of ends a clean-up window has carried out. See offsite_ends.go. OffsiteEnds map[string]OffsiteEnd `json:"offsite_ends,omitempty"` OffsiteEndsDone []OffsiteEndDone `json:"offsite_ends_done,omitempty"` + // OffsiteHubAcceptsEnded: the hub's last window answer said it reads end records (offsite_ends.go). + OffsiteHubAcceptsEnded bool `json:"offsite_hub_accepts_ended,omitempty"` // UnhealthyStops (v0.269.0, decision 28) — when the box last stopped each app for a crash loop or an // out-of-memory storm (RFC3339). Survives the hold being lifted by Start, so a second stop within // 24 h is known to be a second one.