diff --git a/CHANGELOG.md b/CHANGELOG.md index a3d0d342..9d59e1a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ **MinAgent: 0.158.1** (unchanged — nothing here asks the agent for anything) +- **Review fix 2 (2026-11-15): a take-back or an install DURING the clean-up window no longer loses the copies.** The + due records and the candidates were read once, before the hub was asked, and never again before the forget — so + „Mégis megtartom” (or an install) while the hub answered still deleted, after the page had said „kept”. The window + now reads the records again immediately before the forget (`Manager.lastLookAtEnded`) and takes out every id whose + record is gone, changed or no longer due. From that look until the forget has ended, a take-back for one of those + apps is answered „not marked, or already deleted”, never „kept” (`KeepOffsiteCopies`, one lock with the look). + **Not covered, said plainly:** an install of the app during the forget itself (restic is already removing). + Tests `TestD72_ATakeBackDuringTheWindowKeepsTheCopies`, `TestD72_AnInstallDuringTheWindowKeepsTheCopies` (three + ways), `TestD72_ATakeBackDuringTheForgetIsNotAnsweredKept` — the first two seen red on the pushed code; 3 mutations + seen red, one not provable and said so (felhom.eu `…/build-release-4-2026-11-15/A/redproof-review-2.txt`). - **The off-site copies of a removed app end 30 days after „delete backup data” — the box's half (R-1374 part 1; D72 and D84, `09` §3 decisions 292 and 309; felhom.eu `07-backup-architecture.md` §8.2f). Built, NOT released; no box has run it. It does nothing without a hub that knows it (felhom.eu `hub/CHANGELOG.md`, Unreleased): an older hub diff --git a/controller/internal/backup/backup.go b/controller/internal/backup/backup.go index fef79f36..9609199b 100644 --- a/controller/internal/backup/backup.go +++ b/controller/internal/backup/backup.go @@ -132,6 +132,10 @@ type Manager struct { // offsiteEndNotify (D84) tells the household ONCE per press that an app's off-site copies are marked for // deletion. Wired in main.go to the notifier; nil-safe. Called only when a record was really written. offsiteEndNotify func(rec settings.OffsiteEnd) + // offsiteEndMu guards offsiteEndRemoving: the apps whose off-site copies a clean-up window is removing RIGHT NOW + // (from its last look at the records until its forget has ended). A take-back for such an app is not „kept”. + offsiteEndMu sync.Mutex + offsiteEndRemoving map[string]bool // offsiteAbandon (v0.290.0, decision 74) hands a due set-aside deletion to the hub. offsiteAbandon OffsiteAbandonClient diff --git a/controller/internal/backup/offbox_ended.go b/controller/internal/backup/offbox_ended.go index 6fd6ae23..2dd45afb 100644 --- a/controller/internal/backup/offbox_ended.go +++ b/controller/internal/backup/offbox_ended.go @@ -245,11 +245,75 @@ func (m *Manager) StartOffsiteEnd(app string) (rec settings.OffsiteEnd, created return rec, true, nil } -// KeepOffsiteCopies is „keep it after all”: it deletes the end record. Returns whether there was one. +// KeepOffsiteCopies is „keep it after all”: it deletes the end record. Returns whether there was one TO KEEP: false +// also while a clean-up window is removing this app's copies right now (its last look at the records is behind it) — +// the page then says „not marked, or already deleted”, never „kept” over copies that are going. func (m *Manager) KeepOffsiteCopies(app string) (bool, error) { + m.offsiteEndMu.Lock() + defer m.offsiteEndMu.Unlock() + if m.offsiteEndRemoving[app] { + m.logger.Printf("[WARN] [offbox] keep-after-all for %s came while the clean-up window is removing its off-site copies — too late, not kept", app) + return false, nil + } return m.settings.CancelOffsiteEnd(app, "the household keeps the copies") } +// lastLookAtEnded reads the end records AGAIN, immediately before the forget, and keeps of the agreed ids only those +// whose EVERY app still has the SAME record (app and removed_at), still due, still not installed. It returns the ids +// and the claims that are left, and marks those apps as „being removed” until endedRemovalOver. One lock with +// KeepOffsiteCopies, so a take-back is either before this look (its record is gone: its ids are taken out here) or +// after it (answered „too late”). +// +// NOT covered, said plainly: an INSTALL of the app during the forget itself. It deletes the record through settings +// (stacks.SaveAppConfig), restic is already removing, and the copies go. The window is one `forget … --prune`. +func (m *Manager) lastLookAtEnded(snaps []guardSnap, claims []OffsiteEndedClaim, endedIDs []string) ([]string, []OffsiteEndedClaim) { + m.offsiteEndMu.Lock() + defer m.offsiteEndMu.Unlock() + m.offsiteEndRemoving = nil + if len(endedIDs) == 0 { + return nil, claims + } + due := dueOffsiteEnds(m.settings.GetOffsiteEnds(), m.now(), m.offsiteEndLiving) + claimed := make(map[string]string, len(claims)) + for _, c := range claims { + claimed[c.App] = c.RemovedAt + } + for app, e := range due { + if at, ok := claimed[app]; !ok || at != e.RemovedAt { + delete(due, app) // a record this window did not claim, or one written anew since: not this window's + } + } + var left []OffsiteEndedClaim + for _, c := range claims { + if _, ok := due[c.App]; ok { + left = append(left, c) + } else { + m.logger.Printf("[WARN] [offbox] ended app %s: its end record changed while the window was being prepared (taken back, installed again, or written anew) — nothing of it is removed", c.App) + } + } + _, candidate := endedCandidates(snaps, due) + var ids []string + for _, id := range endedIDs { + if candidate[id] { + ids = append(ids, id) + } + } + if len(ids) > 0 { + m.offsiteEndRemoving = map[string]bool{} + for app := range due { + m.offsiteEndRemoving[app] = true + } + } + return ids, left +} + +// endedRemovalOver ends the „being removed” mark (the forget has ended, whatever its result). +func (m *Manager) endedRemovalOver() { + m.offsiteEndMu.Lock() + m.offsiteEndRemoving = nil + m.offsiteEndMu.Unlock() +} + // PendingOffsiteEnds and OffsiteEndsHistory are what the Backups page shows. func (m *Manager) PendingOffsiteEnds() []settings.OffsiteEnd { return m.settings.GetOffsiteEnds() } func (m *Manager) OffsiteEndsHistory() []settings.OffsiteEndDone { diff --git a/controller/internal/backup/offbox_ended_race_test.go b/controller/internal/backup/offbox_ended_race_test.go new file mode 100644 index 00000000..5f1617d5 --- /dev/null +++ b/controller/internal/backup/offbox_ended_race_test.go @@ -0,0 +1,135 @@ +package backup + +import ( + "context" + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-controller/internal/settings" +) + +// Review of 2026-11-15, finding 2: the due records and the candidates were read ONCE, before the hub was asked, and +// never again before the forget. A take-back („Mégis megtartom”) or an install in that interval — the hub's answer, +// the dry run, the guard: seconds to minutes — still lost the copies, while the page had just said „the off-site +// copies are kept”. The window now reads the records again IMMEDIATELY before the forget. +// RED-PROOF: felhom.eu/documentation/audits/build-release-4-2026-11-15/A/redproof-review-2.txt. + +// racingHub runs `during` while the hub answers — after the box has sent its claims, before it removes anything. +type racingHub struct { + *agreeingHub + during func() +} + +func (h *racingHub) Open(ctx context.Context, n int, ended []OffsiteEndedClaim) (OffsiteWindow, error) { + w, err := h.agreeingHub.Open(ctx, n, ended) + h.during() + return w, err +} + +func TestD72_ATakeBackDuringTheWindowKeepsTheCopies(t *testing.T) { + m, sett, er, hub := endedBox(t, "immich") + pressed(t, sett, "kimai", 31) + pressed(t, sett, "vikunja", 31) + m.SetOffsiteWindowClient(&racingHub{agreeingHub: hub, during: func() { + if kept, err := m.KeepOffsiteCopies("kimai"); err != nil || !kept { + t.Fatalf("keep: %v %v", kept, err) + } + }}) + m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") + if len(hub.claims) != 2 { + t.Fatalf("fixture: both apps must have been claimed before the take-back, got %+v", hub.claims) + } + if _, ok := sett.GetOffsiteEnd("kimai"); ok { + t.Fatal("fixture: the take-back did not delete the record") + } + got := er.removedIDs() + for _, id := range got { + if id == "k1-full" { + t.Fatalf("the page said „the off-site copies are kept”, no record exists, and the window removed kimai's copy (%v)", got) + } + } + // the OTHER app's end is not disturbed by the take-back + if strings.Join(got, ",") != strings.Join(ids("v1", "v2", "v3"), ",") { + t.Fatalf("removed %v, want vikunja's three and nothing of kimai", got) + } + if r := hub.closed[0].RemovedForEndedApps; strings.Join(r, ",") != strings.Join(ids("v1", "v2", "v3"), ",") { + t.Fatalf("removed_for_ended_apps = %v", r) + } +} + +// The same interval, an INSTALL: the record is deleted by the install (stacks) and the app is living again. +func TestD72_AnInstallDuringTheWindowKeepsTheCopies(t *testing.T) { + for _, how := range []string{"the install deleted the record", "the app is living again (the record is still there)", "removed again: a NEW record with a new time"} { + t.Run(how, func(t *testing.T) { + m, sett, er, hub := endedBox(t, "immich") + pressed(t, sett, "kimai", 31) + live := map[string]bool{"immich": true} + m.SetOffsiteEndLiving(func(app string) bool { return live[app] }) + m.SetOffsiteWindowClient(&racingHub{agreeingHub: hub, during: func() { + switch how { + case "the install deleted the record": + if _, err := sett.CancelOffsiteEnd("kimai", "the app is installed again"); err != nil { + t.Fatal(err) + } + live["kimai"] = true + case "the app is living again (the record is still there)": + live["kimai"] = true + default: // installed, removed again with „delete backup data”: the 30 days start again + if _, err := sett.CancelOffsiteEnd("kimai", "the app is installed again"); err != nil { + t.Fatal(err) + } + if _, _, err := sett.StartOffsiteEnd("kimai", m.now()); err != nil { + t.Fatal(err) + } + } + }}) + m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") + if len(hub.claims) != 1 { + t.Fatalf("fixture: kimai must have been claimed first: %+v", hub.claims) + } + if got := er.removedIDs(); len(got) != 0 { + t.Fatalf("the window removed %v of an app that was installed again while the hub answered", got) + } + if len(sett.GetOffsiteEndsDone()) != 0 { + t.Fatalf("a history line for an end that did not happen: %+v", sett.GetOffsiteEndsDone()) + } + }) + } +} + +// A take-back that comes while the forget itself runs is TOO LATE, and the page must not say „kept”: the answer is +// „nothing to keep”, the record stays for the window to close, and the copies go as the window had decided. +func TestD72_ATakeBackDuringTheForgetIsNotAnsweredKept(t *testing.T) { + m, sett, er, hub := endedBox(t, "immich") + pressed(t, sett, "kimai", 31) + base := er.run + var keptAnswer, asked bool + m.SetOffboxRunner(func(ctx context.Context, env []string, args ...string) ([]byte, error) { + if contains(args, "forget") && !contains(args, "--dry-run") { + k, err := m.KeepOffsiteCopies("kimai") + if err != nil { + t.Fatal(err) + } + keptAnswer, asked = k, true + } + return base(ctx, env, args...) + }) + m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") + if !asked || keptAnswer { + t.Fatalf("asked=%v kept=%v — a take-back during the forget was answered „kept” while the copies were being removed", asked, keptAnswer) + } + if got := er.removedIDs(); strings.Join(got, ",") != strings.Join(ids("k1"), ",") { + t.Fatalf("removed %v", got) + } + if _, ok := sett.GetOffsiteEnd("kimai"); ok || len(sett.GetOffsiteEndsDone()) != 1 { + t.Fatalf("the end must be closed as carried out: pending=%v history=%+v", ok, sett.GetOffsiteEndsDone()) + } + // after the window the mark is gone: a later record can be taken back again + pressed(t, sett, "vikunja", 1) + if k, err := m.KeepOffsiteCopies("vikunja"); err != nil || !k { + t.Fatalf("after the window a take-back must work again: %v %v", k, err) + } + _ = hub +} + +var _ = settings.OffsiteEndAfter diff --git a/controller/internal/backup/offbox_window.go b/controller/internal/backup/offbox_window.go index fb1d6b69..b55f6e6a 100644 --- a/controller/internal/backup/offbox_window.go +++ b/controller/internal/backup/offbox_window.go @@ -276,6 +276,13 @@ func (m *Manager) offsiteWindowRetention(ctx context.Context, base, env []string if len(endedIDs) > 0 { m.logger.Printf("[INFO] [offbox] clean-up window %d: %d snapshot(s) of removed apps are due and agreed by the hub — removed beside the policy's %d (D84)", w.ID, len(endedIDs), len(ids)) } + // THE LAST LOOK (review of 2026-11-15). The records were read before the hub was asked; the hub's answer, the dry + // run and the guard took seconds to minutes. A take-back („Mégis megtartom”) or an install in that time deleted + // the record — so the records are read AGAIN here, immediately before the forget, and an id whose record is gone, + // changed or no longer due is taken out. From here until the forget has ended, a take-back for one of these apps + // is answered „not marked, or already deleted” (KeepOffsiteCopies), never „kept”. + endedIDs, claims = m.lastLookAtEnded(snaps, claims, endedIDs) + defer m.endedRemovalOver() ids = append(ids, endedIDs...) if len(ids) == 0 { res.Outcome = "nothing"