package backup import ( "context" "errors" "fmt" "os" "path/filepath" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/reconcile" ) // R-86 — the restore-test follows the BACKUP, not the clock. // // Every test here DRIVES time (`s.now` is injected and stepped) rather than waiting for it. A test // that slept could not say anything about a 24-hour rule in under 24 hours, and one that only // asserted "no error" would pass against a scheduler that never ran anything at all — which is // precisely the failure mode §8.1's trap produces. So the assertions are: did a test run, on WHICH // archive, and did a second evaluation correctly run NOTHING. // ── the fake tier storage ──────────────────────────────────────────────────────────────────── // archiveStub is one archive on a tier: its volid and when it landed. type archiveStub struct { volid string landed time.Time } // tierStorage is a TierPicker over per-tier archive lists. It implements the SAME contract as the // production picker (*BackupRunner).PickSettledRestoreCandidateOn — newest archive that landed at // or before the cutoff — which is itself covered against a fake PVE API in backup_test.go, and // end-to-end by the live run. Naming the seam explicitly: everything below is true up to this // picker; that the real picker obeys the same rule is asserted there, not here. type tierStorage struct { archives map[string][]archiveStub err map[string]error // target → lookup failure } func (ts *tierStorage) pick(_ context.Context, target string, notAfter time.Time) (string, time.Time, error) { if e, ok := ts.err[target]; ok && e != nil { return "", time.Time{}, e } var best archiveStub for _, a := range ts.archives[target] { if !notAfter.IsZero() && a.landed.After(notAfter) { continue // not settled yet } if best.volid == "" || a.landed.After(best.landed) { best = a } } return best.volid, best.landed, nil } // dueHarness is a scheduler with a driven clock over a fake tier storage. type dueHarness struct { s *Scheduler rr *rotRunner st *RestoreTestState ts *tierStorage clock time.Time path string } func newDueHarness(t *testing.T, start time.Time, settle time.Duration, pass bool, tiers []string, ts *tierStorage) *dueHarness { t.Helper() return newDueHarnessAt(t, filepath.Join(t.TempDir(), "rt.json"), start, settle, pass, tiers, ts) } func newDueHarnessAt(t *testing.T, statePath string, start time.Time, settle time.Duration, pass bool, tiers []string, ts *tierStorage) *dueHarness { t.Helper() h := &dueHarness{rr: &rotRunner{pass: pass}, ts: ts, clock: start, path: statePath} h.st = NewRestoreTestState(statePath) h.s = NewScheduler(SchedulerOptions{ Runner: h.rr, Store: NewStore(), Spec: func(_ context.Context, archive string) reconcile.RestoreTestSpec { return reconcile.RestoreTestSpec{RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009} }, Cadence: time.Hour, Settle: settle, Logger: quiet(), Tiers: tiers, TierPick: ts.pick, State: h.st, InFlight: &InFlight{}, }) h.s.now = func() time.Time { return h.clock } return h } // advance steps the clock by step, evaluating once at every step — the scheduler's real shape. func (h *dueHarness) advance(step, total time.Duration) { for elapsed := time.Duration(0); elapsed < total; elapsed += step { h.clock = h.clock.Add(step) h.s.tick(context.Background()) } } var day0 = time.Date(2026, 8, 1, 2, 0, 0, 0, time.UTC) // dailyArchives lands one archive a day at 02:00 for n days, starting at day0. func dailyArchives(tier string, n int) []archiveStub { out := make([]archiveStub, 0, n) for d := 0; d < n; d++ { out = append(out, archiveStub{ volid: fmt.Sprintf("%s:backup/vzdump-lxc-9201-day%d.tar.zst", tier, d), landed: day0.AddDate(0, 0, d), }) } return out } // ── SCENARIO A — a daily tier is proved daily, on its own archive ──────────────────────────── // // THE TRAP THIS PINS (§8.1). R-86 reads "trigger a tier ~24 h after its own newest archive", and // the literal implementation of that — *due when the newest archive is at least `settle` old* — is // NEVER true on a daily tier: a new archive lands every day, so the newest archive's age resets to // zero long before it reaches 24 h. The literal reading silently switches restore-testing OFF for // the tier that matters most. // // COMPANION RED-PROOF (observed 2026-08-03). In Scheduler.evaluateTier, the per-archive comparison // was replaced by the naive age rule: // // - if ok && proven == archive { … not due … } // + if s.now().Sub(landed) < s.settle { … not due … } // and the proven-archive check deleted // // and the picker cutoff was removed (`cutoff := time.Time{}`), i.e. exactly "is the newest archive // old enough". Result: // // --- FAIL: TestDue_DailyTierIsProvedDailyOnItsOwnArchive // restoretest_due_test.go: a daily tier must be proved once per day; got 0 run(s) over 5 days // // Zero runs — restore-testing off. Restored immediately afterwards. func TestDue_DailyTierIsProvedDailyOnItsOwnArchive(t *testing.T) { ts := &tierStorage{archives: map[string][]archiveStub{"local": dailyArchives("local", 6)}} h := newDueHarness(t, day0.Add(time.Hour), 24*time.Hour, true, []string{"local"}, ts) // Five days, evaluated hourly. h.advance(time.Hour, 5*24*time.Hour) got := h.rr.seen() if len(got) != 5 { t.Fatalf("a daily tier must be proved once per day; got %d run(s) over 5 days: %v", len(got), got) } // And each run must be on the archive that settled that day — day0's on day 1, and so on. for i, a := range got { want := fmt.Sprintf("local:backup/vzdump-lxc-9201-day%d.tar.zst", i) if a != want { t.Fatalf("run %d tested %q, want %q — the test is not following the archive", i+1, a, want) } } // The newest archive is NEVER the one tested: it has not settled. if last := got[len(got)-1]; last == "local:backup/vzdump-lxc-9201-day5.tar.zst" { t.Fatal("the still-settling archive was tested — the settle cutoff is not being applied") } } // ── SCENARIO B — a weekly tier is proved weekly, not every other day ───────────────────────── func TestDue_WeeklyTierIsProvedOncePerArchive(t *testing.T) { ts := &tierStorage{archives: map[string][]archiveStub{"felhom-pbs": { {volid: "felhom-pbs:backup/ct/9201/w0", landed: day0}, {volid: "felhom-pbs:backup/ct/9201/w1", landed: day0.AddDate(0, 0, 7)}, {volid: "felhom-pbs:backup/ct/9201/w2", landed: day0.AddDate(0, 0, 14)}, }}} h := newDueHarness(t, day0.Add(time.Hour), 24*time.Hour, true, []string{"felhom-pbs"}, ts) // Three weeks, evaluated every 6 hours — 84 evaluations. h.advance(6*time.Hour, 21*24*time.Hour) got := h.rr.seen() want := []string{ "felhom-pbs:backup/ct/9201/w0", "felhom-pbs:backup/ct/9201/w1", "felhom-pbs:backup/ct/9201/w2", } if len(got) != len(want) { t.Fatalf("a weekly tier must be proved ONCE PER ARCHIVE (3 archives over 3 weeks); got %d run(s): %v", len(got), got) } for i := range want { if got[i] != want[i] { t.Fatalf("run %d tested %q, want %q", i+1, got[i], want[i]) } } } // ── SCENARIO C — an agent restart does not change the schedule ─────────────────────────────── // // This is the defect a person actually notices: today every deploy restarts the ticker, so a // restore-test runs one interval after each deploy regardless of what has already been proven. // // COMPANION RED-PROOF (observed 2026-08-03): revert the state to per-tier TIME by making // ProvenArchive ignore the stored archive — // // - if !ok || p.Archive == "" { return "", false } // + return "", false // per-tier time only, the pre-R-86 state // // → --- FAIL: TestDue_RestartRunsNothing // restoretest_due_test.go:226: an agent restart must not trigger a restore-test; 2 restart(s) // produced 4 run(s) // // Four: the same already-proven archive re-tested on EVERY evaluation after EVERY restart, which is // today's behaviour with the ticker's phase reset by the deploy. Restored. func TestDue_RestartRunsNothing(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "rt.json") ts := &tierStorage{archives: map[string][]archiveStub{"local": dailyArchives("local", 2)}} start := day0.AddDate(0, 0, 1).Add(time.Hour) // day 1, 03:00 — day0's archive has settled h := newDueHarnessAt(t, path, start, 24*time.Hour, true, []string{"local"}, ts) h.s.tick(context.Background()) if n := len(h.rr.seen()); n != 1 { t.Fatalf("precondition: the settled archive should have been proved once; got %d run(s)", n) } // --- two restarts: brand-new scheduler + brand-new state object over the SAME file --- total := 0 for i := 0; i < 2; i++ { h2 := newDueHarnessAt(t, path, start.Add(time.Duration(i+1)*time.Hour), 24*time.Hour, true, []string{"local"}, ts) h2.s.tick(context.Background()) h2.s.tick(context.Background()) total += len(h2.rr.seen()) } if total != 0 { t.Fatalf("an agent restart must not trigger a restore-test; 2 restart(s) produced %d run(s)", total) } } // ── SCENARIO D — a new archive makes a tier due even if it was tested yesterday ────────────── func TestDue_NewSettledArchiveMakesAProvedTierDueAgain(t *testing.T) { ts := &tierStorage{archives: map[string][]archiveStub{"local": dailyArchives("local", 2)}} h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(time.Hour), 24*time.Hour, true, []string{"local"}, ts) h.s.tick(context.Background()) // proves day0's archive h.s.tick(context.Background()) // nothing new has settled → nothing if n := len(h.rr.seen()); n != 1 { t.Fatalf("want exactly 1 run before the new archive settles, got %d: %v", n, h.rr.seen()) } // Day 2, 03:00 — day1's archive has now settled. h.clock = day0.AddDate(0, 0, 2).Add(time.Hour) h.s.tick(context.Background()) got := h.rr.seen() if len(got) != 2 { t.Fatalf("a newly settled archive must make the tier due again; got %v", got) } if got[1] != "local:backup/vzdump-lxc-9201-day1.tar.zst" { t.Fatalf("the NEW archive must be the one tested; got %q", got[1]) } } // ── SCENARIO E — a failing tier keeps being retried, and earns no proof ────────────────────── // // COMPANION RED-PROOF (observed 2026-08-03): give credit on failure in Scheduler.tick — // // - if rt.Pass && s.rtState != nil && target != "" { // + if s.rtState != nil && target != "" { // // → --- FAIL: TestDue_FailingTierIsRetriedAndNeverProven // restoretest_due_test.go: a failing tier must keep being retried; got 1 run(s) over 3 // evaluations // // A single failure would have retired the archive as proven — a permanently broken DR tier looking // freshly verified, which is the loudest signal this system produces going silent. Restored. func TestDue_FailingTierIsRetriedAndNeverProven(t *testing.T) { ts := &tierStorage{archives: map[string][]archiveStub{"local": dailyArchives("local", 1)}} h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(time.Hour), 24*time.Hour, false, []string{"local"}, ts) for i := 0; i < 3; i++ { h.s.tick(context.Background()) } got := h.rr.seen() if len(got) != 3 { t.Fatalf("a failing tier must keep being retried; got %d run(s) over 3 evaluations: %v", len(got), got) } if _, ok := h.st.ProvenArchive("local"); ok { t.Fatal("a FAILED restore-test must not record the archive as proven") } if _, ok := h.st.LastSuccess("local"); ok { t.Fatal("a FAILED restore-test must not stamp the tier as proven") } } // ── SCENARIO F — two tiers due at once do not run at once ──────────────────────────────────── func TestDue_TwoDueTiersRunOneAtATime(t *testing.T) { ts := &tierStorage{archives: map[string][]archiveStub{ "local": {{volid: "local:backup/a.tar.zst", landed: day0}}, "felhom-pbs": {{volid: "felhom-pbs:backup/ct/9201/a", landed: day0}}, }} h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(time.Hour), 24*time.Hour, true, []string{"local", "felhom-pbs"}, ts) // Both tiers are due at this instant. due := h.s.EvaluateDue(context.Background()) if len(due) != 2 || !due[0].Due || !due[1].Due { t.Fatalf("precondition: both tiers should be due; got %v", due) } h.s.tick(context.Background()) if n := len(h.rr.seen()); n != 1 { t.Fatalf("ONE evaluation must start ONE restore-test, never two multi-GB restores over one link; got %d: %v", n, h.rr.seen()) } // The other tier was DEFERRED, not cancelled: it is still due and runs on the next evaluation. h.s.tick(context.Background()) got := h.rr.seen() if len(got) != 2 || got[0] == got[1] { t.Fatalf("the deferred tier must run on the NEXT evaluation, on its own archive; got %v", got) } } // The heavy-operation gate still holds, and a tier deferred behind a backup stays DUE. func TestDue_DeferredBehindABackupStaysDue(t *testing.T) { ts := &tierStorage{archives: map[string][]archiveStub{"local": {{volid: "local:backup/a.tar.zst", landed: day0}}}} h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(time.Hour), 24*time.Hour, true, []string{"local"}, ts) gate := &InFlight{} h.s.inFlight = gate release, _, _ := gate.TryAcquire("backup:felhom-pbs") h.s.tick(context.Background()) if n := len(h.rr.seen()); n != 0 { t.Fatalf("the restore-test must DEFER while a backup holds the gate; got %d run(s)", n) } if due := h.s.EvaluateDue(context.Background()); !due[0].Due { t.Fatal("a deferred tier must remain DUE — deferral is not dismissal") } release() h.s.tick(context.Background()) if n := len(h.rr.seen()); n != 1 { t.Fatalf("must resume once the gate frees; got %d run(s)", n) } } // ── SCENARIO H — a newborn box is UNKNOWN, not stale and not a fault ───────────────────────── func TestDue_NewbornTierIsNotDueAndNotAnError(t *testing.T) { ts := &tierStorage{archives: map[string][]archiveStub{"felhom-pbs": nil}} h := newDueHarness(t, day0, 24*time.Hour, true, []string{"felhom-pbs"}, ts) due := h.s.EvaluateDue(context.Background()) if len(due) != 1 { t.Fatalf("want one verdict, got %v", due) } v := due[0] if v.Due || v.Err != nil || v.Archive != "" { t.Fatalf("a tier with no archive is UNKNOWN — not due, not an error; got %+v", v) } if v.Reason == "" { t.Fatal("every verdict must carry a reason — a due-check that cannot say why is a quiet path") } h.s.tick(context.Background()) if n := len(h.rr.seen()); n != 0 { t.Fatalf("a newborn tier must not be restore-tested; got %d run(s)", n) } } // An archive that exists but has NOT settled yet is not a candidate — and that is not an error. func TestDue_UnsettledArchiveIsNotACandidate(t *testing.T) { ts := &tierStorage{archives: map[string][]archiveStub{"local": {{volid: "local:backup/fresh.tar.zst", landed: day0}}}} h := newDueHarness(t, day0.Add(2*time.Hour), 24*time.Hour, true, []string{"local"}, ts) if v := h.s.EvaluateDue(context.Background())[0]; v.Due || v.Archive != "" { t.Fatalf("an archive 2h old must not be a candidate under a 24h settle lag; got %+v", v) } h.s.tick(context.Background()) if n := len(h.rr.seen()); n != 0 { t.Fatalf("nothing settled → no run; got %d", n) } } // A tier whose archives cannot be LISTED is UNKNOWN — never silently "not due", and never silent. // Treating a lookup failure as "not due" would retire a tier the moment its storage stopped // answering, which is the same absence-is-not-evidence error this monitor family keeps making. func TestDue_LookupFailureIsUnknownNotNotDue(t *testing.T) { boom := errors.New("storage unreachable") ts := &tierStorage{ archives: map[string][]archiveStub{"local": {{volid: "local:backup/a.tar.zst", landed: day0}}}, err: map[string]error{"felhom-pbs": boom}, } h := newDueHarness(t, day0.AddDate(0, 0, 1), 24*time.Hour, true, []string{"local", "felhom-pbs"}, ts) var pbs DueVerdict for _, v := range h.s.EvaluateDue(context.Background()) { if v.Target == "felhom-pbs" { pbs = v } } if pbs.Err == nil { t.Fatal("a lookup failure must travel in the verdict, not be swallowed") } if pbs.Due { t.Fatal("a tier we could not list must not be reported DUE — we have no archive to test") } if pbs.Reason == "" { t.Fatal("the failure must be explained, not merely flagged") } // And the OTHER tier still runs: one tier's storage being unreadable must not cost the other // tier its proof. h.s.tick(context.Background()) if got := h.rr.seen(); len(got) != 1 || got[0] != "local:backup/a.tar.zst" { t.Fatalf("the readable tier must still be proved; got %v", got) } } // ── the state's migration (§8.2) ───────────────────────────────────────────────────────────── // A pre-R-86 state file carries a TIME and no archive. It must keep its time (rotation ordering // survives the upgrade) and yield NO proven archive, so each tier is due exactly once. Reading a // legacy time as proof of the CURRENT archive would mark an unproven archive proven — a guarantee // invented by a migration. func TestRestoreTestState_LegacyFileMigratesToNothingProven(t *testing.T) { path := filepath.Join(t.TempDir(), "rt.json") legacy := `{"local":"2026-08-01T02:00:00Z","felhom-pbs":"2026-07-30T02:00:00Z"}` if err := writeFileForTest(path, legacy); err != nil { t.Fatal(err) } st := NewRestoreTestState(path) if _, ok := st.ProvenArchive("local"); ok { t.Fatal("a legacy record names no archive — it must NOT be read as proof of the current one") } at, ok := st.LastSuccess("local") if !ok || !at.Equal(time.Date(2026, 8, 1, 2, 0, 0, 0, time.UTC)) { t.Fatalf("the legacy TIME must survive (rotation ordering depends on it); got %v ok=%v", at, ok) } // Ordering still works off the legacy times. if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "felhom-pbs" { t.Fatalf("oldest-first must still order legacy records; got %v", got) } } // The new shape round-trips, archive and all. func TestRestoreTestState_ArchiveRoundTrips(t *testing.T) { path := filepath.Join(t.TempDir(), "rt.json") now := time.Now().UTC().Truncate(time.Second) st := NewRestoreTestState(path) if err := st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/x", now); err != nil { t.Fatal(err) } re := NewRestoreTestState(path) got, ok := re.ProvenArchive("felhom-pbs") if !ok || got != "felhom-pbs:backup/ct/9201/x" { t.Fatalf("the proven ARCHIVE must survive a restart; got %q ok=%v", got, ok) } at, ok := re.LastSuccess("felhom-pbs") if !ok || !at.Equal(now) { t.Fatalf("the proven TIME must survive too; got %v ok=%v", at, ok) } } // writeFileForTest is a tiny helper so the legacy-migration fixture reads clearly above. func writeFileForTest(path, content string) error { return os.WriteFile(path, []byte(content), 0o600) }