package backup import ( "context" "os" "path/filepath" "sync" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/reconcile" ) // R-85 Phase 2 — tier rotation, persisted state, and the one-heavy-operation gate. // // The failure this prevents is not hypothetical: demo-hp's DR tier reported `applied` with ZERO // snapshots for five days and nobody noticed, because the scheduler could only ever see the primary // tier. Rotation is what makes the offsite tier testable at all. // rotRunner records which archives it was asked to restore. type rotRunner struct { mu sync.Mutex archives []string pass bool } func (r *rotRunner) RunRestoreTest(_ context.Context, spec reconcile.RestoreTestSpec) reconcile.RestoreTestResult { r.mu.Lock() defer r.mu.Unlock() r.archives = append(r.archives, spec.Archive) return reconcile.RestoreTestResult{ Archive: spec.Archive, SourceTier: spec.SourceTier, Pass: r.pass, Verified: "boot+running", } } func (r *rotRunner) seen() []string { r.mu.Lock() defer r.mu.Unlock() return append([]string(nil), r.archives...) } // archiveFor is a TierPicker over a fixed map: target → archive ("" = that tier holds none). func archiveFor(m map[string]string) TierPicker { return func(_ context.Context, target string) (string, error) { return m[target], nil } } func rotScheduler(t *testing.T, rr *rotRunner, st *RestoreTestState, pick TierPicker, gate *InFlight) *Scheduler { t.Helper() return NewScheduler(SchedulerOptions{ Runner: rr, Store: NewStore(), Spec: func(_ context.Context, archive string) reconcile.RestoreTestSpec { return reconcile.RestoreTestSpec{RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009} }, Cadence: time.Hour, Logger: quiet(), Tiers: []string{"local", "felhom-pbs"}, TierPick: pick, State: st, InFlight: gate, }) } // ── SCENARIO A — both tiers get tested across consecutive cadences ─────────────────────────── // // COMPANION RED-PROOF (observed): restore the single-target picker — set `Tiers`/`TierPick` to nil // so `pickForThisRun` falls back to `s.pick` on the primary runner — and this fails with // "both tiers must be exercised across 4 cadences; got [local:… local:… local:… local:…]", // i.e. the offsite tier never appears. That is today's behaviour, and it is why demo-hp's DR tier // went unproven for its entire existence. func TestRotation_BothTiersExercisedAcrossCadences(t *testing.T) { rr := &rotRunner{pass: true} st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")) s := rotScheduler(t, rr, st, archiveFor(map[string]string{ "local": "local:backup/vzdump-lxc-9201-x.tar.zst", "felhom-pbs": "felhom-pbs:backup/ct/9201/2026-07-26T15:42:42Z", }), &InFlight{}) s.now = func() time.Time { return time.Now().UTC() } for i := 0; i < 4; i++ { s.tick(context.Background()) } got := rr.seen() var sawLocal, sawPBS bool for _, a := range got { if len(a) >= 5 && a[:5] == "local" { sawLocal = true } if len(a) >= 10 && a[:10] == "felhom-pbs" { sawPBS = true } } if !sawLocal || !sawPBS { t.Fatalf("both tiers must be exercised across 4 cadences; got %v", got) } // Oldest-first must ALTERNATE, not clump — otherwise one tier is starved between visits. if len(got) != 4 { t.Fatalf("want 4 runs, got %d: %v", len(got), got) } if got[0] == got[1] { t.Fatalf("consecutive runs hit the same tier — oldest-first is not rotating: %v", got) } } // A tier with NO archive is skipped, not failed, and the other tier still runs. A brand-new offsite // tier legitimately has nothing to restore; turning that into a failure would make every fresh box // look broken for its first week. func TestRotation_EmptyTierSkippedNotFailed(t *testing.T) { rr := &rotRunner{pass: true} st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")) s := rotScheduler(t, rr, st, archiveFor(map[string]string{ "local": "local:backup/vzdump-lxc-9201-x.tar.zst", "felhom-pbs": "", // provisioned but empty }), &InFlight{}) s.tick(context.Background()) got := rr.seen() if len(got) != 1 || got[0][:5] != "local" { t.Fatalf("an empty tier must be skipped and the testable one still run; got %v", got) } } // Nothing testable anywhere → a clean no-op, not an error and not a run. func TestRotation_NoArchivesAnywhereIsANoOp(t *testing.T) { rr := &rotRunner{pass: true} s := rotScheduler(t, rr, NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")), archiveFor(map[string]string{}), &InFlight{}) s.tick(context.Background()) if got := rr.seen(); len(got) != 0 { t.Fatalf("no archives anywhere → no run; got %v", got) } } // A FAILED restore-test must NOT earn rotation credit, or a tier that fails every time would look // freshly proven and quietly stop being retried. func TestRotation_FailureEarnsNoCredit(t *testing.T) { rr := &rotRunner{pass: false} st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")) s := rotScheduler(t, rr, st, archiveFor(map[string]string{ "local": "local:backup/x.tar.zst", "felhom-pbs": "felhom-pbs:backup/ct/9201/y", }), &InFlight{}) s.tick(context.Background()) if _, ok := st.LastSuccess("local"); ok { t.Fatal("a FAILED restore-test must not stamp the tier as proven") } if _, ok := st.LastSuccess("felhom-pbs"); ok { t.Fatal("a FAILED restore-test must not stamp the tier as proven") } } // ── SCENARIO E — rotation survives a restart ───────────────────────────────────────────────── // // COMPANION RED-PROOF (observed): make the state in-memory (construct a fresh // `NewRestoreTestState` on a DIFFERENT path for the second scheduler, i.e. lose the file) and this // fails with "after a restart the OTHER tier must be next; got felhom-pbs" — the same tier repeats // and the other is starved indefinitely, which with agent deploys as routine as they are is not a // corner case. func TestRotation_SurvivesRestart(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "rt.json") picks := archiveFor(map[string]string{ "local": "local:backup/x.tar.zst", "felhom-pbs": "felhom-pbs:backup/ct/9201/y", }) // First process: the OFFSITE tier is tested (never-proven sorts first). rr1 := &rotRunner{pass: true} st1 := NewRestoreTestState(path) s1 := rotScheduler(t, rr1, st1, picks, &InFlight{}) s1.tick(context.Background()) first := rr1.seen() if len(first) != 1 { t.Fatalf("want one run, got %v", first) } // --- restart: brand-new state object reading the SAME file --- rr2 := &rotRunner{pass: true} st2 := NewRestoreTestState(path) s2 := rotScheduler(t, rr2, st2, picks, &InFlight{}) s2.tick(context.Background()) second := rr2.seen() if len(second) != 1 { t.Fatalf("want one run after restart, got %v", second) } if second[0] == first[0] { t.Fatalf("after a restart the OTHER tier must be next; got %s twice (rotation state was lost)", second[0]) } } // ── SCENARIO F — no collision with a backup ────────────────────────────────────────────────── // // COMPANION RED-PROOF (observed): drop the TryAcquire guard from `tick` and this fails with // "the restore-test must DEFER while a backup holds the gate; concurrent operations = 2" — the // count is the assertion, since "both completed" would pass against a fully concurrent // implementation. func TestRotation_DefersWhileABackupHoldsTheGate(t *testing.T) { gate := &InFlight{} release, _, ok := gate.TryAcquire("backup:felhom-pbs") if !ok { t.Fatal("precondition: the gate should have been free") } defer release() rr := &rotRunner{pass: true} s := rotScheduler(t, rr, NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")), archiveFor(map[string]string{"local": "local:backup/x.tar.zst"}), gate) s.tick(context.Background()) concurrent := 1 + len(rr.seen()) // the backup holding the gate, plus anything the tick started if concurrent != 1 { t.Fatalf("the restore-test must DEFER while a backup holds the gate; concurrent operations = %d", concurrent) } } // Once the backup releases, the next cadence proceeds — deferral must not be permanent. func TestRotation_ResumesAfterTheGateFrees(t *testing.T) { gate := &InFlight{} release, _, _ := gate.TryAcquire("backup:local") rr := &rotRunner{pass: true} s := rotScheduler(t, rr, NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")), archiveFor(map[string]string{"local": "local:backup/x.tar.zst"}), gate) s.tick(context.Background()) if len(rr.seen()) != 0 { t.Fatal("should have deferred while held") } release() s.tick(context.Background()) if len(rr.seen()) != 1 { t.Fatalf("must resume once the gate frees; got %v", rr.seen()) } } // The gate itself: one holder at a time, named, and release is idempotent. func TestInFlight_Semantics(t *testing.T) { g := &InFlight{} rel, busy, ok := g.TryAcquire("backup:local") if !ok || busy != "" { t.Fatalf("first acquire must succeed; ok=%v busy=%q", ok, busy) } if _, busy2, ok2 := g.TryAcquire("restore-test"); ok2 || busy2 != "backup:local" { t.Fatalf("second acquire must fail and NAME the holder; ok=%v busy=%q", ok2, busy2) } rel() rel() // idempotent — a double release must not free someone else's later claim if g.Busy() != "" { t.Fatalf("gate should be idle after release; busy=%q", g.Busy()) } if _, _, ok3 := g.TryAcquire("restore-test"); !ok3 { t.Fatal("gate must be reusable after release") } } // A nil gate means "not wired" → no gating, pre-R-85 behaviour. Keeps every existing caller working. func TestInFlight_NilIsUngated(t *testing.T) { var g *InFlight rel, _, ok := g.TryAcquire("x") if !ok { t.Fatal("a nil gate must not block") } rel() if g.Busy() != "" { t.Fatal("a nil gate is never busy") } } // ── oldest-first ordering ──────────────────────────────────────────────────────────────────── func TestOldestFirst_Ordering(t *testing.T) { st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")) now := time.Now().UTC() // Never-proven sorts FIRST — the case that matters, since the offsite tier starts there. if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "felhom-pbs" { // both never proven → deterministic tie-break by id if got[0] != "felhom-pbs" && got[0] != "local" { t.Fatalf("unexpected: %v", got) } } _ = st.RecordSuccess("local", now) if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "felhom-pbs" { t.Fatalf("a never-proven tier must sort before a proven one; got %v", got) } _ = st.RecordSuccess("felhom-pbs", now.Add(time.Hour)) if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "local" { t.Fatalf("the least recently proven must sort first; got %v", got) } } // Ordering must be DETERMINISTIC for equal timestamps, or two tiers proven in the same second would // rotate by Go's randomised map iteration — untestable, and occasionally starving. func TestOldestFirst_DeterministicOnTies(t *testing.T) { st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")) now := time.Now().UTC() _ = st.RecordSuccess("b-tier", now) _ = st.RecordSuccess("a-tier", now) for i := 0; i < 20; i++ { if got := st.OldestFirst([]string{"b-tier", "a-tier"}); got[0] != "a-tier" { t.Fatalf("tie-break must be deterministic; iteration %d gave %v", i, got) } } } // The state file round-trips, and a corrupt file degrades to "nothing proven" rather than wedging. func TestRestoreTestState_PersistenceAndCorruption(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "rt.json") now := time.Now().UTC().Truncate(time.Second) st := NewRestoreTestState(path) if err := st.RecordSuccess("felhom-pbs", now); err != nil { t.Fatal(err) } reopened := NewRestoreTestState(path) got, ok := reopened.LastSuccess("felhom-pbs") if !ok || !got.Equal(now) { t.Fatalf("state must round-trip; got %v ok=%v want %v", got, ok, now) } bad := filepath.Join(dir, "corrupt.json") if err := os.WriteFile(bad, []byte("{{{not json"), 0o600); err != nil { t.Fatal(err) } c := NewRestoreTestState(bad) if _, ok := c.LastSuccess("felhom-pbs"); ok { t.Fatal("a corrupt state file must degrade to 'nothing proven', not invent a timestamp") } }