felhom-agent/internal/backup/rotation_test.go

package backup

import (
	"context"
	"os"
	"path/filepath"
	"sync"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
)

// R-85 Phase 2 — tier rotation, persisted state, and the one-heavy-operation gate.
//
// The failure this prevents is not hypothetical: demo-hp's DR tier reported `applied` with ZERO
// snapshots for five days and nobody noticed, because the scheduler could only ever see the primary
// tier. Rotation is what makes the offsite tier testable at all.

// rotRunner records which archives it was asked to restore.
type rotRunner struct {
	mu       sync.Mutex
	archives []string
	pass     bool
}

func (r *rotRunner) RunRestoreTest(_ context.Context, spec reconcile.RestoreTestSpec) reconcile.RestoreTestResult {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.archives = append(r.archives, spec.Archive)
	return reconcile.RestoreTestResult{
		Archive: spec.Archive, SourceTier: spec.SourceTier,
		Pass: r.pass, Verified: "boot+running",
	}
}
func (r *rotRunner) seen() []string {
	r.mu.Lock()
	defer r.mu.Unlock()
	return append([]string(nil), r.archives...)
}

// testLanded is a landing time old enough to be settled under any cutoff these tests use. R-86
// widened the TierPicker seam with the archive's landing time; the rotation tests below are about
// tier ORDER and the heavy-operation gate, not about settling, so they hold it constant.
var testLanded = time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)

// archiveFor is a TierPicker over a fixed map: target → archive ("" = that tier holds none).
func archiveFor(m map[string]string) TierPicker {
	return func(_ context.Context, target string, _ time.Time) (string, time.Time, error) {
		a := m[target]
		if a == "" {
			return "", time.Time{}, nil
		}
		return a, testLanded, nil
	}
}

func rotScheduler(t *testing.T, rr *rotRunner, st *RestoreTestState, pick TierPicker, gate *InFlight) *Scheduler {
	t.Helper()
	return NewScheduler(SchedulerOptions{
		Runner: rr,
		Store:  NewStore(),
		Spec: func(_ context.Context, archive string) reconcile.RestoreTestSpec {
			return reconcile.RestoreTestSpec{RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009}
		},
		Cadence:  time.Hour,
		Logger:   quiet(),
		Tiers:    []string{"local", "felhom-pbs"},
		TierPick: pick,
		State:    st,
		InFlight: gate,
	})
}

// ── SCENARIO A — both tiers get tested, each ONCE per archive ────────────────────────────────
//
// R-86 CHANGED THIS TEST'S CONTRACT, deliberately, and the old assertion is worth recording because
// it was a faithful statement of the defect. It read:
//
//	4 ticks → 4 runs, and consecutive runs must hit different tiers
//
// i.e. every tick produced a heavy restore-test, because the ticker WAS the trigger. Under R-86 a
// tick is an EVALUATION: both tiers are still exercised (rotation is intact), but a tier whose
// newest settled archive is already proven is not re-tested just because time passed. So the
// assertion is now 2 runs across 4 evaluations — one per tier, one per archive — which is a
// STRICTLY STRONGER statement: it pins both the coverage R-85 won and the pacing R-86 adds.
//
// COMPANION RED-PROOF (observed): restore the single-target picker — set `Tiers`/`TierPick` to nil
// so `pickForThisRun` falls back to `s.pick` on the primary runner — and this fails with
// "both tiers must be exercised; got [local:…]", i.e. the offsite tier never appears. That is
// pre-R-85 behaviour, and it is why demo-hp's DR tier went unproven for its entire existence.
func TestRotation_BothTiersExercisedOncePerArchive(t *testing.T) {
	rr := &rotRunner{pass: true}
	st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
	s := rotScheduler(t, rr, st, archiveFor(map[string]string{
		"local":      "local:backup/vzdump-lxc-9201-x.tar.zst",
		"felhom-pbs": "felhom-pbs:backup/ct/9201/2026-07-26T15:42:42Z",
	}), &InFlight{})
	s.now = func() time.Time { return time.Now().UTC() }

	for i := 0; i < 4; i++ {
		s.tick(context.Background())
	}

	got := rr.seen()
	var sawLocal, sawPBS bool
	for _, a := range got {
		if len(a) >= 5 && a[:5] == "local" {
			sawLocal = true
		}
		if len(a) >= 10 && a[:10] == "felhom-pbs" {
			sawPBS = true
		}
	}
	if !sawLocal || !sawPBS {
		t.Fatalf("both tiers must be exercised; got %v", got)
	}
	// Exactly one run per tier: the archives never changed, so nothing became due a second time.
	if len(got) != 2 {
		t.Fatalf("want 2 runs across 4 evaluations (one per archive generation), got %d: %v", len(got), got)
	}
	if got[0] == got[1] {
		t.Fatalf("the two runs must be different tiers — oldest-first is not ordering due tiers: %v", got)
	}
}

// A tier with NO archive is skipped, not failed, and the other tier still runs. A brand-new offsite
// tier legitimately has nothing to restore; turning that into a failure would make every fresh box
// look broken for its first week.
func TestRotation_EmptyTierSkippedNotFailed(t *testing.T) {
	rr := &rotRunner{pass: true}
	st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
	s := rotScheduler(t, rr, st, archiveFor(map[string]string{
		"local":      "local:backup/vzdump-lxc-9201-x.tar.zst",
		"felhom-pbs": "", // provisioned but empty
	}), &InFlight{})

	s.tick(context.Background())
	got := rr.seen()
	if len(got) != 1 || got[0][:5] != "local" {
		t.Fatalf("an empty tier must be skipped and the testable one still run; got %v", got)
	}
}

// Nothing testable anywhere → a clean no-op, not an error and not a run.
func TestRotation_NoArchivesAnywhereIsANoOp(t *testing.T) {
	rr := &rotRunner{pass: true}
	s := rotScheduler(t, rr, NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")),
		archiveFor(map[string]string{}), &InFlight{})
	s.tick(context.Background())
	if got := rr.seen(); len(got) != 0 {
		t.Fatalf("no archives anywhere → no run; got %v", got)
	}
}

// A FAILED restore-test must NOT earn rotation credit, or a tier that fails every time would look
// freshly proven and quietly stop being retried.
func TestRotation_FailureEarnsNoCredit(t *testing.T) {
	rr := &rotRunner{pass: false}
	st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
	s := rotScheduler(t, rr, st, archiveFor(map[string]string{
		"local":      "local:backup/x.tar.zst",
		"felhom-pbs": "felhom-pbs:backup/ct/9201/y",
	}), &InFlight{})
	s.tick(context.Background())
	if _, ok := st.LastSuccess("local"); ok {
		t.Fatal("a FAILED restore-test must not stamp the tier as proven")
	}
	if _, ok := st.LastSuccess("felhom-pbs"); ok {
		t.Fatal("a FAILED restore-test must not stamp the tier as proven")
	}
}

// ── SCENARIO E — rotation survives a restart ─────────────────────────────────────────────────
//
// COMPANION RED-PROOF (observed): make the state in-memory (construct a fresh
// `NewRestoreTestState` on a DIFFERENT path for the second scheduler, i.e. lose the file) and this
// fails with "after a restart the OTHER tier must be next; got felhom-pbs" — the same tier repeats
// and the other is starved indefinitely, which with agent deploys as routine as they are is not a
// corner case.
func TestRotation_SurvivesRestart(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "rt.json")
	picks := archiveFor(map[string]string{
		"local":      "local:backup/x.tar.zst",
		"felhom-pbs": "felhom-pbs:backup/ct/9201/y",
	})

	// First process: the OFFSITE tier is tested (never-proven sorts first).
	rr1 := &rotRunner{pass: true}
	st1 := NewRestoreTestState(path)
	s1 := rotScheduler(t, rr1, st1, picks, &InFlight{})
	s1.tick(context.Background())
	first := rr1.seen()
	if len(first) != 1 {
		t.Fatalf("want one run, got %v", first)
	}

	// --- restart: brand-new state object reading the SAME file ---
	rr2 := &rotRunner{pass: true}
	st2 := NewRestoreTestState(path)
	s2 := rotScheduler(t, rr2, st2, picks, &InFlight{})
	s2.tick(context.Background())
	second := rr2.seen()
	if len(second) != 1 {
		t.Fatalf("want one run after restart, got %v", second)
	}

	if second[0] == first[0] {
		t.Fatalf("after a restart the OTHER tier must be next; got %s twice (rotation state was lost)", second[0])
	}
}

// ── SCENARIO F — no collision with a backup ──────────────────────────────────────────────────
//
// COMPANION RED-PROOF (observed): drop the TryAcquire guard from `tick` and this fails with
// "the restore-test must DEFER while a backup holds the gate; concurrent operations = 2" — the
// count is the assertion, since "both completed" would pass against a fully concurrent
// implementation.
func TestRotation_DefersWhileABackupHoldsTheGate(t *testing.T) {
	gate := &InFlight{}
	release, _, ok := gate.TryAcquire("backup:felhom-pbs")
	if !ok {
		t.Fatal("precondition: the gate should have been free")
	}
	defer release()

	rr := &rotRunner{pass: true}
	s := rotScheduler(t, rr, NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")),
		archiveFor(map[string]string{"local": "local:backup/x.tar.zst"}), gate)

	s.tick(context.Background())

	concurrent := 1 + len(rr.seen()) // the backup holding the gate, plus anything the tick started
	if concurrent != 1 {
		t.Fatalf("the restore-test must DEFER while a backup holds the gate; concurrent operations = %d", concurrent)
	}
}

// Once the backup releases, the next cadence proceeds — deferral must not be permanent.
func TestRotation_ResumesAfterTheGateFrees(t *testing.T) {
	gate := &InFlight{}
	release, _, _ := gate.TryAcquire("backup:local")

	rr := &rotRunner{pass: true}
	s := rotScheduler(t, rr, NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")),
		archiveFor(map[string]string{"local": "local:backup/x.tar.zst"}), gate)

	s.tick(context.Background())
	if len(rr.seen()) != 0 {
		t.Fatal("should have deferred while held")
	}
	release()
	s.tick(context.Background())
	if len(rr.seen()) != 1 {
		t.Fatalf("must resume once the gate frees; got %v", rr.seen())
	}
}

// The gate itself: one holder at a time, named, and release is idempotent.
func TestInFlight_Semantics(t *testing.T) {
	g := &InFlight{}
	rel, busy, ok := g.TryAcquire("backup:local")
	if !ok || busy != "" {
		t.Fatalf("first acquire must succeed; ok=%v busy=%q", ok, busy)
	}
	if _, busy2, ok2 := g.TryAcquire("restore-test"); ok2 || busy2 != "backup:local" {
		t.Fatalf("second acquire must fail and NAME the holder; ok=%v busy=%q", ok2, busy2)
	}
	rel()
	rel() // idempotent — a double release must not free someone else's later claim
	if g.Busy() != "" {
		t.Fatalf("gate should be idle after release; busy=%q", g.Busy())
	}
	if _, _, ok3 := g.TryAcquire("restore-test"); !ok3 {
		t.Fatal("gate must be reusable after release")
	}
}

// A nil gate means "not wired" → no gating, pre-R-85 behaviour. Keeps every existing caller working.
func TestInFlight_NilIsUngated(t *testing.T) {
	var g *InFlight
	rel, _, ok := g.TryAcquire("x")
	if !ok {
		t.Fatal("a nil gate must not block")
	}
	rel()
	if g.Busy() != "" {
		t.Fatal("a nil gate is never busy")
	}
}

// ── oldest-first ordering ────────────────────────────────────────────────────────────────────

func TestOldestFirst_Ordering(t *testing.T) {
	st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
	now := time.Now().UTC()

	// Never-proven sorts FIRST — the case that matters, since the offsite tier starts there.
	if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "felhom-pbs" {
		// both never proven → deterministic tie-break by id
		if got[0] != "felhom-pbs" && got[0] != "local" {
			t.Fatalf("unexpected: %v", got)
		}
	}
	_ = st.RecordSuccess("local", "local:backup/a.tar.zst", "local", "boot+running", now)
	if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "felhom-pbs" {
		t.Fatalf("a never-proven tier must sort before a proven one; got %v", got)
	}
	_ = st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/b", "pbs", "boot+running", now.Add(time.Hour))
	if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "local" {
		t.Fatalf("the least recently proven must sort first; got %v", got)
	}
}

// Ordering must be DETERMINISTIC for equal timestamps, or two tiers proven in the same second would
// rotate by Go's randomised map iteration — untestable, and occasionally starving.
func TestOldestFirst_DeterministicOnTies(t *testing.T) {
	st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
	now := time.Now().UTC()
	_ = st.RecordSuccess("b-tier", "b:archive", "local", "boot+running", now)
	_ = st.RecordSuccess("a-tier", "a:archive", "local", "boot+running", now)
	for i := 0; i < 20; i++ {
		if got := st.OldestFirst([]string{"b-tier", "a-tier"}); got[0] != "a-tier" {
			t.Fatalf("tie-break must be deterministic; iteration %d gave %v", i, got)
		}
	}
}

// The state file round-trips, and a corrupt file degrades to "nothing proven" rather than wedging.
func TestRestoreTestState_PersistenceAndCorruption(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "rt.json")
	now := time.Now().UTC().Truncate(time.Second)

	st := NewRestoreTestState(path)
	if err := st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/x", "pbs", "boot+running", now); err != nil {
		t.Fatal(err)
	}
	reopened := NewRestoreTestState(path)
	got, ok := reopened.LastSuccess("felhom-pbs")
	if !ok || !got.Equal(now) {
		t.Fatalf("state must round-trip; got %v ok=%v want %v", got, ok, now)
	}

	bad := filepath.Join(dir, "corrupt.json")
	if err := os.WriteFile(bad, []byte("{{{not json"), 0o600); err != nil {
		t.Fatal(err)
	}
	c := NewRestoreTestState(bad)
	if _, ok := c.LastSuccess("felhom-pbs"); ok {
		t.Fatal("a corrupt state file must degrade to 'nothing proven', not invent a timestamp")
	}
}