R-86: restore-test follows the backup, not the clock (v0.121.0)
gates / gates (push) Failing after 7s

The ticker survives as the EVALUATION interval only. A tier is DUE when its
newest archive that has settled for `settle` (default 24h) has not been proven:
daily tier -> proved daily on yesterday's archive, weekly tier -> weekly on its
own, newborn -> UNKNOWN.

The trap avoided: the literal reading ("newest archive is >= 24h old") is NEVER
true on a daily tier, so it silently switches restore-testing off where it
matters most. Red-proved at 0 runs over 5 simulated days.

- state records WHICH archive was proven; legacy files keep their time and yield
  no proven archive (each tier due once after the upgrade, deliberately)
- two knobs replace one: restore_test_eval_interval_seconds (6h, measured) and
  restore_test_settle_seconds (24h). The old cadence key keeps its DISABLE
  meaning verbatim and now seeds the settle lag, with a start-up WARN.
- due-check runs BEFORE the heavy-op gate (a frequent poll must not make a
  starting backup record a failure, F-A1)
- candidate picker skips implausible archives (a phantom would be due forever)
- new read-only --selftest=restore-test-due prints the verdict + its cost
This commit is contained in:
2026-08-03 14:54:57 +02:00
parent 1b14cfd0b4
commit 4618169036
13 changed files with 1273 additions and 123 deletions
+36 -17
View File
@@ -39,10 +39,19 @@ func (r *rotRunner) seen() []string {
return append([]string(nil), r.archives...)
}
// testLanded is a landing time old enough to be settled under any cutoff these tests use. R-86
// widened the TierPicker seam with the archive's landing time; the rotation tests below are about
// tier ORDER and the heavy-operation gate, not about settling, so they hold it constant.
var testLanded = time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
// archiveFor is a TierPicker over a fixed map: target → archive ("" = that tier holds none).
func archiveFor(m map[string]string) TierPicker {
return func(_ context.Context, target string) (string, error) {
return m[target], nil
return func(_ context.Context, target string, _ time.Time) (string, time.Time, error) {
a := m[target]
if a == "" {
return "", time.Time{}, nil
}
return a, testLanded, nil
}
}
@@ -63,14 +72,24 @@ func rotScheduler(t *testing.T, rr *rotRunner, st *RestoreTestState, pick TierPi
})
}
// ── SCENARIO A — both tiers get tested across consecutive cadences ───────────────────────────
// ── SCENARIO A — both tiers get tested, each ONCE per archive ────────────────────────────────
//
// R-86 CHANGED THIS TEST'S CONTRACT, deliberately, and the old assertion is worth recording because
// it was a faithful statement of the defect. It read:
//
// 4 ticks → 4 runs, and consecutive runs must hit different tiers
//
// i.e. every tick produced a heavy restore-test, because the ticker WAS the trigger. Under R-86 a
// tick is an EVALUATION: both tiers are still exercised (rotation is intact), but a tier whose
// newest settled archive is already proven is not re-tested just because time passed. So the
// assertion is now 2 runs across 4 evaluations — one per tier, one per archive — which is a
// STRICTLY STRONGER statement: it pins both the coverage R-85 won and the pacing R-86 adds.
//
// COMPANION RED-PROOF (observed): restore the single-target picker — set `Tiers`/`TierPick` to nil
// so `pickForThisRun` falls back to `s.pick` on the primary runner — and this fails with
// "both tiers must be exercised across 4 cadences; got [local:… local:… local:… local:…]",
// i.e. the offsite tier never appears. That is today's behaviour, and it is why demo-hp's DR tier
// went unproven for its entire existence.
func TestRotation_BothTiersExercisedAcrossCadences(t *testing.T) {
// "both tiers must be exercised; got [local:…]", i.e. the offsite tier never appears. That is
// pre-R-85 behaviour, and it is why demo-hp's DR tier went unproven for its entire existence.
func TestRotation_BothTiersExercisedOncePerArchive(t *testing.T) {
rr := &rotRunner{pass: true}
st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
s := rotScheduler(t, rr, st, archiveFor(map[string]string{
@@ -94,14 +113,14 @@ func TestRotation_BothTiersExercisedAcrossCadences(t *testing.T) {
}
}
if !sawLocal || !sawPBS {
t.Fatalf("both tiers must be exercised across 4 cadences; got %v", got)
t.Fatalf("both tiers must be exercised; got %v", got)
}
// Oldest-first must ALTERNATE, not clump — otherwise one tier is starved between visits.
if len(got) != 4 {
t.Fatalf("want 4 runs, got %d: %v", len(got), got)
// Exactly one run per tier: the archives never changed, so nothing became due a second time.
if len(got) != 2 {
t.Fatalf("want 2 runs across 4 evaluations (one per archive generation), got %d: %v", len(got), got)
}
if got[0] == got[1] {
t.Fatalf("consecutive runs hit the same tier — oldest-first is not rotating: %v", got)
t.Fatalf("the two runs must be different tiers — oldest-first is not ordering due tiers: %v", got)
}
}
@@ -284,11 +303,11 @@ func TestOldestFirst_Ordering(t *testing.T) {
t.Fatalf("unexpected: %v", got)
}
}
_ = st.RecordSuccess("local", now)
_ = st.RecordSuccess("local", "local:backup/a.tar.zst", now)
if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "felhom-pbs" {
t.Fatalf("a never-proven tier must sort before a proven one; got %v", got)
}
_ = st.RecordSuccess("felhom-pbs", now.Add(time.Hour))
_ = st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/b", now.Add(time.Hour))
if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "local" {
t.Fatalf("the least recently proven must sort first; got %v", got)
}
@@ -299,8 +318,8 @@ func TestOldestFirst_Ordering(t *testing.T) {
func TestOldestFirst_DeterministicOnTies(t *testing.T) {
st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
now := time.Now().UTC()
_ = st.RecordSuccess("b-tier", now)
_ = st.RecordSuccess("a-tier", now)
_ = st.RecordSuccess("b-tier", "b:archive", now)
_ = st.RecordSuccess("a-tier", "a:archive", now)
for i := 0; i < 20; i++ {
if got := st.OldestFirst([]string{"b-tier", "a-tier"}); got[0] != "a-tier" {
t.Fatalf("tie-break must be deterministic; iteration %d gave %v", i, got)
@@ -315,7 +334,7 @@ func TestRestoreTestState_PersistenceAndCorruption(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
st := NewRestoreTestState(path)
if err := st.RecordSuccess("felhom-pbs", now); err != nil {
if err := st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/x", now); err != nil {
t.Fatal(err)
}
reopened := NewRestoreTestState(path)