R-86: restore-test follows the backup, not the clock (v0.121.0)
gates / gates (push) Failing after 7s
gates / gates (push) Failing after 7s
The ticker survives as the EVALUATION interval only. A tier is DUE when its
newest archive that has settled for `settle` (default 24h) has not been proven:
daily tier -> proved daily on yesterday's archive, weekly tier -> weekly on its
own, newborn -> UNKNOWN.
The trap avoided: the literal reading ("newest archive is >= 24h old") is NEVER
true on a daily tier, so it silently switches restore-testing off where it
matters most. Red-proved at 0 runs over 5 simulated days.
- state records WHICH archive was proven; legacy files keep their time and yield
no proven archive (each tier due once after the upgrade, deliberately)
- two knobs replace one: restore_test_eval_interval_seconds (6h, measured) and
restore_test_settle_seconds (24h). The old cadence key keeps its DISABLE
meaning verbatim and now seeds the settle lag, with a start-up WARN.
- due-check runs BEFORE the heavy-op gate (a frequent poll must not make a
starting backup record a failure, F-A1)
- candidate picker skips implausible archives (a phantom would be due forever)
- new read-only --selftest=restore-test-due prints the verdict + its cost
This commit is contained in:
@@ -39,10 +39,19 @@ func (r *rotRunner) seen() []string {
|
||||
return append([]string(nil), r.archives...)
|
||||
}
|
||||
|
||||
// testLanded is a landing time old enough to be settled under any cutoff these tests use. R-86
|
||||
// widened the TierPicker seam with the archive's landing time; the rotation tests below are about
|
||||
// tier ORDER and the heavy-operation gate, not about settling, so they hold it constant.
|
||||
var testLanded = time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
// archiveFor is a TierPicker over a fixed map: target → archive ("" = that tier holds none).
|
||||
func archiveFor(m map[string]string) TierPicker {
|
||||
return func(_ context.Context, target string) (string, error) {
|
||||
return m[target], nil
|
||||
return func(_ context.Context, target string, _ time.Time) (string, time.Time, error) {
|
||||
a := m[target]
|
||||
if a == "" {
|
||||
return "", time.Time{}, nil
|
||||
}
|
||||
return a, testLanded, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,14 +72,24 @@ func rotScheduler(t *testing.T, rr *rotRunner, st *RestoreTestState, pick TierPi
|
||||
})
|
||||
}
|
||||
|
||||
// ── SCENARIO A — both tiers get tested across consecutive cadences ───────────────────────────
|
||||
// ── SCENARIO A — both tiers get tested, each ONCE per archive ────────────────────────────────
|
||||
//
|
||||
// R-86 CHANGED THIS TEST'S CONTRACT, deliberately, and the old assertion is worth recording because
|
||||
// it was a faithful statement of the defect. It read:
|
||||
//
|
||||
// 4 ticks → 4 runs, and consecutive runs must hit different tiers
|
||||
//
|
||||
// i.e. every tick produced a heavy restore-test, because the ticker WAS the trigger. Under R-86 a
|
||||
// tick is an EVALUATION: both tiers are still exercised (rotation is intact), but a tier whose
|
||||
// newest settled archive is already proven is not re-tested just because time passed. So the
|
||||
// assertion is now 2 runs across 4 evaluations — one per tier, one per archive — which is a
|
||||
// STRICTLY STRONGER statement: it pins both the coverage R-85 won and the pacing R-86 adds.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): restore the single-target picker — set `Tiers`/`TierPick` to nil
|
||||
// so `pickForThisRun` falls back to `s.pick` on the primary runner — and this fails with
|
||||
// "both tiers must be exercised across 4 cadences; got [local:… local:… local:… local:…]",
|
||||
// i.e. the offsite tier never appears. That is today's behaviour, and it is why demo-hp's DR tier
|
||||
// went unproven for its entire existence.
|
||||
func TestRotation_BothTiersExercisedAcrossCadences(t *testing.T) {
|
||||
// "both tiers must be exercised; got [local:…]", i.e. the offsite tier never appears. That is
|
||||
// pre-R-85 behaviour, and it is why demo-hp's DR tier went unproven for its entire existence.
|
||||
func TestRotation_BothTiersExercisedOncePerArchive(t *testing.T) {
|
||||
rr := &rotRunner{pass: true}
|
||||
st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
|
||||
s := rotScheduler(t, rr, st, archiveFor(map[string]string{
|
||||
@@ -94,14 +113,14 @@ func TestRotation_BothTiersExercisedAcrossCadences(t *testing.T) {
|
||||
}
|
||||
}
|
||||
if !sawLocal || !sawPBS {
|
||||
t.Fatalf("both tiers must be exercised across 4 cadences; got %v", got)
|
||||
t.Fatalf("both tiers must be exercised; got %v", got)
|
||||
}
|
||||
// Oldest-first must ALTERNATE, not clump — otherwise one tier is starved between visits.
|
||||
if len(got) != 4 {
|
||||
t.Fatalf("want 4 runs, got %d: %v", len(got), got)
|
||||
// Exactly one run per tier: the archives never changed, so nothing became due a second time.
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want 2 runs across 4 evaluations (one per archive generation), got %d: %v", len(got), got)
|
||||
}
|
||||
if got[0] == got[1] {
|
||||
t.Fatalf("consecutive runs hit the same tier — oldest-first is not rotating: %v", got)
|
||||
t.Fatalf("the two runs must be different tiers — oldest-first is not ordering due tiers: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -284,11 +303,11 @@ func TestOldestFirst_Ordering(t *testing.T) {
|
||||
t.Fatalf("unexpected: %v", got)
|
||||
}
|
||||
}
|
||||
_ = st.RecordSuccess("local", now)
|
||||
_ = st.RecordSuccess("local", "local:backup/a.tar.zst", now)
|
||||
if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "felhom-pbs" {
|
||||
t.Fatalf("a never-proven tier must sort before a proven one; got %v", got)
|
||||
}
|
||||
_ = st.RecordSuccess("felhom-pbs", now.Add(time.Hour))
|
||||
_ = st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/b", now.Add(time.Hour))
|
||||
if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "local" {
|
||||
t.Fatalf("the least recently proven must sort first; got %v", got)
|
||||
}
|
||||
@@ -299,8 +318,8 @@ func TestOldestFirst_Ordering(t *testing.T) {
|
||||
func TestOldestFirst_DeterministicOnTies(t *testing.T) {
|
||||
st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
|
||||
now := time.Now().UTC()
|
||||
_ = st.RecordSuccess("b-tier", now)
|
||||
_ = st.RecordSuccess("a-tier", now)
|
||||
_ = st.RecordSuccess("b-tier", "b:archive", now)
|
||||
_ = st.RecordSuccess("a-tier", "a:archive", now)
|
||||
for i := 0; i < 20; i++ {
|
||||
if got := st.OldestFirst([]string{"b-tier", "a-tier"}); got[0] != "a-tier" {
|
||||
t.Fatalf("tie-break must be deterministic; iteration %d gave %v", i, got)
|
||||
@@ -315,7 +334,7 @@ func TestRestoreTestState_PersistenceAndCorruption(t *testing.T) {
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
|
||||
st := NewRestoreTestState(path)
|
||||
if err := st.RecordSuccess("felhom-pbs", now); err != nil {
|
||||
if err := st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/x", now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened := NewRestoreTestState(path)
|
||||
|
||||
Reference in New Issue
Block a user