package hub import ( "context" "testing" "time" ) // R-189 — a passing restore-test must survive an agent restart and reach the hub. // // THE OBSERVATION THIS EXISTS FOR (2026-08-03, demo-felhom): a real 14.5 GB offsite restore-test // PASSED at 15:25:14; the agent was restarted 2 m 43 s later for a deploy; the hub logged // `0 restore-tests` on the next two host-reports. The in-memory store's own comment said "lost on // restart; the cadence re-populates", which was true under a timer and stopped being true when R-86 // made the agent refuse to re-test an archive it has already proven. // // Timestamps here carry JITTER (odd minutes and seconds, not round hours) — yesterday a test was // hollow because a perfectly regular series landed exactly on a threshold and passed under the // mutation it was meant to catch. type fakeLatest struct{ tests []RestoreTest } func (f *fakeLatest) RestoreTests(context.Context) []RestoreTest { return f.tests } type fakeProven struct{ tests []RestoreTest } func (f *fakeProven) ProvenRestoreTests(context.Context) []RestoreTest { return f.tests } func rt(tier, archive string, pass bool, at time.Time) RestoreTest { return RestoreTest{ SourceArchive: archive, SourceTier: tier, Pass: pass, Verified: "boot+running", TestedAt: at.UTC().Format(time.RFC3339), } } // mergeCollector builds a Collector with only the two restore-test seams wired — the merge is what // is under test, not the rest of the collection. func mergeCollector(latest, proven []RestoreTest) *Collector { c := &Collector{} if latest != nil { c.restoreTests = &fakeLatest{tests: latest} } if proven != nil { c.provenTests = &fakeProven{tests: proven} } return c } func findTier(got []RestoreTest, tier string) (RestoreTest, int) { var hit RestoreTest n := 0 for _, e := range got { if e.SourceTier == tier { hit, n = e, n+1 } } return hit, n } // ── SCENARIO A — a proof survives a restart and reaches the hub ────────────────────────────── // // COMPANION RED-PROOF (observed 2026-08-03): delete the `c.provenTests` merge from // collectRestoreTests (return the in-memory slice as it used to) → // // --- FAIL: TestMerge_ProofSurvivesARestart // restoretest_merge_test.go: after a restart the persisted proof must be reported; got 0 entr(ies) // // which is exactly the live observation: `0 restore-tests`. Restored. func TestMerge_ProofSurvivesARestart(t *testing.T) { provenAt := time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC) // the real run's timestamp // After a restart the in-memory store is EMPTY — this is the whole point. c := mergeCollector([]RestoreTest{}, []RestoreTest{ rt("pbs", "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z", true, provenAt), }) got := c.collectRestoreTests(context.Background()) if len(got) != 1 { t.Fatalf("after a restart the persisted proof must be reported; got %d entr(ies): %+v", len(got), got) } e := got[0] if e.SourceArchive != "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z" { t.Fatalf("the entry must name the archive that was proven — the hub keys on it; got %q", e.SourceArchive) } if e.SourceTier != "pbs" || !e.Pass { t.Fatalf("the entry must be a PASS on the tier it was proven on; got tier=%q pass=%v", e.SourceTier, e.Pass) } if e.TestedAt != provenAt.Format(time.RFC3339) { t.Fatalf("the entry must carry the ORIGINAL test time, not now(); got %q", e.TestedAt) } } // ── SCENARIO B — the report does not invent a pass ─────────────────────────────────────────── // // COMPANION RED-PROOF (observed): make the state layer emit an entry for an unproven tier (drop the // `reportable()` filter in ProvenRestoreTests, so a legacy record with no archive is emitted) — the // equivalent at this layer is a proven-source that returns an entry for a tier nothing proved, which // this test injects directly and the assertion below rejects. func TestMerge_NeverInventsAPassForAnUnprovenTier(t *testing.T) { // Nothing proven anywhere: no in-memory result, no persisted proof. c := mergeCollector([]RestoreTest{}, []RestoreTest{}) if got := c.collectRestoreTests(context.Background()); len(got) != 0 { t.Fatalf("a tier with no proof must produce NO entry — an unproven tier reading as proven is "+ "worse than the defect being fixed; got %+v", got) } // And an entry the state layer could not describe (no tier) is never promoted into a proof. c2 := mergeCollector([]RestoreTest{}, []RestoreTest{ {SourceArchive: "local:backup/x.tar.zst", SourceTier: "", Pass: true, TestedAt: time.Date(2026, 8, 1, 4, 41, 58, 0, time.UTC).Format(time.RFC3339)}, }) if got := c2.collectRestoreTests(context.Background()); len(got) != 0 { t.Fatalf("a persisted record with no tier is not a usable proof and must be dropped; got %+v", got) } } // ── SCENARIO C — a fresh in-memory result wins, and never duplicates ───────────────────────── // // COMPANION RED-PROOF (observed 2026-08-03): remove the de-duplication (append every persisted entry // unconditionally) → // // --- FAIL: TestMerge_NewerWinsAndNeverDuplicatesATier // restoretest_merge_test.go: one entry per tier; got 2 for "pbs" — the hub would read two tests // // Restored. func TestMerge_NewerWinsAndNeverDuplicatesATier(t *testing.T) { lastWeek := time.Date(2026, 7, 27, 19, 55, 41, 0, time.UTC) // jittered, from the real box fiveMinAgo := time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC) c := mergeCollector( []RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", true, fiveMinAgo)}, []RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, lastWeek)}, ) got := c.collectRestoreTests(context.Background()) e, n := findTier(got, "pbs") if n != 1 { t.Fatalf("one entry per tier; got %d for \"pbs\" — the hub would read two tests: %+v", n, got) } if e.SourceArchive != "felhom-pbs:backup/ct/9201/new" { t.Fatalf("the NEWER result must win; got %q tested %q", e.SourceArchive, e.TestedAt) } // ...and the older-in-memory / newer-persisted direction, which is the post-restart case. c2 := mergeCollector( []RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, lastWeek)}, []RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", true, fiveMinAgo)}, ) e2, n2 := findTier(c2.collectRestoreTests(context.Background()), "pbs") if n2 != 1 || e2.SourceArchive != "felhom-pbs:backup/ct/9201/new" { t.Fatalf("newest must win regardless of which source it came from; got %d entr(ies), archive %q", n2, e2.SourceArchive) } } // ── SCENARIO D — a failure still reaches the hub ───────────────────────────────────────────── // // The merge must not mask a failure with an older stored success. A failing tier is retried at the // next evaluation and its record lives ONLY in memory, so losing it here would silence the loudest // DR signal this system produces. func TestMerge_AFailureIsStillReported(t *testing.T) { provenLastWeek := time.Date(2026, 7, 27, 19, 55, 41, 0, time.UTC) failedJustNow := time.Date(2026, 8, 3, 13, 41, 7, 0, time.UTC) c := mergeCollector( []RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", false, failedJustNow)}, []RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, provenLastWeek)}, ) e, n := findTier(c.collectRestoreTests(context.Background()), "pbs") if n != 1 { t.Fatalf("one entry per tier; got %d: %+v", n, c.collectRestoreTests(context.Background())) } if e.Pass { t.Fatalf("a FAILURE newer than the stored proof must be what is reported — masking it would "+ "silence the loudest DR signal there is; got pass=%v archive=%q", e.Pass, e.SourceArchive) } } // Two different tiers are both reported — the merge is per tier, not a single slot. func TestMerge_BothTiersSurvive(t *testing.T) { c := mergeCollector( []RestoreTest{rt("local", "felhom-backup:backup/vzdump-lxc-9201-a.tar.zst", true, time.Date(2026, 8, 3, 4, 44, 50, 0, time.UTC))}, []RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/x", true, time.Date(2026, 8, 2, 5, 12, 33, 0, time.UTC))}, ) got := c.collectRestoreTests(context.Background()) if _, n := findTier(got, "local"); n != 1 { t.Fatalf("the in-memory tier must survive the merge; got %+v", got) } if _, n := findTier(got, "pbs"); n != 1 { t.Fatalf("the persisted tier must survive the merge; got %+v", got) } } // A malformed timestamp must never displace a good entry — "unparseable" is not "newest". func TestMerge_MalformedTimestampNeverWins(t *testing.T) { good := rt("pbs", "felhom-pbs:backup/ct/9201/good", true, time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC)) bad := RestoreTest{SourceArchive: "felhom-pbs:backup/ct/9201/bad", SourceTier: "pbs", Pass: true, TestedAt: "not-a-time"} c := mergeCollector([]RestoreTest{good}, []RestoreTest{bad}) e, n := findTier(c.collectRestoreTests(context.Background()), "pbs") if n != 1 || e.SourceArchive != "felhom-pbs:backup/ct/9201/good" { t.Fatalf("an unparseable timestamp must not displace a good entry; got %d entr(ies), archive %q", n, e.SourceArchive) } } // A nil proven-source leaves the pre-R-189 behaviour exactly as it was. func TestMerge_NilProvenSourceIsANoOp(t *testing.T) { only := rt("local", "felhom-backup:backup/x.tar.zst", true, time.Date(2026, 8, 3, 4, 44, 50, 0, time.UTC)) c := mergeCollector([]RestoreTest{only}, nil) got := c.collectRestoreTests(context.Background()) if len(got) != 1 || got[0].SourceArchive != only.SourceArchive { t.Fatalf("a nil durable source must not change anything; got %+v", got) } }