package monitor import ( "encoding/json" "io" "log" "path/filepath" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // R-85 Part 2 — the restore-test result must be HEARD. // // Before this, a failed restore-test was a `[WARN]` line in the ingest handler and nothing else — // no event, no notification, no gauge. That was true for the LOCAL tier that was already being // tested, which means the loudest DR signal this system produces was in practice inaudible. type capturedEvent struct { customerID, eventType, severity, message string } func rtStore(t *testing.T) *store.Store { t.Helper() st, err := store.New(filepath.Join(t.TempDir(), "rt.db"), log.New(io.Discard, "", 0)) if err != nil { t.Fatal(err) } t.Cleanup(func() { st.Close() }) if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "k", RetrievalPassword: "p"}); err != nil { t.Fatal(err) } if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil { t.Fatal(err) } return st } // rtReport builds a host-report carrying storage targets and zero or one restore-test. func rtReport(t *testing.T, tier string, pass bool, testedAt time.Time, errMsg string) string { t.Helper() type rt struct { SourceArchive string `json:"source_archive"` SourceTier string `json:"source_tier"` Pass bool `json:"pass"` Error string `json:"error,omitempty"` TestedAt string `json:"tested_at"` } type stg struct { Name string `json:"name"` Type string `json:"type"` Content string `json:"content"` } payload := struct { RestoreTests []rt `json:"restore_tests"` StorageTargets []stg `json:"storage_targets"` }{ StorageTargets: []stg{ {Name: "local", Type: "local", Content: "backup,iso"}, {Name: "felhom-pbs", Type: "pbs", Content: "backup"}, }, } if tier != "" { payload.RestoreTests = []rt{{ SourceArchive: tier + ":backup/ct/9201/x", SourceTier: tier, Pass: pass, Error: errMsg, TestedAt: testedAt.UTC().Format(time.RFC3339), }} } b, err := json.Marshal(payload) if err != nil { t.Fatal(err) } return string(b) } func rtChecker(st *store.Store, got *[]capturedEvent, logTo io.Writer) *RestoreTestChecker { if logTo == nil { logTo = io.Discard } return NewRestoreTestChecker(st, func(cid, et, sev, msg, _, _ string) { *got = append(*got, capturedEvent{cid, et, sev, msg}) }, log.New(logTo, "", 0)) } // ── SCENARIO B — a failed restore-test is HEARD ────────────────────────────────────────────── // // The assertion is that a NOTIFICATION IS EMITTED, not that a handler ran. The hollow version of // this test checks for a log line, which passes against exactly the code this task replaces. // // COMPANION RED-PROOF (observed): make checkFailure `return` before emit (the pre-R-85 shape — log // and stop) and this fails with "a FAILED restore-test must EMIT an operator event; got 0 event(s)". // Restored. func TestRestoreTest_FailureEmitsAnOperatorEvent(t *testing.T) { st := rtStore(t) now := time.Now().UTC() if err := st.SaveHostReport("h1", "c1", []byte(rtReport(t, "pbs", false, now, "restore task: context deadline exceeded")), store.HostReportDenorm{}); err != nil { t.Fatal(err) } var got []capturedEvent rtChecker(st, &got, nil).Check() var fail *capturedEvent for i := range got { if got[i].eventType == EventRestoreTestFailed { fail = &got[i] } } if fail == nil { t.Fatalf("a FAILED restore-test must EMIT an operator event; got %d event(s): %+v", len(got), got) } if fail.severity != "error" { t.Fatalf("a failed restore-test is an error, got severity %q", fail.severity) } if !strings.Contains(fail.message, "pbs") { t.Fatalf("the TIER must be named — 'a restore-test failed' without saying which tier is not actionable; got %q", fail.message) } } // Edge-triggered: a permanently failing tier must not emit on every sweep. func TestRestoreTest_FailureDoesNotSpam(t *testing.T) { st := rtStore(t) now := time.Now().UTC() st.SaveHostReport("h1", "c1", []byte(rtReport(t, "pbs", false, now, "boom")), store.HostReportDenorm{}) var got []capturedEvent c := rtChecker(st, &got, nil) c.Check() c.Check() c.Check() n := 0 for _, e := range got { if e.eventType == EventRestoreTestFailed { n++ } } if n != 1 { t.Fatalf("a persistently failing tier must report ONCE per failing run, got %d", n) } } // A PASS must not emit a failure event. func TestRestoreTest_PassEmitsNoFailure(t *testing.T) { st := rtStore(t) now := time.Now().UTC() st.SaveHostReport("h1", "c1", []byte(rtReport(t, "pbs", true, now, "")), store.HostReportDenorm{}) var got []capturedEvent rtChecker(st, &got, nil).Check() for _, e := range got { if e.eventType == EventRestoreTestFailed { t.Fatalf("a PASSING restore-test must not emit a failure; got %+v", e) } } } // ── SCENARIO D — a newborn box does not alarm (the R-81 lesson) ───────────────────────────── // // COMPANION RED-PROOF (observed): remove the anchor branch from assessRestoreProven (treat "never // proven" as MISSED outright) and this fails with "a newborn box must NOT alarm; got // restore_test_stale ...". That is the fourth instance of no-signal-as-bad-signal, and it would // have been written by the same hand that just fixed the third. func TestRestoreTest_NewbornDoesNotAlarm(t *testing.T) { st := rtStore(t) now := time.Now().UTC() // A report with NO restore-test at all, and first contact one hour ago. st.SaveHostReport("h1", "c1", []byte(rtReport(t, "", false, now, "")), store.HostReportDenorm{}) var got []capturedEvent var logbuf strings.Builder rtChecker(st, &got, &logbuf).Check() for _, e := range got { if e.eventType == EventRestoreTestStale { t.Fatalf("a newborn box must NOT alarm; got %s: %s", e.eventType, e.message) } } // ...but the deferral must be VISIBLE, or quiet is indistinguishable from not-checked. if !strings.Contains(logbuf.String(), "not restore-proven yet") { t.Fatalf("the deferred verdict must be logged; log:\n%s", logbuf.String()) } } // The boundary, pinned by name so a refactor has to delete an obviously-named contract. func TestRestoreTest_Contract_UnprovenIsUnknownUntilTheAnchorElapses(t *testing.T) { now := time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC) cases := []struct { name string watched time.Duration wantMissed bool }{ {"newborn, 1h", time.Hour, false}, {"just inside", restoreProvenStaleAfter - time.Minute, false}, {"exactly at the limit", restoreProvenStaleAfter, false}, {"just outside", restoreProvenStaleAfter + time.Minute, true}, {"long past", 30 * 24 * time.Hour, true}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { got := assessRestoreProven("pbs", time.Time{}, now.Add(-c.watched), now) if got.missed() != c.wantMissed { t.Fatalf("CONTRACT VIOLATED: unproven for %s (limit %s) → missed=%v, want %v (reason %q)", c.watched, restoreProvenStaleAfter, got.missed(), c.wantMissed, got.reason) } if !c.wantMissed && got.verdict != verdictUnknown { t.Fatalf("a deferred tier must be UNKNOWN (visible), not OK; got verdict=%d", got.verdict) } }) } } // ── SCENARIO C — an unproven tier becomes visible, and is DISTINCT from a failure ──────────── func TestRestoreTest_StaleIsSeparateFromFailure(t *testing.T) { now := time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC) stale := assessRestoreProven("pbs", now.Add(-9*24*time.Hour), now.Add(-60*24*time.Hour), now) if !stale.missed() { t.Fatalf("a tier last proven 9 days ago (limit %s) must be stale; got %q", restoreProvenStaleAfter, stale.reason) } // The wording must not read as "broken" — that is the other signal. if !strings.Contains(stale.reason, "unverified, not known-broken") { t.Fatalf("staleness must say UNVERIFIED, not broken — merging the two is the thing this avoids; got %q", stale.reason) } fresh := assessRestoreProven("pbs", now.Add(-2*24*time.Hour), now.Add(-60*24*time.Hour), now) if fresh.verdict != verdictOK { t.Fatalf("a tier proven 2 days ago is fine; got verdict=%d reason=%q", fresh.verdict, fresh.reason) } } // The two signals must be DIFFERENT event types — merging them would collapse "your DR is broken" // into "your DR is unverified", and the second is the one that quietly becomes the first. func TestRestoreTest_EventTypesAreDistinct(t *testing.T) { if EventRestoreTestFailed == EventRestoreTestStale { t.Fatal("failure and staleness must be distinct event types") } } // A tier the box does not HAVE is never reported stale — the Slice-C gate, for the same reason: // without it, absence-is-not-failure would be re-introduced one level down. func TestRestoreTest_TierNotPresentIsNeverStale(t *testing.T) { var hr hostReportRestoreTests if err := json.Unmarshal([]byte(`{"storage_targets":[{"name":"local","type":"local","content":"backup"}]}`), &hr); err != nil { t.Fatal(err) } tiers := expectedRestoreTiers(hr) for _, tr := range tiers { if tr == "pbs" { t.Fatalf("a box with no PBS storage must not expect a pbs tier; got %v", tiers) } } if len(tiers) != 1 || tiers[0] != "local" { t.Fatalf("want just the local tier; got %v", tiers) } } // The window scan recovers per-tier proof even though each report carries only the LATEST run. func TestRestoreTest_LastProvenPerTierAcrossTheWindow(t *testing.T) { now := time.Now().UTC() rows := []store.HostReportRow{ {ReceivedAt: now.Add(-1 * time.Hour), ReportJSON: rtReportRaw("pbs", true, now.Add(-2*time.Hour))}, {ReceivedAt: now.Add(-3 * time.Hour), ReportJSON: rtReportRaw("local", true, now.Add(-4*time.Hour))}, {ReceivedAt: now.Add(-5 * time.Hour), ReportJSON: `{{{malformed`}, } got := lastProvenPerTier(rows) if _, ok := got["pbs"]; !ok { t.Fatalf("the pbs tier's proof must be recovered from the window; got %v", got) } if _, ok := got["local"]; !ok { t.Fatalf("the local tier's proof must be recovered even though a LATER report shows only pbs; got %v", got) } } func rtReportRaw(tier string, pass bool, at time.Time) string { return `{"restore_tests":[{"source_archive":"` + tier + `:x","source_tier":"` + tier + `","pass":` + boolStr(pass) + `,"tested_at":"` + at.UTC().Format(time.RFC3339) + `"}]}` } func boolStr(b bool) string { if b { return "true" } return "false" }