package quiesce import ( "context" "testing" "time" ) // R-97a/b — a failing backup must be HEARD, and must not blame the apps. // // Both halves are inert-seam risks, so nothing here asserts "no error occurred". The failure mode // this suite exists to catch is a route that silently does not exist — which is exactly what an // empty event list looks like. // recEvent is one captured hub event. type recEvent struct { kind string // "failed" | "recovered" tier string msg string err string } // recNotifier captures what would reach the hub. type recNotifier struct{ events []recEvent } func (r *recNotifier) BackupFailed(tier, message, errMsg string) { r.events = append(r.events, recEvent{"failed", tier, message, errMsg}) } func (r *recNotifier) BackupRecovered(tier, message string) { r.events = append(r.events, recEvent{"recovered", tier, message, ""}) } func (r *recNotifier) count(kind, tier string) int { n := 0 for _, e := range r.events { if e.kind == kind && (tier == "" || e.tier == tier) { n++ } } return n } // ── SCENARIO A — heard ONCE, not once per retry ────────────────────────────────────────────── // // COMPANION RED-PROOF (observed): drop the `n == 1` guard in noteTierFailure so every failure emits, // then step past each backoff — this fails with // // "a failing tier must be reported ONCE per run of failures, got 4" // // Restored. func TestNotify_FailingTierIsReportedOncePerRun(t *testing.T) { now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC) be := tierBackendFailing("felhom-pbs", "felhom-pbs") st := &fakeStacks{running: []string{"bookstack"}} l := breakerLoop(t, be, st, &now) rec := &recNotifier{} l.SetTierNotifier(rec) for i := 0; i < 4; i++ { // four ATTEMPTS, each past the previous backoff if err := l.runOnce(context.Background()); err != nil { t.Fatal(err) } now = now.Add(backoffFor(i+1) + time.Minute) } if got := rec.count("failed", "felhom-pbs"); got != 1 { t.Fatalf("a failing tier must be reported ONCE per run of failures, got %d", got) } if rec.events[0].tier != "felhom-pbs" { t.Fatalf("the TIER must be named — 'a backup failed' without saying which tier is not actionable; got %q", rec.events[0].tier) } if rec.events[0].err == "" { t.Fatal("the underlying error must travel with the event, or the operator has to go digging") } } // ── SCENARIO B — recovery is announced ─────────────────────────────────────────────────────── func TestNotify_RecoveryIsAnnounced(t *testing.T) { now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC) be := tierBackendFailing("felhom-pbs", "felhom-pbs") st := &fakeStacks{running: []string{"bookstack"}} l := breakerLoop(t, be, st, &now) rec := &recNotifier{} l.SetTierNotifier(rec) if err := l.runOnce(context.Background()); err != nil { // fail → reported t.Fatal(err) } be.mu.Lock() be.phases["felhom-pbs"] = []string{phaseDone} be.phaseIdx["felhom-pbs"] = 0 be.mu.Unlock() now = now.Add(backoffFor(1) + time.Minute) if err := l.runOnce(context.Background()); err != nil { t.Fatal(err) } if got := rec.count("recovered", "felhom-pbs"); got != 1 { t.Fatalf("an operator told a tier BROKE must be told it HEALED; recovery events = %d", got) } } // A tier that never failed must not emit a recovery on every healthy backup. func TestNotify_HealthyTierIsSilent(t *testing.T) { now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC) be := tierBackendFailing("nothing-fails", "local") st := &fakeStacks{running: []string{"bookstack"}} l := breakerLoop(t, be, st, &now) rec := &recNotifier{} l.SetTierNotifier(rec) for i := 0; i < 3; i++ { if err := l.runOnce(context.Background()); err != nil { t.Fatal(err) } be.mu.Lock() be.phaseIdx["local"] = 0 be.mu.Unlock() now = now.Add(5 * time.Minute) } if len(rec.events) != 0 { t.Fatalf("a healthy tier must emit NOTHING; got %+v", rec.events) } } // ── SCENARIO C — one failing tier does not mask another ────────────────────────────────────── // // The controller half: BOTH tiers must emit, each naming itself. (The hub half — the operator // cooldown keyed customerID:eventType, which would swallow the second tier's EMAIL within the hour — // is fixed hub-side by cooldownTierSuffix; see REPORT.md.) // // COMPANION RED-PROOF (observed): key the breaker on a constant instead of t.target (a single global // failure record) and this fails with // // "both tiers must be reported; got local=1 felhom-pbs=0" // // Restored. func TestNotify_BothFailingTiersAreReported(t *testing.T) { now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC) be := newTierBackend() for _, tr := range []string{"local", "felhom-pbs"} { be.tiers = append(be.tiers, BackupTier{Target: tr}) be.dueSet[tr] = true be.phases[tr] = []string{phaseFailed} } st := &fakeStacks{running: []string{"bookstack"}} l := breakerLoop(t, be, st, &now) rec := &recNotifier{} l.SetTierNotifier(rec) if err := l.runOnce(context.Background()); err != nil { t.Fatal(err) } gotLocal, gotPBS := rec.count("failed", "local"), rec.count("failed", "felhom-pbs") if gotLocal != 1 || gotPBS != 1 { t.Fatalf("both tiers must be reported; got local=%d felhom-pbs=%d — one broken tier must not mask another", gotLocal, gotPBS) } } // A nil notifier (unprovisioned guest) must not panic. func TestNotify_NilNotifierIsSafe(t *testing.T) { now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC) be := tierBackendFailing("felhom-pbs", "felhom-pbs") st := &fakeStacks{running: []string{"bookstack"}} l := breakerLoop(t, be, st, &now) // no SetTierNotifier if err := l.runOnce(context.Background()); err != nil { t.Fatalf("a loop with no notifier must still back up: %v", err) } } // ── SCENARIO E — quiesce does not blame the apps ───────────────────────────────────────────── // // COMPANION RED-PROOF (observed): make SuppressedStacks return nil unconditionally (the pre-fix // shape) and this fails with // // "R-97b: the apps we stopped for a backup must not be reported down; bookstack still alarmed" // // Restored. func TestSuppress_StacksWeStoppedAreNotReportedDown(t *testing.T) { now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC) be := tierBackendFailing("felhom-pbs", "felhom-pbs") st := &fakeStacks{running: []string{"bookstack", "immich"}} l := breakerLoop(t, be, st, &now) if err := l.runOnce(context.Background()); err != nil { t.Fatal(err) } // Immediately after the cycle: both stacks are inside the grace window. sup := l.SuppressedStacks() for _, name := range []string{"bookstack", "immich"} { if !sup[name] { t.Fatalf("R-97b: the apps we stopped for a backup must not be reported down; %s still alarmed", name) } } // A stack the cycle never touched is NOT suppressed — the window is scoped, not blanket. if sup["vaultwarden"] { t.Fatal("suppression must cover only the stacks THIS cycle stopped, not every app") } } // ── SCENARIO F — an app that really fails to restart STILL alarms ──────────────────────────── // // F is what makes E safe: a suite containing only E would pass against permanent suppression, which // turns a loud false alarm into a silent real one — the R-88 Scenario D trap in a new costume. // // COMPANION RED-PROOF (observed): drop the expiry check in SuppressedStacks (return every entry // regardless of `until`) and this fails with // // "CONTRACT VIOLATED: suppression must EXPIRE — bookstack is still suppressed 181s after the // cycle; permanent suppression is a silent real alarm" // // Restored. func TestSuppress_ContractGraceWindowExpires(t *testing.T) { now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC) be := tierBackendFailing("felhom-pbs", "felhom-pbs") st := &fakeStacks{running: []string{"bookstack"}} l := breakerLoop(t, be, st, &now) if err := l.runOnce(context.Background()); err != nil { t.Fatal(err) } if !l.SuppressedStacks()["bookstack"] { t.Fatal("setup: bookstack should be suppressed immediately after the cycle") } // Just inside the window — still suppressed. now = now.Add(quiesceAlarmGrace - time.Second) if !l.SuppressedStacks()["bookstack"] { t.Fatalf("suppression must hold for the full grace window (%s) — an app needs time to come up", quiesceAlarmGrace) } // Past the window — it MUST alarm again. now = now.Add(2 * time.Second) if l.SuppressedStacks()["bookstack"] { t.Fatalf("CONTRACT VIOLATED: suppression must EXPIRE — bookstack is still suppressed %s after the cycle; "+ "permanent suppression is a silent real alarm", quiesceAlarmGrace+time.Second) } } // A long-running cycle must keep suppressing throughout — a first full offsite snapshot legitimately // runs for hours, and the app is legitimately down that whole time. func TestSuppress_HoldsForTheWholeCycleHowLongItRuns(t *testing.T) { now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC) l := breakerLoop(t, newTierBackend(), &fakeStacks{}, &now) l.markQuiesced([]string{"immich"}) now = now.Add(6 * time.Hour) // still quiesced; no unquiesce yet if !l.SuppressedStacks()["immich"] { t.Fatal("an app stopped by a still-running cycle must stay suppressed, however long the backup takes") } l.markUnquiesced([]string{"immich"}) now = now.Add(quiesceAlarmGrace + time.Second) if l.SuppressedStacks()["immich"] { t.Fatal("once unquiesced, the grace window must still expire") } }