R-97: a failing backup is heard, and stops blaming the apps (v0.177.0)
R-97a: internal/quiesce had no route to the hub at all — three failed whole-guest backups on 2026-07-27 produced zero events. TierNotifier is a seam (not an import), wired by an init-only setter because main.go builds the notifier after the loop. Edge-triggered: the failure fires when the R-88 breaker ARMS, not per retry, and recovery rides recordSuccess's existing bool. Uses NEW operator-only event types; reusing backup_failed would have emailed the customer in Hungarian about a backup they cannot act on, since it has a customerMessages entry and is in live enabled_events. Requires hub >= v0.78.0. R-97b: v0.164.0's state filter cannot see an app caught MID-RESTART, which is how BookStack alarmed. The fix is a suppression window keyed to the quiesce CYCLE, consumed at the same single derivation point. 180s grace, derived from the deploy flow's 120s health timeout and Mealie's 60s start_period; it expires, so an app that genuinely fails to come back still alarms.
This commit is contained in:
@@ -0,0 +1,251 @@
|
||||
package quiesce
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// R-97a/b — a failing backup must be HEARD, and must not blame the apps.
|
||||
//
|
||||
// Both halves are inert-seam risks, so nothing here asserts "no error occurred". The failure mode
|
||||
// this suite exists to catch is a route that silently does not exist — which is exactly what an
|
||||
// empty event list looks like.
|
||||
|
||||
// recEvent is one captured hub event.
|
||||
type recEvent struct {
|
||||
kind string // "failed" | "recovered"
|
||||
tier string
|
||||
msg string
|
||||
err string
|
||||
}
|
||||
|
||||
// recNotifier captures what would reach the hub.
|
||||
type recNotifier struct{ events []recEvent }
|
||||
|
||||
func (r *recNotifier) BackupFailed(tier, message, errMsg string) {
|
||||
r.events = append(r.events, recEvent{"failed", tier, message, errMsg})
|
||||
}
|
||||
func (r *recNotifier) BackupRecovered(tier, message string) {
|
||||
r.events = append(r.events, recEvent{"recovered", tier, message, ""})
|
||||
}
|
||||
func (r *recNotifier) count(kind, tier string) int {
|
||||
n := 0
|
||||
for _, e := range r.events {
|
||||
if e.kind == kind && (tier == "" || e.tier == tier) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ── SCENARIO A — heard ONCE, not once per retry ──────────────────────────────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): drop the `n == 1` guard in noteTierFailure so every failure emits,
|
||||
// then step past each backoff — this fails with
|
||||
//
|
||||
// "a failing tier must be reported ONCE per run of failures, got 4"
|
||||
//
|
||||
// Restored.
|
||||
func TestNotify_FailingTierIsReportedOncePerRun(t *testing.T) {
|
||||
now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
||||
be := tierBackendFailing("felhom-pbs", "felhom-pbs")
|
||||
st := &fakeStacks{running: []string{"bookstack"}}
|
||||
l := breakerLoop(t, be, st, &now)
|
||||
rec := &recNotifier{}
|
||||
l.SetTierNotifier(rec)
|
||||
|
||||
for i := 0; i < 4; i++ { // four ATTEMPTS, each past the previous backoff
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(backoffFor(i+1) + time.Minute)
|
||||
}
|
||||
if got := rec.count("failed", "felhom-pbs"); got != 1 {
|
||||
t.Fatalf("a failing tier must be reported ONCE per run of failures, got %d", got)
|
||||
}
|
||||
if rec.events[0].tier != "felhom-pbs" {
|
||||
t.Fatalf("the TIER must be named — 'a backup failed' without saying which tier is not actionable; got %q", rec.events[0].tier)
|
||||
}
|
||||
if rec.events[0].err == "" {
|
||||
t.Fatal("the underlying error must travel with the event, or the operator has to go digging")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO B — recovery is announced ───────────────────────────────────────────────────────
|
||||
func TestNotify_RecoveryIsAnnounced(t *testing.T) {
|
||||
now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
||||
be := tierBackendFailing("felhom-pbs", "felhom-pbs")
|
||||
st := &fakeStacks{running: []string{"bookstack"}}
|
||||
l := breakerLoop(t, be, st, &now)
|
||||
rec := &recNotifier{}
|
||||
l.SetTierNotifier(rec)
|
||||
|
||||
if err := l.runOnce(context.Background()); err != nil { // fail → reported
|
||||
t.Fatal(err)
|
||||
}
|
||||
be.mu.Lock()
|
||||
be.phases["felhom-pbs"] = []string{phaseDone}
|
||||
be.phaseIdx["felhom-pbs"] = 0
|
||||
be.mu.Unlock()
|
||||
now = now.Add(backoffFor(1) + time.Minute)
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := rec.count("recovered", "felhom-pbs"); got != 1 {
|
||||
t.Fatalf("an operator told a tier BROKE must be told it HEALED; recovery events = %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A tier that never failed must not emit a recovery on every healthy backup.
|
||||
func TestNotify_HealthyTierIsSilent(t *testing.T) {
|
||||
now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
||||
be := tierBackendFailing("nothing-fails", "local")
|
||||
st := &fakeStacks{running: []string{"bookstack"}}
|
||||
l := breakerLoop(t, be, st, &now)
|
||||
rec := &recNotifier{}
|
||||
l.SetTierNotifier(rec)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
be.mu.Lock()
|
||||
be.phaseIdx["local"] = 0
|
||||
be.mu.Unlock()
|
||||
now = now.Add(5 * time.Minute)
|
||||
}
|
||||
if len(rec.events) != 0 {
|
||||
t.Fatalf("a healthy tier must emit NOTHING; got %+v", rec.events)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO C — one failing tier does not mask another ──────────────────────────────────────
|
||||
//
|
||||
// The controller half: BOTH tiers must emit, each naming itself. (The hub half — the operator
|
||||
// cooldown keyed customerID:eventType, which would swallow the second tier's EMAIL within the hour —
|
||||
// is fixed hub-side by cooldownTierSuffix; see REPORT.md.)
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): key the breaker on a constant instead of t.target (a single global
|
||||
// failure record) and this fails with
|
||||
//
|
||||
// "both tiers must be reported; got local=1 felhom-pbs=0"
|
||||
//
|
||||
// Restored.
|
||||
func TestNotify_BothFailingTiersAreReported(t *testing.T) {
|
||||
now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
||||
be := newTierBackend()
|
||||
for _, tr := range []string{"local", "felhom-pbs"} {
|
||||
be.tiers = append(be.tiers, BackupTier{Target: tr})
|
||||
be.dueSet[tr] = true
|
||||
be.phases[tr] = []string{phaseFailed}
|
||||
}
|
||||
st := &fakeStacks{running: []string{"bookstack"}}
|
||||
l := breakerLoop(t, be, st, &now)
|
||||
rec := &recNotifier{}
|
||||
l.SetTierNotifier(rec)
|
||||
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gotLocal, gotPBS := rec.count("failed", "local"), rec.count("failed", "felhom-pbs")
|
||||
if gotLocal != 1 || gotPBS != 1 {
|
||||
t.Fatalf("both tiers must be reported; got local=%d felhom-pbs=%d — one broken tier must not mask another",
|
||||
gotLocal, gotPBS)
|
||||
}
|
||||
}
|
||||
|
||||
// A nil notifier (unprovisioned guest) must not panic.
|
||||
func TestNotify_NilNotifierIsSafe(t *testing.T) {
|
||||
now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
||||
be := tierBackendFailing("felhom-pbs", "felhom-pbs")
|
||||
st := &fakeStacks{running: []string{"bookstack"}}
|
||||
l := breakerLoop(t, be, st, &now) // no SetTierNotifier
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatalf("a loop with no notifier must still back up: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO E — quiesce does not blame the apps ─────────────────────────────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): make SuppressedStacks return nil unconditionally (the pre-fix
|
||||
// shape) and this fails with
|
||||
//
|
||||
// "R-97b: the apps we stopped for a backup must not be reported down; bookstack still alarmed"
|
||||
//
|
||||
// Restored.
|
||||
func TestSuppress_StacksWeStoppedAreNotReportedDown(t *testing.T) {
|
||||
now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
||||
be := tierBackendFailing("felhom-pbs", "felhom-pbs")
|
||||
st := &fakeStacks{running: []string{"bookstack", "immich"}}
|
||||
l := breakerLoop(t, be, st, &now)
|
||||
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Immediately after the cycle: both stacks are inside the grace window.
|
||||
sup := l.SuppressedStacks()
|
||||
for _, name := range []string{"bookstack", "immich"} {
|
||||
if !sup[name] {
|
||||
t.Fatalf("R-97b: the apps we stopped for a backup must not be reported down; %s still alarmed", name)
|
||||
}
|
||||
}
|
||||
// A stack the cycle never touched is NOT suppressed — the window is scoped, not blanket.
|
||||
if sup["vaultwarden"] {
|
||||
t.Fatal("suppression must cover only the stacks THIS cycle stopped, not every app")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO F — an app that really fails to restart STILL alarms ────────────────────────────
|
||||
//
|
||||
// F is what makes E safe: a suite containing only E would pass against permanent suppression, which
|
||||
// turns a loud false alarm into a silent real one — the R-88 Scenario D trap in a new costume.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): drop the expiry check in SuppressedStacks (return every entry
|
||||
// regardless of `until`) and this fails with
|
||||
//
|
||||
// "CONTRACT VIOLATED: suppression must EXPIRE — bookstack is still suppressed 181s after the
|
||||
// cycle; permanent suppression is a silent real alarm"
|
||||
//
|
||||
// Restored.
|
||||
func TestSuppress_ContractGraceWindowExpires(t *testing.T) {
|
||||
now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
||||
be := tierBackendFailing("felhom-pbs", "felhom-pbs")
|
||||
st := &fakeStacks{running: []string{"bookstack"}}
|
||||
l := breakerLoop(t, be, st, &now)
|
||||
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !l.SuppressedStacks()["bookstack"] {
|
||||
t.Fatal("setup: bookstack should be suppressed immediately after the cycle")
|
||||
}
|
||||
// Just inside the window — still suppressed.
|
||||
now = now.Add(quiesceAlarmGrace - time.Second)
|
||||
if !l.SuppressedStacks()["bookstack"] {
|
||||
t.Fatalf("suppression must hold for the full grace window (%s) — an app needs time to come up", quiesceAlarmGrace)
|
||||
}
|
||||
// Past the window — it MUST alarm again.
|
||||
now = now.Add(2 * time.Second)
|
||||
if l.SuppressedStacks()["bookstack"] {
|
||||
t.Fatalf("CONTRACT VIOLATED: suppression must EXPIRE — bookstack is still suppressed %s after the cycle; "+
|
||||
"permanent suppression is a silent real alarm", quiesceAlarmGrace+time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// A long-running cycle must keep suppressing throughout — a first full offsite snapshot legitimately
|
||||
// runs for hours, and the app is legitimately down that whole time.
|
||||
func TestSuppress_HoldsForTheWholeCycleHowLongItRuns(t *testing.T) {
|
||||
now := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
||||
l := breakerLoop(t, newTierBackend(), &fakeStacks{}, &now)
|
||||
l.markQuiesced([]string{"immich"})
|
||||
now = now.Add(6 * time.Hour) // still quiesced; no unquiesce yet
|
||||
if !l.SuppressedStacks()["immich"] {
|
||||
t.Fatal("an app stopped by a still-running cycle must stay suppressed, however long the backup takes")
|
||||
}
|
||||
l.markUnquiesced([]string{"immich"})
|
||||
now = now.Add(quiesceAlarmGrace + time.Second)
|
||||
if l.SuppressedStacks()["immich"] {
|
||||
t.Fatal("once unquiesced, the grace window must still expire")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user