package backup import ( "errors" "io" "log" "testing" "time" ) // ── R-330 — the nightly volume dump must not alarm about the app it is holding down ────────────── // // The defect these pin, measured on demo-hp 2026-08-30 with controller 0.223.0: `DumpAppVolumesSafe` // stopped a stack for ~13 s to tar its volumes while the `deadapp-check` job ran every 30 s, so the // scan caught the stack mid-cycle and pushed `app_start_failed` to the customer. 61 e-mails. // // The tests are split by what they must not lose: // - the SUPPRESSION exists and covers the whole window (the fix), and // - it can NEVER latch (the thing the fix must not break) — a restart that failed, a hold nothing // released, and a stranded set from an earlier operation each let the alarm through. // // RED-PROOF (run 2026-08-30, recorded in REPORT.md): removing the `g.markStopped(stackNames)` call // from `AppStopGuard.Begin` fails TestVolumeDump_SuppressesTheAlarmForTheAppItIsHolding with // `suppressed at stop = map[]` — the exact pre-fix shape that produced the e-mails. // suppressWatchProvider drives the REAL DumpAppVolumesSafe and records the alarm-suppression set at // the two moments that matter: while the app is stopped, and at the restart call. Asserting on the // suppression set (what the dead-app scanner actually reads) rather than on a log line is the // standing-rule-3 positive observable — and it is the CONSEQUENCE, not the mechanism. type suppressWatchProvider struct { StackDataProvider guard *AppStopGuard suppressedAtStop map[string]bool suppressedAtStart map[string]bool startErr error } func (p *suppressWatchProvider) GetDockerVolumes(string) []string { return nil } func (p *suppressWatchProvider) StopStack(string) error { p.suppressedAtStop = p.guard.SuppressedStacks() return nil } func (p *suppressWatchProvider) StartStack(string) error { p.suppressedAtStart = p.guard.SuppressedStacks() return p.startErr } // newSuppressManager builds a Manager over a real guard and returns both. func newSuppressManager(t *testing.T, p *suppressWatchProvider) *Manager { t.Helper() dir := t.TempDir() lg := log.New(io.Discard, "", 0) m := &Manager{logger: lg, stackProvider: p, systemDataPath: dir} m.appStop = NewAppStopGuard(markerPath(dir), lg) p.guard = m.appStop return m } func TestVolumeDump_SuppressesTheAlarmForTheAppItIsHolding(t *testing.T) { // THE FIX. Through the production path, not by calling Begin from the test: an earlier sibling // test in this package was rewritten for exactly that reason — proving the guard works is not // proving DumpAppVolumesSafe uses it. p := &suppressWatchProvider{} m := newSuppressManager(t, p) if err := m.DumpAppVolumesSafe("bookstack"); err != nil { t.Fatalf("DumpAppVolumesSafe: %v", err) } if !p.suppressedAtStop["bookstack"] { t.Fatalf("suppressed at stop = %v, want bookstack — the dead-app scan runs every 30s and the "+ "stack is down for ~13s, so an unsuppressed window is the false `app_start_failed` e-mail "+ "the customer received nightly", p.suppressedAtStop) } if !p.suppressedAtStart["bookstack"] { t.Fatalf("suppressed at restart = %v, want bookstack — the app is STILL down at this point", p.suppressedAtStart) } // And it is still suppressed after the restart returns: R-97b's BookStack alarmed while `starting` // / `unhealthy`, which is neither stopped nor healthy, so a window that ends at the restart CALL // closes too early to fix anything. if !m.appStop.SuppressedStacks()["bookstack"] { t.Fatal("the suppression ended the instant the restart returned — an app that is up but not " + "yet healthy still reads as down, which is the exact shape R-97b was filed for") } } func TestVolumeDump_SuppressionExpiresSoARealOutageStillAlarms(t *testing.T) { // The window is a BOUNDED DELAY in reporting a real failure, never its loss. Without this the fix // would trade a loud false alarm for a silent real one — F-CRIT-1 and R-88 Scenario D. p := &suppressWatchProvider{} m := newSuppressManager(t, p) base := time.Now() m.appStop.now = func() time.Time { return base } if err := m.DumpAppVolumesSafe("bookstack"); err != nil { t.Fatalf("DumpAppVolumesSafe: %v", err) } if !m.appStop.SuppressedStacks()["bookstack"] { t.Fatal("not suppressed immediately after the restart") } // One second before the grace closes: still suppressed. m.appStop.now = func() time.Time { return base.Add(appStopAlarmGrace - time.Second) } if !m.appStop.SuppressedStacks()["bookstack"] { t.Fatalf("the grace closed early — a stack restarted %s ago must still be exempt", appStopAlarmGrace-time.Second) } // At the grace boundary: the alarm owns it again. m.appStop.now = func() time.Time { return base.Add(appStopAlarmGrace) } if m.appStop.SuppressedStacks()["bookstack"] { t.Fatalf("still suppressed %s after the restart — an app that genuinely failed to come back "+ "would never be reported", appStopAlarmGrace) } } func TestVolumeDump_FailedRestartAlarmsImmediately(t *testing.T) { // The primary anti-latch mechanism. We stopped it, we could not give it back: it is genuinely // down, and it must alarm on the NEXT scan — not after any grace at all. p := &suppressWatchProvider{startErr: errors.New("compose up failed")} m := newSuppressManager(t, p) if err := m.DumpAppVolumesSafe("bookstack"); err == nil { t.Fatal("a failed restart must surface as an error") } if got := m.appStop.SuppressedStacks(); got["bookstack"] { t.Fatalf("suppressed = %v after a restart that FAILED — the app is down and the alarm is the "+ "only thing that would tell anyone, which is F-CRIT-1 exactly", got) } // The durable marker is a SEPARATE concern and must be untouched by the release: it is what the // next startup retries from. Losing it here would trade a false alarm for a lost recovery. if !markerExists(t, m.systemDataPath) { t.Fatal("ReleaseFailed also cleared the crash marker — the next startup would not retry the restart") } } func TestSuppression_CannotOutliveTheBackstop(t *testing.T) { // The belt-and-braces for a hold nothing ever released. `End()` runs only on a restart that // SUCCEEDED, so a future failure path that forgets ReleaseFailed would otherwise silence an app // forever. Exceeding the backstop means something is wrong, and the right answer when something // is wrong is to let the alarm through. base := time.Now() g := NewAppStopGuard(markerPath(t.TempDir()), log.New(io.Discard, "", 0)) g.now = func() time.Time { return base } if err := g.Begin("volume-dump:romm", ReasonVolumeDump, []string{"romm"}); err != nil { t.Fatalf("Begin: %v", err) } if !g.SuppressedStacks()["romm"] { t.Fatal("not suppressed while held") } g.now = func() time.Time { return base.Add(appStopMaxHold - time.Minute) } if !g.SuppressedStacks()["romm"] { t.Fatalf("the backstop fired early — a legitimate long operation would start alarming mid-run") } g.now = func() time.Time { return base.Add(appStopMaxHold) } if g.SuppressedStacks()["romm"] { t.Fatalf("a hold open-ended for %s is still suppressing the alarm — nothing released it and "+ "nothing ever will", appStopMaxHold) } } func TestBeginReplacesThePreviousOperationsSet(t *testing.T) { // The marker file holds ONE operation, so a new Begin proves the previous one is over. Without // this, a set stranded by an operation that died between Begin and End would suppress its stacks // for the whole appStopMaxHold, even though a later operation has since taken over. g := NewAppStopGuard(markerPath(t.TempDir()), log.New(io.Discard, "", 0)) if err := g.Begin("volume-dump:romm", ReasonVolumeDump, []string{"romm"}); err != nil { t.Fatalf("Begin romm: %v", err) } if err := g.Begin("volume-dump:kimai", ReasonVolumeDump, []string{"kimai"}); err != nil { t.Fatalf("Begin kimai: %v", err) } got := g.SuppressedStacks() if got["romm"] { t.Fatalf("suppressed = %v — romm's hold survived into a later operation that is not holding it", got) } if !got["kimai"] { t.Fatalf("suppressed = %v, want kimai — the current operation's own stack is not exempt", got) } } func TestSuppression_FailedBeginSuppressesNothing(t *testing.T) { // A Begin whose write failed means the caller REFUSES to stop the app (DumpAppVolumesSafe returns // early). Nothing is stopped, so nothing may be exempt — suppressing here would hide a genuinely // dead app that this operation never touched. g := NewAppStopGuard("/proc/felhom-nonexistent-dir/appstop.json", log.New(io.Discard, "", 0)) if err := g.Begin("volume-dump:romm", ReasonVolumeDump, []string{"romm"}); err == nil { t.Skip("the unwritable path became writable — this environment cannot exercise the failure") } if got := g.SuppressedStacks(); got["romm"] { t.Fatalf("suppressed = %v after a Begin that FAILED — the app was never stopped", got) } } func TestNilGuardSuppressesNothing(t *testing.T) { // An unprovisioned guest has no guard. Suppressing nothing is the correct default, and nil-safety // is what lets the caller in main.go stay branch-free. var g *AppStopGuard if got := g.SuppressedStacks(); len(got) != 0 { t.Fatalf("a nil guard suppressed %v", got) } g.ReleaseFailed("romm") // must not panic }