package quiesce import ( "context" "fmt" "testing" "time" ) // F-A1 (Campaign 8): the agent's HTTP 409 is its R-85 single-flight gate refusing while a // restore-test holds it — contention, not failure. The controller armed the R-88 breaker and // emailed the operator anyway, on both boxes. // // Scenarios E (contention is not failure), F (the tier stays due), G (a REAL error still alarms) // and H (unending contention IS a fault). G and H are what make E safe: a suite containing only E // passes against a controller that swallows every failure as contention and never alarms again. // recordingNotifier captures what would reach the operator. type recordingNotifier struct { failed []string recovered []string } func (r *recordingNotifier) BackupFailed(tier, message, errMsg string) { r.failed = append(r.failed, tier+"|"+message) } func (r *recordingNotifier) BackupRecovered(tier, message string) { r.recovered = append(r.recovered, tier) } func busyErr() error { return fmt.Errorf("%w: agentapi: POST /backup: HTTP 409", ErrTierBusy) } // Scenario E — a 409 must NOT arm the breaker and must NOT notify. // // RED-PROOF: delete the `errors.Is(err, ErrTierBusy)` branch in the start path → the 409 falls into // noteTierFailure and this fails on both counts ("breaker armed after a 409" and "operator was // notified"). func TestContention_409IsNotAFailure(t *testing.T) { be := &fakeBackend{due: true, startErr: busyErr()} st := &fakeStacks{running: []string{"paperless-ngx"}} l := testLoop(t, be, st) n := &recordingNotifier{} l.SetTierNotifier(n) if err := l.runOnce(context.Background()); err != nil { t.Fatalf("runOnce: %v", err) } if got := l.breaker.failuresFor(""); got != 0 { t.Errorf("breaker armed after a 409 (%d consecutive failures) — contention is not failure", got) } if len(n.failed) != 0 { t.Errorf("operator was notified for a 409: %v", n.failed) } // the apps must still come back — a refusal must never hold the customer's stacks down if len(st.startedNames()) != 1 { t.Errorf("stacks not restarted after a 409: %v", st.startedNames()) } } // Scenario F — the tier stays DUE and the next cycle retries once the gate frees. // // RED-PROOF: mark the tier satisfied on a 409 (e.g. call noteTierSuccess) → the retry never // happens and startCalls stays at 1. func TestContention_TierStaysDueAndRetries(t *testing.T) { be := &fakeBackend{due: true, startErr: busyErr()} st := &fakeStacks{running: []string{"paperless-ngx"}} l := testLoop(t, be, st) if err := l.runOnce(context.Background()); err != nil { t.Fatalf("cycle 1: %v", err) } // the restore-test finishes; the contention skip window must not outlive it artificially be.startErr = nil be.phases = []string{"done"} l.contention.clear("") if err := l.runOnce(context.Background()); err != nil { t.Fatalf("cycle 2: %v", err) } if be.startCalls != 2 { t.Errorf("backup attempted %d times, want 2 — the tier must stay due and retry after contention", be.startCalls) } } // Scenario G — a REAL failure (not 409) must still arm the breaker and notify, exactly as before. // // RED-PROOF: treat every start error as contention → this fails with "a real 500 was swallowed as // contention", which is the over-correction that would make E worthless. func TestContention_RealFailureStillAlarms(t *testing.T) { be := &fakeBackend{due: true, startErr: fmt.Errorf("agentapi: POST /backup: HTTP 500")} st := &fakeStacks{running: []string{"paperless-ngx"}} l := testLoop(t, be, st) n := &recordingNotifier{} l.SetTierNotifier(n) _ = l.runOnce(context.Background()) if got := l.breaker.failuresFor(""); got != 1 { t.Errorf("breaker did not arm on a real 500 (failures=%d) — a real 500 was swallowed as contention", got) } if len(n.failed) != 1 { t.Fatalf("operator was NOT notified of a real failure: %v", n.failed) } } // Scenario H — unending contention IS a fault, and it alarms ONCE. // // RED-PROOF: remove the contentionAlarmAfter check in noteTierContention → contention is silent // forever and this fails with "no alarm after ...". func TestContention_UnendingContentionAlarmsOnce(t *testing.T) { l := testLoop(t, &fakeBackend{}, &fakeStacks{}) n := &recordingNotifier{} l.SetTierNotifier(n) base := time.Now() now := base l.now = func() time.Time { return now } // contention starts, then persists past the bound l.noteTierContention("felhom-pbs", "felhom-pbs") for _, step := range []time.Duration{30 * time.Minute, 2 * time.Hour, contentionAlarmAfter + time.Minute, contentionAlarmAfter + 2*time.Hour} { now = base.Add(step) l.noteTierContention("felhom-pbs", "felhom-pbs") } if len(n.failed) == 0 { t.Fatalf("no alarm after %s of unbroken contention — that is a silence path, the exact thing this fix must not create", contentionAlarmAfter) } if len(n.failed) != 1 { t.Errorf("alarmed %d times, want exactly 1 (edge-triggered like R-97a): %v", len(n.failed), n.failed) } if got := n.failed[0]; !contains(got, "BLOCKED") { t.Errorf("the alarm must name CONTENTION, not a backup failure; got %q", got) } } // Below the bound, contention must stay quiet — otherwise every normal restore-test pages someone. func TestContention_BelowTheBoundIsQuiet(t *testing.T) { l := testLoop(t, &fakeBackend{}, &fakeStacks{}) n := &recordingNotifier{} l.SetTierNotifier(n) base := time.Now() now := base l.now = func() time.Time { return now } // the longest restore-test actually observed on the fleet was 12m01s for _, step := range []time.Duration{0, 5 * time.Minute, 12 * time.Minute, 90 * time.Minute} { now = base.Add(step) l.noteTierContention("felhom-pbs", "felhom-pbs") } if len(n.failed) != 0 { t.Errorf("contention alarmed before %s: %v — a normal restore-test must never page anyone", contentionAlarmAfter, n.failed) } } // A contended tier is dropped BEFORE any stack is stopped. Without this, removing the (wrong) // failure treatment would leave the loop re-quiescing every poll for the whole restore-test — // stopping the customer's apps each time, which is worse than the bug being fixed. func TestContention_ContendedTierIsDroppedBeforeStopping(t *testing.T) { be := &fakeBackend{due: true, startErr: busyErr()} st := &fakeStacks{running: []string{"paperless-ngx"}} l := testLoop(t, be, st) if err := l.runOnce(context.Background()); err != nil { t.Fatalf("cycle 1: %v", err) } stoppedAfterFirst := len(st.stoppedNames()) // immediately after, still inside contentionRetryAfter if err := l.runOnce(context.Background()); err != nil { t.Fatalf("cycle 2: %v", err) } if got := len(st.stoppedNames()); got != stoppedAfterFirst { t.Errorf("stacks were stopped again while the tier was still contended (%d -> %d) — that is app thrash", stoppedAfterFirst, got) } if be.startCalls != 1 { t.Errorf("backup re-attempted while contended (%d calls) — the tier should have been dropped before quiescing", be.startCalls) } } // The tracker's own contract: clear() reports whether there was a run to end. func TestContentionTracker_ClearReportsTheEdge(t *testing.T) { c := newContentionTracker() if c.clear("x") { t.Error("clear() on an unknown target reported an edge") } c.note("x", time.Now()) if !c.clear("x") { t.Error("clear() did not report the edge after a noted contention") } } func contains(hay, needle string) bool { for i := 0; i+len(needle) <= len(hay); i++ { if hay[i:i+len(needle)] == needle { return true } } return false }