F-CRIT-1 + F-A1: one alarm that never fired, one that fired wrongly (v0.179.0)

F-CRIT-1 — an app that failed to restart after a quiesce never alarmed, for two
independent reasons, either of which alone kept it dead:
  1. restartAll returned nothing, so the failure was logged and dropped and no
     caller could learn a customer's app had not come back. It now returns the
     stacks that failed; both call sites record the outcome.
  2. classifyRunStates whitelists StateStopped on invariant I1 ('StateStopped
     means the user stopped it'). The quiesce loop stops stacks by the same
     compose-down path, so a failed restart is also StateStopped and was
     whitelisted into silence. Loop.FailedRestarts() is now the only thing that
     lifts the whitelist, so genuine user stops stay silent (v0.164.0 pinned).

F-A1 — HTTP 409 is the agent's single-flight gate refusing while a restore-test
holds it, not a failure. agentapi now returns a typed *StatusError on POST, the
adapter maps 409 -> quiesce.ErrTierBusy, and the loop defers: no breaker, no
event, no operator email, tier stays DUE.

Two traps avoided. Silence: contention outliving contentionAlarmAfter (3h, set
by the agent's own 120m PBS restore-test ceiling) raises its own BLOCKED signal.
App thrash: removing the failure treatment also removes the breaker's deferral,
so a contended tier is dropped BEFORE anything stops (contentionRetryAfter 15m,
against a 12m01s longest observed restore-test).

Three comments corrected; the invariant rule added to both CLAUDE.md copies.
Six red-proofs, all observed failing.

controller/internal/quiesce/failed_restart_test.go

@@ -0,0 +1,130 @@
+package quiesce
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+)
+
+// F-CRIT-1 cause 1 (Campaign 8): `restartAll` used to return nothing, so a failed restart was
+// logged and dropped and no caller could ever learn a customer's app had not come back.
+//
+// Scenario D — the outcome must reach the caller.
+
+// stacksWithStartFailures is a Stacks whose StartStack fails for named stacks.
+type stacksWithStartFailures struct {
+	mu       sync.Mutex
+	running  []string
+	startErr map[string]error
+	started  []string
+	stopped  []string
+}
+
+func (f *stacksWithStartFailures) RunningAppStacks() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.running...)
+}
+func (f *stacksWithStartFailures) StopStack(name string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.stopped = append(f.stopped, name)
+	return nil
+}
+func (f *stacksWithStartFailures) StartStack(name string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.started = append(f.started, name)
+	if f.startErr != nil {
+		return f.startErr[name]
+	}
+	return nil
+}
+
+// Scenario D — restartAll RETURNS the stacks that would not start.
+//
+// RED-PROOF: revert restartAll to `func (l *Loop) restartAll(stacks []string)` with the error only
+// logged → this file does not compile ("l.restartAll(...) used as value"), which is the loudest
+// possible form of "the caller cannot learn".
+func TestRestartAll_ReturnsTheStacksThatFailed(t *testing.T) {
+	boom := errors.New("compose up: exit code 1")
+	st := &stacksWithStartFailures{startErr: map[string]error{"immich": boom}}
+	l := testLoop(t, &fakeBackend{}, st)
+
+	failed := l.restartAll([]string{"bookstack", "immich", "docmost"})
+
+	if len(failed) != 1 || failed[0] != "immich" {
+		t.Fatalf("restartAll returned %v, want exactly [immich]", failed)
+	}
+	// every stack must still have been ATTEMPTED — one failure must not abort the rest
+	if len(st.started) != 3 {
+		t.Errorf("attempted %d restarts, want 3 — a failure must not stop the loop restarting the others", len(st.started))
+	}
+}
+
+// Scenario A (cause 1) — a failed restart is REMEMBERED on the Loop, so the classifier can see it.
+//
+// RED-PROOF: drop the `l.noteRestartOutcome(...)` call at the unquiesce site → FailedRestarts() is
+// empty and this fails with "want immich to be recorded as a failed restart".
+func TestFailedRestarts_RecordedAfterACycle(t *testing.T) {
+	boom := errors.New("compose up: exit code 1")
+	st := &stacksWithStartFailures{
+		running:  []string{"bookstack", "immich"},
+		startErr: map[string]error{"immich": boom},
+	}
+	be := &fakeBackend{due: true, phases: []string{"done"}}
+	l := testLoop(t, be, st)
+
+	if err := l.runOnce(context.Background()); err != nil {
+		t.Fatalf("runOnce: %v", err)
+	}
+
+	failed := l.FailedRestarts()
+	if !failed["immich"] {
+		t.Errorf("want immich to be recorded as a failed restart, got %v", failed)
+	}
+	if failed["bookstack"] {
+		t.Errorf("bookstack restarted fine and must NOT be recorded as failed, got %v", failed)
+	}
+}
+
+// The flag must CLEAR when a later cycle restarts the stack successfully — otherwise a fixed app
+// carries its old failure forever and would alarm the next time it is legitimately stopped.
+func TestFailedRestarts_ClearedOnALaterSuccess(t *testing.T) {
+	boom := errors.New("compose up: exit code 1")
+	st := &stacksWithStartFailures{
+		running:  []string{"immich"},
+		startErr: map[string]error{"immich": boom},
+	}
+	be := &fakeBackend{due: true, phases: []string{"done"}}
+	l := testLoop(t, be, st)
+
+	if err := l.runOnce(context.Background()); err != nil {
+		t.Fatalf("cycle 1: %v", err)
+	}
+	if !l.FailedRestarts()["immich"] {
+		t.Fatal("precondition: immich should be flagged after the failing cycle")
+	}
+
+	// the operator fixes the app; the next cycle restarts it cleanly
+	st.mu.Lock()
+	st.startErr = nil
+	st.mu.Unlock()
+	if err := l.runOnce(context.Background()); err != nil {
+		t.Fatalf("cycle 2: %v", err)
+	}
+
+	if l.FailedRestarts()["immich"] {
+		t.Error("the flag survived a SUCCESSFUL restart — a fixed app would carry its old failure forever")
+	}
+}
+
+// FailedRestarts must be nil-safe on a nil *Loop, for the same reason SuppressedStacks is: an
+// unprovisioned guest has no loop and the caller must not need a branch.
+func TestFailedRestarts_NilSafe(t *testing.T) {
+	var l *Loop
+	if got := l.FailedRestarts(); got != nil {
+		t.Errorf("nil Loop returned %v, want nil", got)
+	}
+}