F-CRIT-1 + F-A1: one alarm that never fired, one that fired wrongly (v0.179.0)

F-CRIT-1 — an app that failed to restart after a quiesce never alarmed, for two
independent reasons, either of which alone kept it dead:
  1. restartAll returned nothing, so the failure was logged and dropped and no
     caller could learn a customer's app had not come back. It now returns the
     stacks that failed; both call sites record the outcome.
  2. classifyRunStates whitelists StateStopped on invariant I1 ('StateStopped
     means the user stopped it'). The quiesce loop stops stacks by the same
     compose-down path, so a failed restart is also StateStopped and was
     whitelisted into silence. Loop.FailedRestarts() is now the only thing that
     lifts the whitelist, so genuine user stops stay silent (v0.164.0 pinned).

F-A1 — HTTP 409 is the agent's single-flight gate refusing while a restore-test
holds it, not a failure. agentapi now returns a typed *StatusError on POST, the
adapter maps 409 -> quiesce.ErrTierBusy, and the loop defers: no breaker, no
event, no operator email, tier stays DUE.

Two traps avoided. Silence: contention outliving contentionAlarmAfter (3h, set
by the agent's own 120m PBS restore-test ceiling) raises its own BLOCKED signal.
App thrash: removing the failure treatment also removes the breaker's deferral,
so a contended tier is dropped BEFORE anything stops (contentionRetryAfter 15m,
against a 12m01s longest observed restore-test).

Three comments corrected; the invariant rule added to both CLAUDE.md copies.
Six red-proofs, all observed failing.

controller/cmd/controller/classify_runstates_test.go

@@ -56,7 +56,7 @@ func TestClassifyRunStates_StoppedIsSuppressed(t *testing.T) {
 		stack("nextcloud", stacks.StateDegraded, true, false),
 	}
 
-	dead, states := classifyRunStates(sts, nil)
+	dead, states := classifyRunStates(sts, nil, nil)
 
 	gotDead := deadNames(dead)
 	if len(gotDead) != 2 || !gotDead["immich"] || !gotDead["nextcloud"] {
@@ -90,7 +90,7 @@ func TestClassifyRunStates_FaultParity(t *testing.T) {
 		stack("nextcloud", stacks.StateDegraded, true, false),
 	}
 
-	dead, states := classifyRunStates(sts, nil)
+	dead, states := classifyRunStates(sts, nil, nil)
 
 	gotDead := deadNames(dead)
 	if len(gotDead) != 2 || !gotDead["immich"] || !gotDead["nextcloud"] {
@@ -116,7 +116,7 @@ func TestClassifyRunStates_SkipsDeployingAndUndeployed(t *testing.T) {
 		stack("mid", stacks.StateDeploying, true, true), // mid-deploy → skipped
 		stack("gone", stacks.StateExited, false, false), // not deployed → skipped
 	}
-	dead, states := classifyRunStates(sts, nil)
+	dead, states := classifyRunStates(sts, nil, nil)
 	if len(dead) != 0 || len(states) != 0 {
 		t.Fatalf("deploying and undeployed stacks must be skipped, got dead=%+v states=%+v", dead, states)
 	}

controller/cmd/controller/failed_restart_classify_test.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
+)
+
+// F-CRIT-1 cause 2 (Campaign 8): `classifyRunStates` whitelisted StateStopped on invariant I1
+// ("StateStopped means the USER stopped it"). The quiesce loop broke I1 by stopping stacks via the
+// same `docker compose down` path, so a stack quiesce stopped and then FAILED to restart was also
+// StateStopped — and was whitelisted into total silence. Live evidence: a customer app dead
+// indefinitely, no banner, no event, no email, while the dead-app scanner ran 11 times over it.
+//
+// The two cases are byte-identical on the Docker side. The ONLY thing that separates them is that
+// the quiesce loop knows it tried to restart and could not — `failedRestart` is that knowledge.
+
+// Scenario A (cause 2) — a stack quiesce failed to restart MUST alarm, despite being StateStopped.
+//
+// RED-PROOF: restore the unconditional whitelist (`down := IsDownState(st.State) &&
+// st.State != stacks.StateStopped && !quiesced[st.Name]`) → immich reports Down=false and stays out
+// of the dead list, and this fails with "a stack that FAILED to restart is silent".
+func TestClassifyRunStates_FailedRestartAlarmsDespiteStateStopped(t *testing.T) {
+	sts := []stacks.Stack{
+		stack("bookstack", stacks.StateRunning, true, false),
+		stack("immich", stacks.StateStopped, true, false), // quiesce stopped it; restart FAILED
+	}
+	failed := map[string]bool{"immich": true}
+
+	dead, states := classifyRunStates(sts, nil, failed)
+
+	if !downByName(states)["immich"] {
+		t.Error("a stack that FAILED to restart is silent (Down=false) — this is F-CRIT-1")
+	}
+	if !deadNames(dead)["immich"] {
+		t.Error("a stack that FAILED to restart is absent from the dashboard dead-list — this is F-CRIT-1")
+	}
+	if downByName(states)["bookstack"] {
+		t.Error("a healthy running stack was marked down")
+	}
+}
+
+// Scenario B — a DELIBERATE user stop must still be silent. This pins v0.164.0 and is what stops
+// the fix above from becoming a regression.
+//
+// RED-PROOF: make the whitelist unconditional in the other direction (drop the `&& !failedRestart`
+// term, i.e. treat every StateStopped as a failed restart) → cwa alarms and this fails with
+// "a deliberate user stop alarmed".
+func TestClassifyRunStates_UserStopStillSilent(t *testing.T) {
+	sts := []stacks.Stack{
+		stack("cwa", stacks.StateStopped, true, false), // the user stopped this from the UI
+		stack("immich", stacks.StateStopped, true, false),
+	}
+	// only immich failed to restart; cwa was never touched by a quiesce
+	failed := map[string]bool{"immich": true}
+
+	dead, states := classifyRunStates(sts, nil, failed)
+	down := downByName(states)
+
+	if down["cwa"] || deadNames(dead)["cwa"] {
+		t.Error("a deliberate user stop alarmed — that is the v0.164.0 regression this must not reintroduce")
+	}
+	if !down["immich"] {
+		t.Error("the failed restart went silent")
+	}
+}
+
+// Scenario B, stronger form — with NO failed restarts at all, behaviour is byte-identical to
+// v0.164.0: every StateStopped is silent.
+func TestClassifyRunStates_NoFailedRestartsIsV0164Behaviour(t *testing.T) {
+	sts := []stacks.Stack{
+		stack("radarr", stacks.StateRunning, true, false),
+		stack("cwa", stacks.StateStopped, true, false),
+		stack("immich", stacks.StateExited, true, false),
+		stack("nextcloud", stacks.StateDegraded, true, false),
+	}
+
+	dead, states := classifyRunStates(sts, nil, nil)
+	down := downByName(states)
+
+	if down["cwa"] {
+		t.Error("stopped alarmed with no failed restarts — v0.164.0 behaviour broken")
+	}
+	if !down["immich"] || !down["nextcloud"] {
+		t.Error("a genuine fault (exited/degraded) stopped alarming")
+	}
+	if got := len(deadNames(dead)); got != 2 {
+		t.Errorf("dead list has %d entries, want exactly {immich, nextcloud}", got)
+	}
+}
+
+// Scenario C — during the R-97b grace window the stack is suppressed even if its restart failed.
+// The grace exists so a slow-starting app is not called dead; it EXPIRES, and the alarm follows.
+//
+// RED-PROOF: drop the `&& !quiesced[st.Name]` term → the app alarms mid-restart on every normal
+// backup, which is the false-alarm R-97b was built to remove.
+func TestClassifyRunStates_GraceWindowStillSuppresses(t *testing.T) {
+	sts := []stacks.Stack{stack("immich", stacks.StateStopped, true, false)}
+	quiesced := map[string]bool{"immich": true} // still inside quiesceAlarmGrace
+	failed := map[string]bool{"immich": true}   // and we already know the restart failed
+
+	dead, states := classifyRunStates(sts, quiesced, failed)
+
+	if downByName(states)["immich"] {
+		t.Error("alarmed while still inside the grace window — R-97b Scenario E broken")
+	}
+	if len(dead) != 0 {
+		t.Errorf("dead list not empty during grace: %v", deadNames(dead))
+	}
+}
+
+// An undeployed or mid-deploy stack is never classified, failed restart or not.
+func TestClassifyRunStates_UndeployedIgnored(t *testing.T) {
+	sts := []stacks.Stack{
+		stack("ghost", stacks.StateStopped, false, false),
+		stack("deploying", stacks.StateStopped, true, true),
+	}
+	dead, states := classifyRunStates(sts, nil, map[string]bool{"ghost": true, "deploying": true})
+	if len(dead) != 0 || len(states) != 0 {
+		t.Errorf("undeployed/deploying stacks were classified: dead=%v states=%v", deadNames(dead), states)
+	}
+}

controller/cmd/controller/main.go

@@ -1203,26 +1203,39 @@ func runBootReconcile(ctx context.Context, mgr bootrecon.StackProvider, logger *
 func scanDeployedAppRunStates(mgr *stacks.Manager, q *quiesce.Loop) ([]web.DeadApp, []notify.AppRunState) {
 	// R-97b: a stack THIS controller stopped for a backup is not a fault. q may be nil (unprovisioned
 	// guest) — SuppressedStacks is nil-safe and returns nothing, i.e. suppress nothing.
-	return classifyRunStates(mgr.GetStacks(), q.SuppressedStacks())
+	return classifyRunStates(mgr.GetStacks(), q.SuppressedStacks(), q.FailedRestarts())
 }
 
 // classifyRunStates is the pure fix-3 derivation over a plain stack slice. It splits the deployed
 // apps into the DEAD list (dashboard banner) and the per-app run states (notifier transition tracker).
 //
 // v0.164.0: a deliberate user stop is NOT a fault and must not alarm anywhere (banner OR email). The
-// down predicate therefore EXCLUDES StateStopped, resting on two invariants:
-//   - I1: the UI stop path Manager.StopStack runs `docker compose down` → containers are removed, and
-//     a deployed stack with zero containers aggregates to StateStopped (manager.go refreshStatusLocked).
-//     So StateStopped means "deployed, deliberately stopped by the user".
-//   - I2: the P2 restart-policy census (2026-07-21, 53 templates / 78 services) found every catalog
-//     service on `unless-stopped`, so a crashing app never comes to rest at `stopped` — faults surface
-//     as StateExited / StateDegraded (and restarting/unhealthy). StateStopped is therefore never a fault.
+// down predicate therefore excludes StateStopped — but NOT unconditionally. That exclusion rested on
+// two invariants, and F-CRIT-1 (Campaign 8) showed the first of them had already become false:
 //
-// If either invariant changes, revisit this suppression. (An out-of-band `docker compose stop` leaves
-// the containers present → StateExited → still alerts, which is correct: out-of-band tampering IS
-// reportable.) IsDownState is intentionally left unchanged — other callers rely on stopped counting as
-// down; the suppression is a filter at this single derivation point only.
-func classifyRunStates(sts []stacks.Stack, quiesced map[string]bool) ([]web.DeadApp, []notify.AppRunState) {
+//   - I1 (AS WRITTEN, AND WRONG): "the UI stop path Manager.StopStack runs `docker compose down` →
+//     containers are removed → a deployed stack with zero containers aggregates to StateStopped, so
+//     StateStopped means 'deployed, deliberately stopped by the user'."
+//     **The quiesce loop stops stacks by the SAME path.** So a stack that a backup quiesce stopped and
+//     then FAILED to restart is also StateStopped — byte-identical to a user stop on the Docker side —
+//     and the whitelist swallowed it. Campaign 8 observed a customer app dead indefinitely with no
+//     banner, no event and no email, while the dead-app scanner ran 11 times over it.
+//   - I2 (still true): the P2 restart-policy census (2026-07-21, 53 templates / 78 services) found
+//     every catalog service on `unless-stopped`, so a CRASHING app never comes to rest at `stopped` —
+//     faults surface as StateExited / StateDegraded (and restarting/unhealthy).
+//
+// I1 RESTATED, TRUE AS OF v0.179.0: StateStopped means "deployed with zero containers", which is
+// EITHER a deliberate user stop OR a quiesce restart that failed. No state test can tell them apart,
+// because they are the same state. The distinguishing fact is not in the state at all — it is that
+// the quiesce loop TRIED to restart the stack and could not, which it now reports via
+// Loop.FailedRestarts(). `failedRestart` is that set, and it is the ONLY thing that lifts the
+// StateStopped whitelist.
+//
+// (An out-of-band `docker compose stop` leaves the containers present → StateExited → still alerts,
+// which is correct: out-of-band tampering IS reportable.) IsDownState is intentionally left unchanged
+// — other callers rely on stopped counting as down; the suppression is a filter at this single
+// derivation point only.
+func classifyRunStates(sts []stacks.Stack, quiesced map[string]bool, failedRestart map[string]bool) ([]web.DeadApp, []notify.AppRunState) {
 	var dead []web.DeadApp
 	var states []notify.AppRunState
 	for _, st := range sts {
@@ -1234,7 +1247,11 @@ func classifyRunStates(sts []stacks.Stack, quiesced map[string]bool) ([]web.Dead
 		// mid-restart is `starting`/`unhealthy`, not StateStopped, so v0.164.0's state filter above
 		// cannot see it. The window EXPIRES (quiesceAlarmGrace): an app that genuinely fails to come
 		// back still alarms on the first scan after it closes.
-		down := stacks.IsDownState(st.State) && st.State != stacks.StateStopped && !quiesced[st.Name]
+		// F-CRIT-1: StateStopped is whitelisted as a deliberate user stop UNLESS the quiesce loop
+		// reports that it stopped this stack and could not restart it. That single term is what turns
+		// an indefinitely-silent dead app back into an alarm, without re-alarming genuine user stops.
+		userStopped := st.State == stacks.StateStopped && !failedRestart[st.Name]
+		down := stacks.IsDownState(st.State) && !userStopped && !quiesced[st.Name]
 		states = append(states, notify.AppRunState{Name: st.Name, DisplayName: st.Meta.DisplayName, Down: down})
 		if down {
 			dead = append(dead, web.DeadApp{Name: st.Name, DisplayName: st.Meta.DisplayName, State: string(st.State)})
@@ -1712,7 +1729,7 @@ func (b quiesceBackend) Due(ctx context.Context) (bool, *int64, error) {
 }
 func (b quiesceBackend) StartBackup(ctx context.Context) (string, error) {
 	r, err := b.c.StartBackup(ctx)
-	return r.JobID, err
+	return r.JobID, mapBusy(err) // F-A1: the untargeted (pre-R-82) path can 409 identically
 }
 func (b quiesceBackend) BackupStatus(ctx context.Context) (string, error) {
 	r, err := b.c.BackupStatus(ctx)
@@ -1749,7 +1766,25 @@ func (b quiesceBackend) DueFor(ctx context.Context, target string) (bool, *int64
 }
 func (b quiesceBackend) StartBackupFor(ctx context.Context, target string) (string, error) {
 	r, err := b.c.StartBackupFor(ctx, target)
-	return r.JobID, err
+	// F-A1: translate the agent's HTTP 409 into the loop's vocabulary, exactly as Tiers translates a
+	// 404 into ErrTiersUnsupported. 409 means the agent's R-85 single-flight gate refused because a
+	// restore-test (or another backup) holds it — CONTENTION, not failure. `internal/quiesce` keeps
+	// no dependency on `internal/agentapi`, so the mapping belongs here at the seam.
+	return r.JobID, mapBusy(err)
+}
+
+// mapBusy converts a 409 from the agent into quiesce.ErrTierBusy, preserving the original error for
+// diagnosis. Any other status passes through untouched — treating a real 5xx as contention would
+// swallow genuine failures, which is the over-correction F-A1's fix must not make.
+func mapBusy(err error) error {
+	if err == nil {
+		return nil
+	}
+	var se *agentapi.StatusError
+	if errors.As(err, &se) && se.Code == http.StatusConflict {
+		return fmt.Errorf("%w: %v", quiesce.ErrTierBusy, err)
+	}
+	return err
 }
 func (b quiesceBackend) BackupStatusFor(ctx context.Context, target string) (string, error) {
 	r, err := b.c.BackupStatusFor(ctx, target)