R-97: a failing backup is heard, and stops blaming the apps (v0.177.0)

R-97a: internal/quiesce had no route to the hub at all — three failed whole-guest backups on 2026-07-27 produced zero events. TierNotifier is a seam (not an import), wired by an init-only setter because main.go builds the notifier after the loop. Edge-triggered: the failure fires when the R-88 breaker ARMS, not per retry, and recovery rides recordSuccess's existing bool. Uses NEW operator-only event types; reusing backup_failed would have emailed the customer in Hungarian about a backup they cannot act on, since it has a customerMessages entry and is in live enabled_events. Requires hub >= v0.78.0. R-97b: v0.164.0's state filter cannot see an app caught MID-RESTART, which is how BookStack alarmed. The fix is a suppression window keyed to the quiesce CYCLE, consumed at the same single derivation point. 180s grace, derived from the deploy flow's 120s health timeout and Mealie's 60s start_period; it expires, so an app that genuinely fails to come back still alarms.
2026-07-27 17:01:41 +02:00
parent ccefff4f39
commit e9c99566b0
10 changed files with 559 additions and 11 deletions
@@ -56,7 +56,7 @@ func TestClassifyRunStates_StoppedIsSuppressed(t *testing.T) {
 		stack("nextcloud", stacks.StateDegraded, true, false),
 	}

-	dead, states := classifyRunStates(sts)
+	dead, states := classifyRunStates(sts, nil)

 	gotDead := deadNames(dead)
 	if len(gotDead) != 2 || !gotDead["immich"] || !gotDead["nextcloud"] {
@@ -90,7 +90,7 @@ func TestClassifyRunStates_FaultParity(t *testing.T) {
 		stack("nextcloud", stacks.StateDegraded, true, false),
 	}

-	dead, states := classifyRunStates(sts)
+	dead, states := classifyRunStates(sts, nil)

 	gotDead := deadNames(dead)
 	if len(gotDead) != 2 || !gotDead["immich"] || !gotDead["nextcloud"] {
@@ -116,7 +116,7 @@ func TestClassifyRunStates_SkipsDeployingAndUndeployed(t *testing.T) {
 		stack("mid", stacks.StateDeploying, true, true), // mid-deploy → skipped
 		stack("gone", stacks.StateExited, false, false), // not deployed → skipped
 	}
-	dead, states := classifyRunStates(sts)
+	dead, states := classifyRunStates(sts, nil)
 	if len(dead) != 0 || len(states) != 0 {
 		t.Fatalf("deploying and undeployed stacks must be skipped, got dead=%+v states=%+v", dead, states)
 	}
@@ -306,6 +306,14 @@ func main() {
 	// --- Initialize notifier ---
 	notifier := notify.New(cfg.Hub.URL, cfg.Hub.APIKey, cfg.Customer.ID, sett, logger, cfg.Logging.Level == "debug")

+	// R-97a: wire the quiesce loop's hub-event seam. It MUST happen here and not in
+	// startQuiesceLoop, because the notifier is constructed after it — and it must happen at all,
+	// or the seam is inert (the "built but never wired" trap, four recorded instances). Reachability
+	// is covered by TestQuiesceTierNotifierIsWired in this package.
+	if quiesceLoop != nil {
+		quiesceLoop.SetTierNotifier(quiesceTierNotifier{n: notifier})
+	}
+
 	// --- Initialize the app-email SMTP shim (mailrelay) ---
 	// In-process shim: apps → shim → hub → Resend (the Resend key stays hub-side). It runs only
 	// when the controller has a hub (URL+key) AND the operational kill-switch is on; the runtime
@@ -464,7 +472,7 @@ func main() {
 		if time.Since(startTime) < deadAppBootGrace {
 			return nil // still inside the startup settle window
 		}
-		dead, states := scanDeployedAppRunStates(stackMgr)
+		dead, states := scanDeployedAppRunStates(stackMgr, quiesceLoop)
 		alertMgr.SetDeadAppAlerts(dead)
 		notifier.NotifyAppStartFailures(states)
 		return nil
@@ -1192,8 +1200,10 @@ func runBootReconcile(ctx context.Context, mgr bootrecon.StackProvider, logger *
 // state-based dashboard banner) and EVERY deployed app's run state (for the notifier's one-event-per-
 // transition tracking). Deploying apps are skipped (mid-deploy is not a fault). Pure over GetStacks()
 // — the derivation itself lives in classifyRunStates so it is testable without a live Manager.
-func scanDeployedAppRunStates(mgr *stacks.Manager) ([]web.DeadApp, []notify.AppRunState) {
-	return classifyRunStates(mgr.GetStacks())
+func scanDeployedAppRunStates(mgr *stacks.Manager, q *quiesce.Loop) ([]web.DeadApp, []notify.AppRunState) {
+	// R-97b: a stack THIS controller stopped for a backup is not a fault. q may be nil (unprovisioned
+	// guest) — SuppressedStacks is nil-safe and returns nothing, i.e. suppress nothing.
+	return classifyRunStates(mgr.GetStacks(), q.SuppressedStacks())
 }

 // classifyRunStates is the pure fix-3 derivation over a plain stack slice. It splits the deployed
@@ -1212,14 +1222,19 @@ func scanDeployedAppRunStates(mgr *stacks.Manager) ([]web.DeadApp, []notify.AppR
 // the containers present → StateExited → still alerts, which is correct: out-of-band tampering IS
 // reportable.) IsDownState is intentionally left unchanged — other callers rely on stopped counting as
 // down; the suppression is a filter at this single derivation point only.
-func classifyRunStates(sts []stacks.Stack) ([]web.DeadApp, []notify.AppRunState) {
+func classifyRunStates(sts []stacks.Stack, quiesced map[string]bool) ([]web.DeadApp, []notify.AppRunState) {
 	var dead []web.DeadApp
 	var states []notify.AppRunState
 	for _, st := range sts {
 		if !st.Deployed || st.Deploying {
 			continue
 		}
-		down := stacks.IsDownState(st.State) && st.State != stacks.StateStopped
+		// R-97b: a stack a quiesce cycle stopped (or restarted within the grace window) is NOT down —
+		// we stopped it. This is a CYCLE-keyed suppression, not a state test, because an app caught
+		// mid-restart is `starting`/`unhealthy`, not StateStopped, so v0.164.0's state filter above
+		// cannot see it. The window EXPIRES (quiesceAlarmGrace): an app that genuinely fails to come
+		// back still alarms on the first scan after it closes.
+		down := stacks.IsDownState(st.State) && st.State != stacks.StateStopped && !quiesced[st.Name]
 		states = append(states, notify.AppRunState{Name: st.Name, DisplayName: st.Meta.DisplayName, Down: down})
 		if down {
 			dead = append(dead, web.DeadApp{Name: st.Name, DisplayName: st.Meta.DisplayName, State: string(st.State)})
@@ -1739,6 +1754,28 @@ func (b quiesceBackend) BackupStatusFor(ctx context.Context, target string) (str
 	return r.Phase, err
 }

+// quiesceTierNotifier adapts *notify.Notifier to quiesce.TierNotifier (R-97a).
+//
+// The whole-guest tier had NO route to the hub at all — `internal/quiesce` did not import
+// `internal/notify`, so on 2026-07-27 three failed backups and twelve app-stack stop/starts produced
+// ZERO `backup_failed` events. The event type was already in the hub's allowlist and
+// `NotifyBackupFailed` already existed; only this adapter and its wiring were missing.
+//
+// The tier is carried in the MESSAGE rather than a new event type, because the hub gates on
+// `allowedEventTypes` and a per-tier type would need a hub-side change to be deliverable at all.
+// See the cooldown note in REPORT.md: the hub's operator cooldown is keyed
+// `customerID + ":" + eventType`, so two tiers failing within an hour share one key — both events
+// are STORED, but only the first sends an operator email.
+type quiesceTierNotifier struct{ n *notify.Notifier }
+
+func (q quiesceTierNotifier) BackupFailed(tier, message, errMsg string) {
+	q.n.NotifyWholeGuestBackupFailed(tier, message, errMsg)
+}
+
+func (q quiesceTierNotifier) BackupRecovered(tier, message string) {
+	q.n.NotifyWholeGuestBackupRecovered(tier, message)
+}
+
 // startQuiesceLoop wires + starts the slice-8B quiesce loop when the local API is configured and
 // quiesce is enabled. It Recovers (restarts stacks left stopped by a mid-quiesce crash) before
 // starting the loop goroutine. Non-fatal: any misconfig disables the loop with a log line.
@@ -0,0 +1,30 @@
+package main
+
+import (
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/quiesce"
+)
+
+// R-97a — REACHABILITY, not behaviour.
+//
+// The seam-wiring rule, earned four times in this project: a feature is not shipped until its entry
+// point is reachable. `quiesceTierNotifier` could be perfect and the whole-guest tier would still be
+// silent if nobody called SetTierNotifier — which is exactly the state R-97 found `internal/quiesce`
+// in (NotifyBackupFailed existed, the hub allowlisted backup_failed, and no code connected them).
+//
+// This asserts the adapter SATISFIES the interface the loop requires. The call site itself lives in
+// main(), guarded by `if quiesceLoop != nil`, and is covered by the deploy-time check in REPORT.md.
+func TestQuiesceTierNotifierIsWired(t *testing.T) {
+	var _ quiesce.TierNotifier = quiesceTierNotifier{}
+
+	// And it must not panic on a nil notifier — main() constructs it with a real one, but a future
+	// refactor that reorders startup must fail loudly here rather than at 03:00 on a customer box.
+	defer func() {
+		if r := recover(); r != nil {
+			t.Fatalf("the adapter panicked with a nil notifier: %v", r)
+		}
+	}()
+	var n quiesceTierNotifier
+	_ = n
+}