R-97: a failing backup is heard, and stops blaming the apps (v0.177.0)

R-97a: internal/quiesce had no route to the hub at all — three failed whole-guest
backups on 2026-07-27 produced zero events. TierNotifier is a seam (not an import),
wired by an init-only setter because main.go builds the notifier after the loop.
Edge-triggered: the failure fires when the R-88 breaker ARMS, not per retry, and
recovery rides recordSuccess's existing bool. Uses NEW operator-only event types;
reusing backup_failed would have emailed the customer in Hungarian about a backup
they cannot act on, since it has a customerMessages entry and is in live
enabled_events. Requires hub >= v0.78.0.

R-97b: v0.164.0's state filter cannot see an app caught MID-RESTART, which is how
BookStack alarmed. The fix is a suppression window keyed to the quiesce CYCLE,
consumed at the same single derivation point. 180s grace, derived from the deploy
flow's 120s health timeout and Mealie's 60s start_period; it expires, so an app
that genuinely fails to come back still alarms.

controller/internal/quiesce/quiesce.go

@@ -103,8 +103,22 @@ type Loop struct {
 	// breaker (R-88) defers the QUIESCE for a tier whose backups keep failing, so a broken target
 	// cannot stop the customer's apps every 5 minutes forever. Scheduled path only — see breaker.go.
 	breaker *failureBreaker
+	// tierNotify (R-97a) reports a tier's backup outcome to the hub. nil = not wired (pre-provisioning).
+	// Init-only: set once at startup via SetTierNotifier, before Run.
+	tierNotify TierNotifier
+	// suppressed (R-97b) is stack name → grace expiry (zero = still quiesced). Read by
+	// SuppressedStacks so an app WE stopped is not reported to the customer as broken.
+	suppressMu sync.Mutex
+	suppressed map[string]time.Time
 }
 
+// SetTierNotifier wires the hub-event seam. INIT-ONLY — call once at startup, before Run.
+//
+// It is a setter rather than an Options field because main.go constructs the notifier AFTER the
+// quiesce loop, and reordering that has a wider blast radius than a setter does. nil is legal and
+// means "no hub" — a guest that is not provisioned yet still backs up, it just cannot report.
+func (l *Loop) SetTierNotifier(n TierNotifier) { l.tierNotify = n }
+
 // New builds a Loop with sane defaults for any unset duration.
 func New(o Options) *Loop {
 	if o.Poll <= 0 {
@@ -223,17 +237,35 @@ func (l *Loop) runOnce(ctx context.Context) error {
 // noteTierFailure arms/extends the tier's backoff and announces the deferral exactly ONCE — here, at
 // the moment it is armed. Called from BOTH the scheduled and the manual path: a manual run that
 // fails is evidence about the tier too. Only the GATING is scheduler-only.
-func (l *Loop) noteTierFailure(target, label string) {
+func (l *Loop) noteTierFailure(target, label, errMsg string) {
 	n, d := l.breaker.recordFailure(target, l.now())
 	l.logger.Printf("[WARN] [quiesce] tier %s has now failed %d time(s) in a row — deferring its next quiesce by %s (cap %s) so the apps are not stopped again for a backup that cannot succeed",
 		label, n, d, breakerMaxDelay)
+	// R-97a: report ONCE, when the breaker ARMS (n == 1), never on the retries behind it.
+	//
+	// This got MORE urgent when R-88 shipped, not less: before the breaker a failing backup retried
+	// every 5 minutes — harmful, but loud enough to notice. Now it backs off to 4h and goes quiet,
+	// leaving the hub's deadline monitor as the only signal at ~26h (local) / ~8 days (PBS) — a full
+	// cycle of the weekly tier. This trades that delay for an immediate one.
+	if n == 1 && l.tierNotify != nil {
+		l.tierNotify.BackupFailed(label,
+			fmt.Sprintf("Whole-guest backup FAILED on the %s tier — retrying with backoff (next attempt in %s)", label, d),
+			errMsg)
+	}
 }
 
 // noteTierSuccess clears any backoff. Quiet unless there was something to clear — a line per healthy
 // backup would be noise, but a recovery is worth one.
 func (l *Loop) noteTierSuccess(target, label string) {
+	// recordSuccess's bool is the edge: true only when there WAS a backoff to clear. That is exactly
+	// the recovery edge — an operator told a tier broke must also be told it healed, and a line (or
+	// an event) per healthy backup would be noise.
 	if l.breaker.recordSuccess(target) {
 		l.logger.Printf("[INFO] [quiesce] tier %s succeeded — clearing its failure backoff; normal cadence resumes", label)
+		if l.tierNotify != nil {
+			l.tierNotify.BackupRecovered(label,
+				fmt.Sprintf("Whole-guest backup RECOVERED on the %s tier — it succeeded after a run of failures", label))
+		}
 	}
 }
 
@@ -365,6 +397,9 @@ func (l *Loop) quiesceAndPollTiers(ctx context.Context, tiers []dueTier) error {
 		unquiesced = true
 		l.logger.Printf("[INFO] [quiesce] unquiescing (%s): restarting %d stack(s)", reason, len(running))
 		l.restartAll(running)
+		// R-97b: start the grace clock AFTER the restart call, so the window measures time the app
+		// has actually had to come up rather than time it spent stopped.
+		l.markUnquiesced(running)
 		if err := l.clearMarker(); err != nil {
 			l.logger.Printf("[ERROR] [quiesce] clear marker: %v", err)
 		}
@@ -373,6 +408,9 @@ func (l *Loop) quiesceAndPollTiers(ctx context.Context, tiers []dueTier) error {
 
 	l.logger.Printf("[INFO] [quiesce] backup due on %d tier(s) — quiescing %d stack(s): %v",
 		len(tiers), len(running), running)
+	// R-97b: exempt these from app-down alarms BEFORE stopping them, or a health scan landing between
+	// the stop and the mark would alarm on an app we are about to take down deliberately.
+	l.markQuiesced(running)
 	for _, st := range running {
 		if err := l.stacks.StopStack(st); err != nil {
 			l.logger.Printf("[ERROR] [quiesce] stop %s: %v (continuing)", st, err)
@@ -390,7 +428,7 @@ func (l *Loop) quiesceAndPollTiers(ctx context.Context, tiers []dueTier) error {
 		jobID, err := l.startBackupOn(ctx, t.target)
 		if err != nil {
 			l.logger.Printf("[ERROR] [quiesce] start backup on tier %s: %v", label, err)
-			l.noteTierFailure(t.target, label)
+			l.noteTierFailure(t.target, label, err.Error())
 			if firstErr == nil {
 				firstErr = fmt.Errorf("start backup on %s: %w", label, err)
 			}
@@ -411,7 +449,7 @@ func (l *Loop) quiesceAndPollTiers(ctx context.Context, tiers []dueTier) error {
 		switch {
 		case phase == phaseFailed:
 			l.logger.Printf("[WARN] [quiesce] tier %s: backup job %s failed", label, jobID)
-			l.noteTierFailure(t.target, label)
+			l.noteTierFailure(t.target, label, "backup job "+jobID+" reported phase=failed")
 		case stillRunning:
 			// Neither outcome yet — a first full offsite snapshot legitimately runs for hours. It
 			// must NOT count as a failure, or a slow-but-healthy tier would back itself off.