felhom-controller/controller/internal/quiesce/suppress.go

package quiesce

import "time"

// R-97b — a quiesce cycle must not make the customer think their apps broke.
//
// THE BUG: during the 2026-07-27 loop the ONLY customer-visible output was
// `app_start_failed — "Telepített alkalmazás nem fut: BookStack"`, on the customer channel, in
// Hungarian, during an outage the BACKUP SYSTEM ITSELF caused, with no indication why. That is worse
// than silence: it tells the customer something is wrong with their app and hands them nothing to do.
//
// WHY v0.164.0's FILTER DOES NOT COVER THIS. That filter is state-based —
// `IsDownState(st.State) && st.State != StateStopped` — and it suppresses DELIBERATELY STOPPED apps.
// BookStack alarmed because the third quiesce cycle caught it MID-RESTART: starting, or up but not
// yet healthy. Those are not `StateStopped`, and no state classification can tell "restarting because
// a backup just stopped me" from "restarting because I keep crashing". The distinguishing fact is not
// in the state at all — it is that *we* stopped it, and we know we did.
//
// So this is a SUPPRESSION WINDOW KEYED TO THE CYCLE, not a state test: the loop already tracks which
// stacks it stopped (it must, to restart exactly those), and it knows when it unquiesced.
//
// ── THE TENSION, WHICH IS THE WHOLE DESIGN ───────────────────────────────────────────────────
//
// Suppress during the cycle and for a grace period after the restart — but an app that GENUINELY
// fails to come back MUST still alarm. Permanent suppression would trade a loud false alarm for a
// silent real one, which is the same over-correction as R-88's Scenario D. The grace window expires;
// it does not latch.

// quiesceAlarmGrace is how long after unquiescing a stack stays exempt from app-down alarms.
//
// Derived from what a restarted app actually needs, not picked round:
//   - the deploy flow allows **120 s** for a stack to come up healthy — the project's own existing
//     answer to "how long is too long";
//   - the slowest catalog healthcheck start_period is Mealie's **60 s**, after which a couple of
//     check intervals must still elapse before a verdict is meaningful.
//
// 180 s clears both with margin. It is deliberately NOT longer: the app-state scan runs on its own
// cadence, so an app that is genuinely dead alarms on the first scan after the window closes —
// making the cost of this suppression a bounded DELAY in reporting a real failure, never its loss.
const quiesceAlarmGrace = 180 * time.Second

// markQuiesced records stacks as exempt for the duration of the cycle. Expiry is set at unquiesce;
// until then the entry is open-ended, because a cycle may legitimately run for hours (a first full
// offsite snapshot) and an app stopped that whole time must not alarm halfway through.
func (l *Loop) markQuiesced(names []string) {
	l.suppressMu.Lock()
	defer l.suppressMu.Unlock()
	if l.suppressed == nil {
		l.suppressed = map[string]time.Time{}
	}
	for _, n := range names {
		l.suppressed[n] = time.Time{} // zero = still quiesced, no expiry yet
	}
}

// markUnquiesced starts the grace clock for the stacks this cycle restarted.
func (l *Loop) markUnquiesced(names []string) {
	until := l.now().Add(quiesceAlarmGrace)
	l.suppressMu.Lock()
	defer l.suppressMu.Unlock()
	if l.suppressed == nil {
		return
	}
	for _, n := range names {
		l.suppressed[n] = until
	}
}

// SuppressedStacks returns the set of stack names currently exempt from app-down alarms — those a
// quiesce cycle stopped, plus those still inside the post-restart grace window.
//
// Nil-safe on a nil *Loop so the caller does not need a branch: a controller with no quiesce loop
// (unprovisioned guest) suppresses nothing, which is the correct default.
func (l *Loop) SuppressedStacks() map[string]bool {
	if l == nil {
		return nil
	}
	now := l.now()
	l.suppressMu.Lock()
	defer l.suppressMu.Unlock()
	out := make(map[string]bool, len(l.suppressed))
	for n, until := range l.suppressed {
		if until.IsZero() || now.Before(until) {
			out[n] = true
			continue
		}
		delete(l.suppressed, n) // expired — reap so the map cannot grow without bound
	}
	return out
}

// ---- F-CRIT-1: the loop remembers which stacks it FAILED to restart -------------------------
//
// THE BUG THIS EXISTS TO KILL. `classifyRunStates` whitelists `StateStopped` because v0.164.0
// (correctly) refused to alarm on a deliberate user stop. That rests on invariant I1: "a deployed
// stack with zero containers was stopped by the user". **The quiesce loop broke I1** — it stops
// stacks by the same `docker compose down` path, so a stack this loop stopped and then FAILED to
// restart is also `StateStopped`, and was therefore whitelisted into total silence. Campaign 8
// observed exactly that: a customer's app dead indefinitely, no banner, no event, no email, while
// the dead-app scanner ran 11 times.
//
// No state test can separate the two cases — they are byte-identical on the Docker side. The
// distinguishing fact is not in the state at all: it is that WE tried to restart it and could not.
// That fact now leaves `restartAll`, and is remembered here.
//
// WHY A SET AND NOT A TIMESTAMP. The flag is only ever consulted for a stack that is ALREADY in a
// down state, so a stale entry cannot manufacture an alarm on a healthy app: if the operator fixes
// the app and starts it by any route, the stack is no longer down and the classifier never reaches
// this. The entry is cleared the moment a later restart of that stack succeeds.

// noteRestartOutcome records the result of one restart pass: `failed` are the stacks that would not
// start, and everything else in `attempted` is cleared. Clearing on success is what stops a fixed
// app from carrying its old failure forever.
func (l *Loop) noteRestartOutcome(attempted, failed []string) {
	if l == nil {
		return
	}
	bad := make(map[string]bool, len(failed))
	for _, n := range failed {
		bad[n] = true
	}
	l.suppressMu.Lock()
	defer l.suppressMu.Unlock()
	if l.restartFailed == nil {
		l.restartFailed = map[string]struct{}{}
	}
	for _, n := range attempted {
		if bad[n] {
			l.restartFailed[n] = struct{}{}
		} else {
			delete(l.restartFailed, n)
		}
	}
}

// FailedRestarts returns the stacks this loop stopped and could not restart. Nil-safe on a nil
// *Loop for the same reason SuppressedStacks is: an unprovisioned guest has no loop and must report
// nothing rather than force a branch on the caller.
func (l *Loop) FailedRestarts() map[string]bool {
	if l == nil {
		return nil
	}
	l.suppressMu.Lock()
	defer l.suppressMu.Unlock()
	out := make(map[string]bool, len(l.restartFailed))
	for n := range l.restartFailed {
		out[n] = true
	}
	return out
}