feat(v0.156.0): dead-primary alerting (R-51) + boot desired-state reconciliation (R-52)
R-51: aggregateState's mixed branch returned StateRunning ("partial"), so a stack whose
MAIN container was dead behind live helpers alerted on nothing — immich-server sat Exited
for 18 h, 100 % unreachable, no banner and no app_start_failed (audit F4). New
StateDegraded: a DOWN member whose docker restart policy is always/unless-stopped is a
fault (degraded, a down state); no/on-failure is a finished one-shot and stays benign; an
unreadable policy fails CLOSED. The unhealthy/restarting/paused/unknown exclusions are
byte-identical — folding unhealthy into down is the flapping fix-3 avoided.
R-52: new internal/bootrecon — one bounded start-once sweep at startup (2 attempts, 30 s
apart) for apps an interrupted boot left behind, inside the 90 s boot grace so a success
is silent and a failure still alerts. A zero-container stack is NEVER touched: the UI's
Stop is compose down, so a deliberate stop survives a reboot.
Both features carry a production-path wiring test (the v0.154.0 / v0.91.0 inert-seam
class). The main() assertion is an AST walk, not strings.Contains — the substring version
passed its own red-proof, because a commented-out call still contains the string.
Red-proofs run and restored: mix branch reverted -> "running" on the immich fixture;
boot hook commented out -> wiring test fails; zero-container gate dropped -> the
user-stopped app gets started.
NOTE: controller/cmd/controller/ is matched by .gitignore's `controller` entry, so new
files there need `git add -f` (and ripgrep silently skips main.go without --no-ignore).
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
// Package bootrecon implements R-52: the bounded, start-ONCE recovery of apps that were left
|
||||
// behind by an interrupted boot.
|
||||
//
|
||||
// The live failure it closes (AUDIT-vacation-remote-ops-2026-07-20, finding F5): a pre-transport
|
||||
// shutdown left `immich-server` and `calibre-web` Exited; ten sibling containers came back and
|
||||
// those two did not, and they were still down ~18 hours later. The controller REPORTED them (the
|
||||
// 30 s deadapp-check) but never started them — deployed-but-stopped was an alarm with no recovery.
|
||||
//
|
||||
// Two deliberate boundaries, both load-bearing:
|
||||
//
|
||||
// - **Bounded, never a loop.** At most `attempts` tries, `retryDelay` apart, then it stops and the
|
||||
// alarm owns the problem. A restart loop would paper over a genuinely broken app forever and
|
||||
// hammer docker while doing it.
|
||||
// - **A user's Stop survives a reboot.** The UI's Stop is `docker compose down`, which REMOVES the
|
||||
// containers; a boot interruption leaves them behind as Exited. So "has containers on disk that
|
||||
// are down" is the boot-orphan signature, and a stack with ZERO containers is deliberately never
|
||||
// touched. This distinction is the whole safety argument — see TestReconcile_UserStoppedAppIsNeverStarted.
|
||||
//
|
||||
// It runs inside the notifier's boot grace (cmd/controller/main.go `deadAppBootGrace`), so a
|
||||
// successful recovery never fires an alert and a failed one alerts honestly once the grace expires.
|
||||
package bootrecon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
|
||||
// StackProvider is the slice of *stacks.Manager this package needs. Declared consumer-side so the
|
||||
// tests can count StartStack calls without a docker anywhere near them.
|
||||
type StackProvider interface {
|
||||
GetStacks() []stacks.Stack
|
||||
StartStack(name string) error
|
||||
RefreshStatus() error
|
||||
}
|
||||
|
||||
const (
|
||||
// DefaultAttempts is the total number of start attempts per boot (not per app per retry-forever).
|
||||
DefaultAttempts = 2
|
||||
// DefaultRetryDelay spaces the attempts. 2 × 30 s fits comfortably inside the 90 s boot grace,
|
||||
// so a recovery that works is silent and one that does not is honest.
|
||||
DefaultRetryDelay = 30 * time.Second
|
||||
)
|
||||
|
||||
// Reconciler performs the start-once sweep. Zero value is not usable — use New.
|
||||
type Reconciler struct {
|
||||
stacks StackProvider
|
||||
logger *log.Logger
|
||||
attempts int
|
||||
retryDelay time.Duration
|
||||
|
||||
// sleep is the inter-attempt wait; injectable so tests never spend 30 real seconds.
|
||||
sleep func(context.Context, time.Duration)
|
||||
}
|
||||
|
||||
// Result is the outcome, returned for logging/testing (the hub learns about failures only through
|
||||
// the existing app_start_failed alarm — this package deliberately pushes no events of its own).
|
||||
type Result struct {
|
||||
Candidates []string // boot-orphaned apps found
|
||||
Recovered []string // running again by the end
|
||||
StillDown []string // still down after the last attempt — the alarm's problem now
|
||||
Attempts int // attempts actually made (0 when there was nothing to do)
|
||||
}
|
||||
|
||||
// New builds a Reconciler with the shipped defaults.
|
||||
func New(p StackProvider, logger *log.Logger) *Reconciler {
|
||||
return &Reconciler{
|
||||
stacks: p,
|
||||
logger: logger,
|
||||
attempts: DefaultAttempts,
|
||||
retryDelay: DefaultRetryDelay,
|
||||
sleep: sleepCtx,
|
||||
}
|
||||
}
|
||||
|
||||
func sleepCtx(ctx context.Context, d time.Duration) {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
|
||||
// isBootOrphan reports whether a stack is an app the boot left behind.
|
||||
//
|
||||
// The gate, term by term:
|
||||
// - Deployed — an app the customer asked to have running.
|
||||
// - not Protected — traefik/cloudflared/felhom-controller have their own supervision; this must
|
||||
// never race the base-stack self-heal.
|
||||
// - not Deploying — mid-deploy is not a fault.
|
||||
// - has containers — the D-case guard: a UI Stop removes them, and a deliberate stop must survive
|
||||
// a reboot.
|
||||
// - IsDownState — stopped/exited/degraded (R-51 included: a boot that half-started a stack is the
|
||||
// same interrupted-boot shape).
|
||||
func isBootOrphan(s stacks.Stack) bool {
|
||||
return s.Deployed && !s.Protected && !s.Deploying &&
|
||||
len(s.Containers) > 0 && stacks.IsDownState(s.State)
|
||||
}
|
||||
|
||||
// Run performs the sweep once and returns what happened. It is safe to call with no boot orphans
|
||||
// (the quiet path logs one DEBUG-free INFO-free line — see below) and it never returns an error:
|
||||
// a failure to start is an app-level fact the alarm reports, not a controller startup failure.
|
||||
func (r *Reconciler) Run(ctx context.Context) Result {
|
||||
var res Result
|
||||
|
||||
pending := map[string]bool{}
|
||||
for _, s := range r.stacks.GetStacks() {
|
||||
if isBootOrphan(s) {
|
||||
pending[s.Name] = true
|
||||
res.Candidates = append(res.Candidates, s.Name)
|
||||
}
|
||||
}
|
||||
sortStrings(res.Candidates)
|
||||
|
||||
if len(pending) == 0 {
|
||||
// The healthy path must be observable — "no alarms" and "never ran" have to be
|
||||
// distinguishable in a log (the v0.91.2 lesson).
|
||||
r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: no boot-orphaned apps (nothing to start)")
|
||||
return res
|
||||
}
|
||||
|
||||
r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: %d boot-orphaned app(s) found: %v — up to %d attempt(s)",
|
||||
len(res.Candidates), res.Candidates, r.attempts)
|
||||
|
||||
for attempt := 1; attempt <= r.attempts && len(pending) > 0; attempt++ {
|
||||
res.Attempts = attempt
|
||||
for _, name := range sortedKeys(pending) {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
start := time.Now()
|
||||
if err := r.stacks.StartStack(name); err != nil {
|
||||
r.logger.Printf("[WARN] [bootrecon] Boot reconciliation attempt %d/%d: start %q failed after %.1fs: %v",
|
||||
attempt, r.attempts, name, time.Since(start).Seconds(), err)
|
||||
continue
|
||||
}
|
||||
r.logger.Printf("[INFO] [bootrecon] Boot reconciliation attempt %d/%d: started %q (took %.1fs)",
|
||||
attempt, r.attempts, name, time.Since(start).Seconds())
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
// Re-read reality rather than trusting a nil error: `compose up -d` exits 0 on a crash-loop
|
||||
// (a session-critical invariant of this repo), so only a fresh docker ps can say whether the
|
||||
// app is actually up.
|
||||
if err := r.stacks.RefreshStatus(); err != nil {
|
||||
r.logger.Printf("[WARN] [bootrecon] Boot reconciliation: status refresh failed: %v", err)
|
||||
}
|
||||
for _, s := range r.stacks.GetStacks() {
|
||||
if pending[s.Name] && !stacks.IsDownState(s.State) {
|
||||
delete(pending, s.Name)
|
||||
res.Recovered = append(res.Recovered, s.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if len(pending) > 0 && attempt < r.attempts {
|
||||
r.sleep(ctx, r.retryDelay)
|
||||
}
|
||||
}
|
||||
|
||||
res.StillDown = sortedKeys(pending)
|
||||
sortStrings(res.Recovered)
|
||||
|
||||
if len(res.StillDown) == 0 {
|
||||
r.logger.Printf("[INFO] [bootrecon] Boot reconciliation complete: %d app(s) recovered in %d attempt(s): %v",
|
||||
len(res.Recovered), res.Attempts, res.Recovered)
|
||||
} else {
|
||||
// Deliberately no hub event here: the app_start_failed alarm fires on its own once the boot
|
||||
// grace expires, and two events for one dead app is how an operator inbox becomes noise.
|
||||
r.logger.Printf("[WARN] [bootrecon] Boot reconciliation gave up after %d attempt(s): recovered=%v still down=%v (the dead-app alarm now owns these)",
|
||||
res.Attempts, res.Recovered, res.StillDown)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// --- tiny local helpers (no dependency on sort ordering semantics elsewhere) ---
|
||||
|
||||
func sortedKeys(m map[string]bool) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sortStrings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func sortStrings(s []string) {
|
||||
for i := 1; i < len(s); i++ {
|
||||
for j := i; j > 0 && s[j] < s[j-1]; j-- {
|
||||
s[j], s[j-1] = s[j-1], s[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user