// Package bootrecon implements R-52: the bounded, start-ONCE recovery of apps that were left // behind by an interrupted boot. // // The live failure it closes (AUDIT-vacation-remote-ops-2026-07-20, finding F5): a pre-transport // shutdown left `immich-server` and `calibre-web` Exited; ten sibling containers came back and // those two did not, and they were still down ~18 hours later. The controller REPORTED them (the // 30 s deadapp-check) but never started them — deployed-but-stopped was an alarm with no recovery. // // Two deliberate boundaries, both load-bearing: // // - **Bounded, never a loop.** At most `attempts` tries, `retryDelay` apart, then it stops and the // alarm owns the problem. A restart loop would paper over a genuinely broken app forever and // hammer docker while doing it. // - **A user's Stop survives a reboot.** The UI's Stop is `docker compose down`, which REMOVES the // containers; a boot interruption leaves them behind as Exited. So "has containers on disk that // are down" is the boot-orphan signature, and a stack with ZERO containers is deliberately never // touched. This distinction is the whole safety argument — see TestReconcile_UserStoppedAppIsNeverStarted. // // It runs inside the notifier's boot grace (cmd/controller/main.go `deadAppBootGrace`), so a // successful recovery never fires an alert and a failed one alerts honestly once the grace expires. package bootrecon import ( "context" "log" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/stacks" ) // StackProvider is the slice of *stacks.Manager this package needs. Declared consumer-side so the // tests can count StartStack calls without a docker anywhere near them. type StackProvider interface { GetStacks() []stacks.Stack StartStack(name string) error RefreshStatus() error } const ( // DefaultAttempts is the total number of start attempts per boot (not per app per retry-forever). DefaultAttempts = 2 // DefaultRetryDelay spaces the attempts. 2 × 30 s fits comfortably inside the 90 s boot grace, // so a recovery that works is silent and one that does not is honest. DefaultRetryDelay = 30 * time.Second ) // Reconciler performs the start-once sweep. Zero value is not usable — use New. type Reconciler struct { stacks StackProvider logger *log.Logger attempts int retryDelay time.Duration // sleep is the inter-attempt wait; injectable so tests never spend 30 real seconds. sleep func(context.Context, time.Duration) } // Result is the outcome, returned for logging/testing (the hub learns about failures only through // the existing app_start_failed alarm — this package deliberately pushes no events of its own). type Result struct { Candidates []string // boot-orphaned apps found Recovered []string // running again by the end StillDown []string // still down after the last attempt — the alarm's problem now Attempts int // attempts actually made (0 when there was nothing to do) } // New builds a Reconciler with the shipped defaults. func New(p StackProvider, logger *log.Logger) *Reconciler { return &Reconciler{ stacks: p, logger: logger, attempts: DefaultAttempts, retryDelay: DefaultRetryDelay, sleep: sleepCtx, } } func sleepCtx(ctx context.Context, d time.Duration) { t := time.NewTimer(d) defer t.Stop() select { case <-ctx.Done(): case <-t.C: } } // isBootOrphan reports whether a stack is an app the boot left behind. // // The gate, term by term: // - Deployed — an app the customer asked to have running. // - not Protected — traefik/cloudflared/felhom-controller have their own supervision; this must // never race the base-stack self-heal. // - not Deploying — mid-deploy is not a fault. // - has containers — the D-case guard: a UI Stop removes them, and a deliberate stop must survive // a reboot. // - IsDownState — stopped/exited/degraded (R-51 included: a boot that half-started a stack is the // same interrupted-boot shape). func isBootOrphan(s stacks.Stack) bool { return s.Deployed && !s.Protected && !s.Deploying && len(s.Containers) > 0 && stacks.IsDownState(s.State) } // Run performs the sweep once and returns what happened. It is safe to call with no boot orphans // (the quiet path logs one DEBUG-free INFO-free line — see below) and it never returns an error: // a failure to start is an app-level fact the alarm reports, not a controller startup failure. func (r *Reconciler) Run(ctx context.Context) Result { var res Result pending := map[string]bool{} for _, s := range r.stacks.GetStacks() { if isBootOrphan(s) { pending[s.Name] = true res.Candidates = append(res.Candidates, s.Name) } } sortStrings(res.Candidates) if len(pending) == 0 { // The healthy path must be observable — "no alarms" and "never ran" have to be // distinguishable in a log (the v0.91.2 lesson). r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: no boot-orphaned apps (nothing to start)") return res } r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: %d boot-orphaned app(s) found: %v — up to %d attempt(s)", len(res.Candidates), res.Candidates, r.attempts) for attempt := 1; attempt <= r.attempts && len(pending) > 0; attempt++ { res.Attempts = attempt for _, name := range sortedKeys(pending) { if ctx.Err() != nil { break } start := time.Now() if err := r.stacks.StartStack(name); err != nil { r.logger.Printf("[WARN] [bootrecon] Boot reconciliation attempt %d/%d: start %q failed after %.1fs: %v", attempt, r.attempts, name, time.Since(start).Seconds(), err) continue } r.logger.Printf("[INFO] [bootrecon] Boot reconciliation attempt %d/%d: started %q (took %.1fs)", attempt, r.attempts, name, time.Since(start).Seconds()) } if ctx.Err() != nil { break } // Re-read reality rather than trusting a nil error: `compose up -d` exits 0 on a crash-loop // (a session-critical invariant of this repo), so only a fresh docker ps can say whether the // app is actually up. if err := r.stacks.RefreshStatus(); err != nil { r.logger.Printf("[WARN] [bootrecon] Boot reconciliation: status refresh failed: %v", err) } for _, s := range r.stacks.GetStacks() { if pending[s.Name] && !stacks.IsDownState(s.State) { delete(pending, s.Name) res.Recovered = append(res.Recovered, s.Name) } } if len(pending) > 0 && attempt < r.attempts { r.sleep(ctx, r.retryDelay) } } res.StillDown = sortedKeys(pending) sortStrings(res.Recovered) if len(res.StillDown) == 0 { r.logger.Printf("[INFO] [bootrecon] Boot reconciliation complete: %d app(s) recovered in %d attempt(s): %v", len(res.Recovered), res.Attempts, res.Recovered) } else { // Deliberately no hub event here: the app_start_failed alarm fires on its own once the boot // grace expires, and two events for one dead app is how an operator inbox becomes noise. r.logger.Printf("[WARN] [bootrecon] Boot reconciliation gave up after %d attempt(s): recovered=%v still down=%v (the dead-app alarm now owns these)", res.Attempts, res.Recovered, res.StillDown) } return res } // --- tiny local helpers (no dependency on sort ordering semantics elsewhere) --- func sortedKeys(m map[string]bool) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sortStrings(out) return out } func sortStrings(s []string) { for i := 1; i < len(s); i++ { for j := i; j > 0 && s[j] < s[j-1]; j-- { s[j], s[j-1] = s[j-1], s[j] } } }