felhom-controller/controller/internal/bootrecon/bootrecon.go

// Package bootrecon implements R-52: the bounded, start-ONCE recovery of apps that were left
// behind by an interrupted boot.
//
// The live failure it closes (AUDIT-vacation-remote-ops-2026-07-20, finding F5): a pre-transport
// shutdown left `immich-server` and `calibre-web` Exited; ten sibling containers came back and
// those two did not, and they were still down ~18 hours later. The controller REPORTED them (the
// 30 s deadapp-check) but never started them — deployed-but-stopped was an alarm with no recovery.
//
// Two deliberate boundaries, both load-bearing:
//
//   - **Bounded, never a loop.** At most `attempts` tries, `retryDelay` apart, then it stops and the
//     alarm owns the problem. A restart loop would paper over a genuinely broken app forever and
//     hammer docker while doing it.
//   - **A user's Stop survives a reboot.** This is still the whole safety argument; only the way it
//     is established changed. Until v0.189.0 it was inferred — the UI's Stop is `docker compose
//     down`, which REMOVES containers, so "zero containers" was read as "the customer stopped it"
//     and left alone. Since v0.189.0 (R-166) the customer's intent is RECORDED in app.yaml and read
//     directly, because the inference could not distinguish a deliberate Stop from a power cut or an
//     interrupted backup, and silently stranded both. An app.yaml with no recorded intent — every
//     app on every box predating the field — keeps the old inference exactly. See isBootOrphan,
//     TestReconcile_UserStoppedAppIsNeverStarted and
//     TestReconcile_LegacyNoDesiredState_BehavesExactlyAsBefore.
//
// It runs inside the notifier's boot grace (cmd/controller/main.go `deadAppBootGrace`), so a
// successful recovery never fires an alert and a failed one alerts honestly once the grace expires.
package bootrecon

import (
	"context"
	"log"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)

// StackProvider is the slice of *stacks.Manager this package needs. Declared consumer-side so the
// tests can count StartStack calls without a docker anywhere near them.
type StackProvider interface {
	GetStacks() []stacks.Stack
	StartStack(name string) error
	RefreshStatus() error
}

const (
	// DefaultAttempts is the total number of start attempts per boot (not per app per retry-forever).
	DefaultAttempts = 2
	// DefaultRetryDelay spaces the attempts. 2 × 30 s fits comfortably inside the 90 s boot grace,
	// so a recovery that works is silent and one that does not is honest.
	DefaultRetryDelay = 30 * time.Second
)

// Reconciler performs the start-once sweep. Zero value is not usable — use New.
type Reconciler struct {
	stacks     StackProvider
	logger     *log.Logger
	attempts   int
	retryDelay time.Duration

	// sleep is the inter-attempt wait; injectable so tests never spend 30 real seconds.
	sleep func(context.Context, time.Duration)
}

// Result is the outcome, returned for logging/testing (the hub learns about failures only through
// the existing app_start_failed alarm — this package deliberately pushes no events of its own).
type Result struct {
	Candidates []string // boot-orphaned apps found
	Recovered  []string // running again by the end
	StillDown  []string // still down after the last attempt — the alarm's problem now
	Attempts   int      // attempts actually made (0 when there was nothing to do)
}

// New builds a Reconciler with the shipped defaults.
func New(p StackProvider, logger *log.Logger) *Reconciler {
	return &Reconciler{
		stacks:     p,
		logger:     logger,
		attempts:   DefaultAttempts,
		retryDelay: DefaultRetryDelay,
		sleep:      sleepCtx,
	}
}

func sleepCtx(ctx context.Context, d time.Duration) {
	t := time.NewTimer(d)
	defer t.Stop()
	select {
	case <-ctx.Done():
	case <-t.C:
	}
}

// isBootOrphan reports whether a stack is an app the boot left behind.
//
// The gate, term by term:
//   - Deployed — an app that is installed. NOTE: `Deployed` means INSTALLED, not "wanted running";
//     the two were conflated until v0.189.0 and that conflation is what the desired-state term below
//     repairs.
//   - not Protected — traefik/cloudflared/felhom-controller have their own supervision; this must
//     never race the base-stack self-heal.
//   - not Deploying — mid-deploy is not a fault.
//   - desired state — see below. REPLACES the old container-count term.
//   - IsDownState — stopped/exited/degraded (R-51 included: a boot that half-started a stack is the
//     same interrupted-boot shape).
//
// ── WHY INTENT REPLACED THE CONTAINER COUNT (R-166, closing R-157 mechanism B) ────────────────────
//
// This gate used to end in `len(s.Containers) > 0`, and its comment called that "the D-case guard":
// a UI Stop is `compose down`, which REMOVES containers, so zero containers was read as "the
// customer stopped this" and left alone. The safety goal was right and still holds. The SIGNAL was
// wrong, because zero containers has at least three causes and the count cannot tell them apart:
//
//	  a deliberate Stop   → must stay down
//	  a power cut mid-compose, or an interrupted deploy → must come back
//	  a backup that stopped the app and died before restarting it → must come back
//
// Two of those three were silently unrecoverable: the app simply stayed gone until a human noticed.
// The count was never capable of separating them, so the fix is not a better inference — it is to
// stop inferring and read what the customer actually asked for, which app.yaml now records.
//
// ── WHAT ABSENT STILL MEANS, AND WHY THE OLD BEHAVIOUR IS KEPT ────────────────────────────────────
//
// DesiredStateUnknown falls back to the ORIGINAL container-count rule, byte-for-byte. This is the
// single most important line in the change. Every app.yaml on every existing box predates the field,
// so absent is what the whole fleet reads on upgrade; treating absent as "running" would start, on
// the first boot after the upgrade, every app its owner had deliberately stopped. The fallback is
// what makes this feature inert for an app nobody has pressed a button on since — see
// TestReconcile_LegacyNoDesiredState_BehavesExactlyAsBefore and its red-proof.
//
// The full decision table (§8.1):
//
//	desired   containers  state        → result
//	stopped   any         any          → never an orphan (the customer said so)
//	running   0           —            → ORPHAN  ← the R-157 case, invisible before v0.189.0
//	running   >0          IsDownState  → ORPHAN  (unchanged)
//	running   >0          up           → not an orphan
//	absent    0           —            → not an orphan  (exactly the pre-v0.189.0 behaviour)
//	absent    >0          IsDownState  → ORPHAN          (exactly the pre-v0.189.0 behaviour)
func isBootOrphan(s stacks.Stack) bool {
	if !s.Deployed || s.Protected || s.Deploying {
		return false
	}
	switch stacks.DesiredStateOf(s) {
	case stacks.DesiredStateStopped:
		// The customer pressed Stop. No observation may overturn that — not a missing container, not
		// a down state, not a reboot. Nothing else in this package starts an app.
		return false
	case stacks.DesiredStateRunning:
		// Wanted running. ANY way of not being up is a fault to repair, including having no
		// containers at all — which is the case the old count term structurally could not see.
		return len(s.Containers) == 0 || stacks.IsDownState(s.State)
	default:
		// DesiredStateUnknown — legacy. Keep the pre-R-166 rule exactly.
		return len(s.Containers) > 0 && stacks.IsDownState(s.State)
	}
}

// Run performs the sweep once and returns what happened. It is safe to call with no boot orphans
// (the quiet path logs one DEBUG-free INFO-free line — see below) and it never returns an error:
// a failure to start is an app-level fact the alarm reports, not a controller startup failure.
func (r *Reconciler) Run(ctx context.Context) Result {
	var res Result

	pending := map[string]bool{}
	for _, s := range r.stacks.GetStacks() {
		if isBootOrphan(s) {
			pending[s.Name] = true
			res.Candidates = append(res.Candidates, s.Name)
		}
	}
	sortStrings(res.Candidates)

	if len(pending) == 0 {
		// The healthy path must be observable — "no alarms" and "never ran" have to be
		// distinguishable in a log (the v0.91.2 lesson).
		r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: no boot-orphaned apps (nothing to start)")
		return res
	}

	r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: %d boot-orphaned app(s) found: %v — up to %d attempt(s)",
		len(res.Candidates), res.Candidates, r.attempts)

	for attempt := 1; attempt <= r.attempts && len(pending) > 0; attempt++ {
		res.Attempts = attempt
		for _, name := range sortedKeys(pending) {
			if ctx.Err() != nil {
				break
			}
			start := time.Now()
			if err := r.stacks.StartStack(name); err != nil {
				r.logger.Printf("[WARN] [bootrecon] Boot reconciliation attempt %d/%d: start %q failed after %.1fs: %v",
					attempt, r.attempts, name, time.Since(start).Seconds(), err)
				continue
			}
			r.logger.Printf("[INFO] [bootrecon] Boot reconciliation attempt %d/%d: started %q (took %.1fs)",
				attempt, r.attempts, name, time.Since(start).Seconds())
		}
		if ctx.Err() != nil {
			break
		}

		// Re-read reality rather than trusting a nil error: `compose up -d` exits 0 on a crash-loop
		// (a session-critical invariant of this repo), so only a fresh docker ps can say whether the
		// app is actually up.
		if err := r.stacks.RefreshStatus(); err != nil {
			r.logger.Printf("[WARN] [bootrecon] Boot reconciliation: status refresh failed: %v", err)
		}
		for _, s := range r.stacks.GetStacks() {
			if pending[s.Name] && !stacks.IsDownState(s.State) {
				delete(pending, s.Name)
				res.Recovered = append(res.Recovered, s.Name)
			}
		}

		if len(pending) > 0 && attempt < r.attempts {
			r.sleep(ctx, r.retryDelay)
		}
	}

	res.StillDown = sortedKeys(pending)
	sortStrings(res.Recovered)

	if len(res.StillDown) == 0 {
		r.logger.Printf("[INFO] [bootrecon] Boot reconciliation complete: %d app(s) recovered in %d attempt(s): %v",
			len(res.Recovered), res.Attempts, res.Recovered)
	} else {
		// Deliberately no hub event here: the app_start_failed alarm fires on its own once the boot
		// grace expires, and two events for one dead app is how an operator inbox becomes noise.
		r.logger.Printf("[WARN] [bootrecon] Boot reconciliation gave up after %d attempt(s): recovered=%v still down=%v (the dead-app alarm now owns these)",
			res.Attempts, res.Recovered, res.StillDown)
	}
	return res
}

// --- tiny local helpers (no dependency on sort ordering semantics elsewhere) ---

func sortedKeys(m map[string]bool) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sortStrings(out)
	return out
}

func sortStrings(s []string) {
	for i := 1; i < len(s); i++ {
		for j := i; j > 0 && s[j] < s[j-1]; j-- {
			s[j], s[j-1] = s[j-1], s[j]
		}
	}
}