felhom-controller/controller/internal/bootrecon/bootrecon.go

// Package bootrecon implements R-52: the bounded, start-ONCE recovery of apps that were left
// behind by an interrupted boot.
//
// The live failure it closes (AUDIT-vacation-remote-ops-2026-07-20, finding F5): a pre-transport
// shutdown left `immich-server` and `calibre-web` Exited; ten sibling containers came back and
// those two did not, and they were still down ~18 hours later. The controller REPORTED them (the
// 30 s deadapp-check) but never started them — deployed-but-stopped was an alarm with no recovery.
//
// Two deliberate boundaries, both load-bearing:
//
//   - **Bounded, never a loop.** At most `attempts` tries, `retryDelay` apart, then it stops and the
//     alarm owns the problem. A restart loop would paper over a genuinely broken app forever and
//     hammer docker while doing it.
//   - **A user's Stop survives a reboot.** This is still the whole safety argument; only the way it
//     is established changed. Until v0.189.0 it was inferred — the UI's Stop is `docker compose
//     down`, which REMOVES containers, so "zero containers" was read as "the customer stopped it"
//     and left alone. Since v0.189.0 (R-166) the customer's intent is RECORDED in app.yaml and read
//     directly, because the inference could not distinguish a deliberate Stop from a power cut or an
//     interrupted backup, and silently stranded both. An app.yaml with no recorded intent — every
//     app on every box predating the field — keeps the old inference exactly. See isBootOrphan,
//     TestReconcile_UserStoppedAppIsNeverStarted and
//     TestReconcile_LegacyNoDesiredState_BehavesExactlyAsBefore.
//
// It runs inside the notifier's boot grace (cmd/controller/main.go `deadAppBootGrace`), so a
// successful recovery never fires an alert and a failed one alerts honestly once the grace expires.
package bootrecon

import (
	"context"
	"log"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)

// StackProvider is the slice of *stacks.Manager this package needs. Declared consumer-side so the
// tests can count StartStack calls without a docker anywhere near them.
type StackProvider interface {
	GetStacks() []stacks.Stack
	StartStack(name string) error
	RefreshStatus() error
}

// StartGate answers the one question this package must ask before starting anything: **may this app
// be started right now?** Declared consumer-side, in the style of StackProvider, so `bootrecon`
// still imports `stacks` alone and knows nothing about settings, the agent, quiesce or the web layer.
//
// It is ONE seam rather than three because the three reasons a boot orphan must NOT be started share
// a shape — something else is deliberately holding this app — and differ only in the reason string:
//
//	the drive-absent gate stopped it     → its drive is not live (R-171, below)
//	a quiesce is holding it for a backup → the quiesce loop restarts its own stacks
//	an app-data operation stopped it     → the app-stop guard's own Recover owns it
//
// A widened boot window (R-157 mechanism A) is what makes the last two reachable at all: the old
// T+5 s single sweep never overlapped them.
//
// R-171 — WHY THIS EXISTS, and it is a regression this package caused. Until v0.189.0 the sweep
// required a stack to still HAVE containers, and an app the drive-absent gate had stopped has zero,
// so such apps were skipped by accident. v0.189.0 replaced that term with the customer's recorded
// intent — correctly — and the drive gate does NOT change `desired_state` (it is not the customer),
// so a gate-stopped app now reads as `running` + zero containers, i.e. a boot orphan. Observed live
// on 2026-08-02: the sweep found and started an app whose drive was unmounted, burned both attempts,
// and handed it to the dead-app alarm — a false alarm about an app the drive gate is deliberately
// holding (audits/DIAG-bootrecon-drive-absent-2026-08-02.md).
//
// The rule itself is not new and is not invented here: the API's own start path already refuses this
// (`startGatedByMissingDrive`, internal/api/router.go) with a Hungarian message to the customer. The
// sweep simply bypassed it by calling Manager.StartStack directly. This seam gives the sweep the
// same question to ask.
//
// CONTRACT — the answer is fail-safe by design (§8.4): an implementation that CANNOT DETERMINE
// whether the drive is live must return false, not true. Not starting is recoverable — the drive
// gate's `Return` branch restarts the app when the drive comes back, and the dead-app alarm reports
// it meanwhile. Starting on an absent drive is not recoverable by anything automatic: compose
// creates the bind sources wherever the mountpoint currently points, which is the guest rootfs.
type StartGate interface {
	// MayStart reports whether the named stack may be started. The reason is for the log line and is
	// only read when may is false.
	MayStart(stackName string) (may bool, reason string)
}

const (
	// DefaultAttempts is the total number of start attempts per boot (not per app per retry-forever).
	DefaultAttempts = 2
	// DefaultRetryDelay spaces the attempts. 2 × 30 s fits comfortably inside the 90 s boot grace,
	// so a recovery that works is silent and one that does not is honest.
	DefaultRetryDelay = 30 * time.Second
)

// Reconciler performs the start-once sweep. Zero value is not usable — use New.
type Reconciler struct {
	stacks     StackProvider
	logger     *log.Logger
	attempts   int
	retryDelay time.Duration

	// sleep is the inter-attempt wait; injectable so tests never spend 30 real seconds.
	sleep func(context.Context, time.Duration)

	// startGate (R-171) refuses to start an app something else is deliberately holding. nil = NOT
	// WIRED, which means "this caller has no such concept" and is permissive — the test fixtures'
	// case. It is NOT the same as "cannot determine", which the gate itself answers with false (see
	// StartGate's contract). Production MUST wire it; TestMainWiresBootDriveGate walks main.go's AST
	// for the call, because an unwired seam here is silently the pre-v0.190.0 behaviour.
	startGate StartGate
}

// Result is the outcome, returned for logging/testing (the hub learns about failures only through
// the existing app_start_failed alarm — this package deliberately pushes no events of its own).
type Result struct {
	Candidates []string // boot-orphaned apps found
	Recovered  []string // running again by the end
	StillDown  []string // still down after the last attempt — the alarm's problem now
	Attempts   int      // attempts actually made (0 when there was nothing to do)
	// HeldByDrive (R-171) are apps that ARE boot orphans by intent but which something else is
	// deliberately holding (an absent drive, a quiesce, an app-data operation), so they were not
	// started. Reported separately from StillDown because they are not a fault this sweep failed to
	// fix — the holder owns their recovery. Collapsing the two would put a deliberately-held app in
	// the same bucket as a broken one, which is the false alarm R-171 removes.
	HeldByDrive []string
}

// SetDriveGate wires the R-171 start refusal. INIT-ONLY — call once, before Run.
func (r *Reconciler) SetDriveGate(g StartGate) { r.startGate = g }

// mayStart asks the gate, or allows when none is wired (see the startGate field comment).
func (r *Reconciler) mayStart(stackName string) (bool, string) {
	if r.startGate == nil {
		return true, ""
	}
	return r.startGate.MayStart(stackName)
}

// New builds a Reconciler with the shipped defaults.
func New(p StackProvider, logger *log.Logger) *Reconciler {
	return &Reconciler{
		stacks:     p,
		logger:     logger,
		attempts:   DefaultAttempts,
		retryDelay: DefaultRetryDelay,
		sleep:      sleepCtx,
	}
}

func sleepCtx(ctx context.Context, d time.Duration) {
	t := time.NewTimer(d)
	defer t.Stop()
	select {
	case <-ctx.Done():
	case <-t.C:
	}
}

// isBootOrphan reports whether a stack is an app the boot left behind.
//
// The gate, term by term:
//   - Deployed — an app that is installed. NOTE: `Deployed` means INSTALLED, not "wanted running";
//     the two were conflated until v0.189.0 and that conflation is what the desired-state term below
//     repairs.
//   - not Protected — traefik/cloudflared/felhom-controller have their own supervision; this must
//     never race the base-stack self-heal.
//   - not Deploying — mid-deploy is not a fault.
//   - desired state — see below. REPLACES the old container-count term.
//   - IsDownState — stopped/exited/degraded (R-51 included: a boot that half-started a stack is the
//     same interrupted-boot shape).
//
// ── WHY INTENT REPLACED THE CONTAINER COUNT (R-166, closing R-157 mechanism B) ────────────────────
//
// This gate used to end in `len(s.Containers) > 0`, and its comment called that "the D-case guard":
// a UI Stop is `compose down`, which REMOVES containers, so zero containers was read as "the
// customer stopped this" and left alone. The safety goal was right and still holds. The SIGNAL was
// wrong, because zero containers has at least three causes and the count cannot tell them apart:
//
//	a deliberate Stop   → must stay down
//	a power cut mid-compose, or an interrupted deploy → must come back
//	a backup that stopped the app and died before restarting it → must come back
//
// Two of those three were silently unrecoverable: the app simply stayed gone until a human noticed.
// The count was never capable of separating them, so the fix is not a better inference — it is to
// stop inferring and read what the customer actually asked for, which app.yaml now records.
//
// ── WHAT ABSENT STILL MEANS, AND WHY THE OLD BEHAVIOUR IS KEPT ────────────────────────────────────
//
// DesiredStateUnknown falls back to the ORIGINAL container-count rule, byte-for-byte. This is the
// single most important line in the change. Every app.yaml on every existing box predates the field,
// so absent is what the whole fleet reads on upgrade; treating absent as "running" would start, on
// the first boot after the upgrade, every app its owner had deliberately stopped. The fallback is
// what makes this feature inert for an app nobody has pressed a button on since — see
// TestReconcile_LegacyNoDesiredState_BehavesExactlyAsBefore and its red-proof.
//
// The full decision table (§8.1):
//
//	desired   containers  state        → result
//	stopped   any         any          → never an orphan (the customer said so)
//	running   0           —            → ORPHAN  ← the R-157 case, invisible before v0.189.0
//	running   >0          IsDownState  → ORPHAN  (unchanged)
//	running   >0          up           → not an orphan
//	absent    0           —            → not an orphan  (exactly the pre-v0.189.0 behaviour)
//	absent    >0          IsDownState  → ORPHAN          (exactly the pre-v0.189.0 behaviour)
func isBootOrphan(s stacks.Stack) bool {
	if !s.Deployed || s.Protected || s.Deploying {
		return false
	}
	switch stacks.DesiredStateOf(s) {
	case stacks.DesiredStateStopped:
		// The customer pressed Stop. No observation may overturn that — not a missing container, not
		// a down state, not a reboot. Nothing else in this package starts an app.
		return false
	case stacks.DesiredStateRunning:
		// Wanted running. ANY way of not being up is a fault to repair, including having no
		// containers at all — which is the case the old count term structurally could not see.
		return len(s.Containers) == 0 || stacks.IsDownState(s.State)
	default:
		// DesiredStateUnknown — legacy. Keep the pre-R-166 rule exactly.
		return len(s.Containers) > 0 && stacks.IsDownState(s.State)
	}
}

// Run performs the sweep once and returns what happened. It is safe to call with no boot orphans
// (the quiet path logs one DEBUG-free INFO-free line — see below) and it never returns an error:
// a failure to start is an app-level fact the alarm reports, not a controller startup failure.
func (r *Reconciler) Run(ctx context.Context) Result {
	var res Result

	pending := map[string]bool{}
	for _, s := range r.stacks.GetStacks() {
		if !isBootOrphan(s) {
			continue
		}
		// R-171: intent says this app should be running and it is not — but if something else is
		// deliberately holding it (absent drive, quiesce, an app-data operation), starting it is the
		// wrong repair. Refuse, loudly, and let the holder own it.
		if live, reason := r.mayStart(s.Name); !live {
			res.HeldByDrive = append(res.HeldByDrive, s.Name)
			r.logger.Printf("[INFO] [bootrecon] %q is a boot orphan by intent but is HELD (%s) — NOT starting it; whatever is holding it owns its recovery",
				s.Name, reason)
			continue
		}
		pending[s.Name] = true
		res.Candidates = append(res.Candidates, s.Name)
	}
	sortStrings(res.Candidates)
	sortStrings(res.HeldByDrive)

	if len(pending) == 0 {
		// The healthy path must be observable — "no alarms" and "never ran" have to be
		// distinguishable in a log (the v0.91.2 lesson). "Nothing to start" and "everything I found
		// is held by an absent drive" must be distinguishable too, or the held case reads as healthy.
		if len(res.HeldByDrive) > 0 {
			r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: nothing to start — %d app(s) held (absent drive / quiesce / app-data operation): %v",
				len(res.HeldByDrive), res.HeldByDrive)
			return res
		}
		r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: no boot-orphaned apps (nothing to start)")
		return res
	}

	r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: %d boot-orphaned app(s) found: %v — up to %d attempt(s)",
		len(res.Candidates), res.Candidates, r.attempts)

	for attempt := 1; attempt <= r.attempts && len(pending) > 0; attempt++ {
		res.Attempts = attempt
		for _, name := range sortedKeys(pending) {
			if ctx.Err() != nil {
				break
			}
			start := time.Now()
			if err := r.stacks.StartStack(name); err != nil {
				r.logger.Printf("[WARN] [bootrecon] Boot reconciliation attempt %d/%d: start %q failed after %.1fs: %v",
					attempt, r.attempts, name, time.Since(start).Seconds(), err)
				continue
			}
			r.logger.Printf("[INFO] [bootrecon] Boot reconciliation attempt %d/%d: started %q (took %.1fs)",
				attempt, r.attempts, name, time.Since(start).Seconds())
		}
		if ctx.Err() != nil {
			break
		}

		// Re-read reality rather than trusting a nil error: `compose up -d` exits 0 on a crash-loop
		// (a session-critical invariant of this repo), so only a fresh docker ps can say whether the
		// app is actually up.
		if err := r.stacks.RefreshStatus(); err != nil {
			r.logger.Printf("[WARN] [bootrecon] Boot reconciliation: status refresh failed: %v", err)
		}
		for _, s := range r.stacks.GetStacks() {
			if pending[s.Name] && !stacks.IsDownState(s.State) {
				delete(pending, s.Name)
				res.Recovered = append(res.Recovered, s.Name)
			}
		}

		if len(pending) > 0 && attempt < r.attempts {
			r.sleep(ctx, r.retryDelay)
		}
	}

	res.StillDown = sortedKeys(pending)
	sortStrings(res.Recovered)

	if len(res.StillDown) == 0 {
		r.logger.Printf("[INFO] [bootrecon] Boot reconciliation complete: %d app(s) recovered in %d attempt(s): %v",
			len(res.Recovered), res.Attempts, res.Recovered)
	} else {
		// Deliberately no hub event here: the app_start_failed alarm fires on its own once the boot
		// grace expires, and two events for one dead app is how an operator inbox becomes noise.
		r.logger.Printf("[WARN] [bootrecon] Boot reconciliation gave up after %d attempt(s): recovered=%v still down=%v (the dead-app alarm now owns these)",
			res.Attempts, res.Recovered, res.StillDown)
	}
	return res
}

// --- tiny local helpers (no dependency on sort ordering semantics elsewhere) ---

func sortedKeys(m map[string]bool) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sortStrings(out)
	return out
}

func sortStrings(s []string) {
	for i := 1; i < len(s); i++ {
		for j := i; j > 0 && s[j] < s[j-1]; j-- {
			s[j], s[j-1] = s[j-1], s[j]
		}
	}
}