felhom-controller/controller/internal/bootrecon/bootrecon.go

// Package bootrecon implements R-52: the bounded, start-ONCE recovery of apps that were left
// behind by an interrupted boot.
//
// The live failure it closes (AUDIT-vacation-remote-ops-2026-07-20, finding F5): a pre-transport
// shutdown left `immich-server` and `calibre-web` Exited; ten sibling containers came back and
// those two did not, and they were still down ~18 hours later. The controller REPORTED them (the
// 30 s deadapp-check) but never started them — deployed-but-stopped was an alarm with no recovery.
//
// Two deliberate boundaries, both load-bearing:
//
//   - **Bounded, never a loop.** At most `attempts` tries, `retryDelay` apart, then it stops and the
//     alarm owns the problem. A restart loop would paper over a genuinely broken app forever and
//     hammer docker while doing it.
//   - **A user's Stop survives a reboot.** The UI's Stop is `docker compose down`, which REMOVES the
//     containers; a boot interruption leaves them behind as Exited. So "has containers on disk that
//     are down" is the boot-orphan signature, and a stack with ZERO containers is deliberately never
//     touched. This distinction is the whole safety argument — see TestReconcile_UserStoppedAppIsNeverStarted.
//
// It runs inside the notifier's boot grace (cmd/controller/main.go `deadAppBootGrace`), so a
// successful recovery never fires an alert and a failed one alerts honestly once the grace expires.
package bootrecon

import (
	"context"
	"log"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)

// StackProvider is the slice of *stacks.Manager this package needs. Declared consumer-side so the
// tests can count StartStack calls without a docker anywhere near them.
type StackProvider interface {
	GetStacks() []stacks.Stack
	StartStack(name string) error
	RefreshStatus() error
}

const (
	// DefaultAttempts is the total number of start attempts per boot (not per app per retry-forever).
	DefaultAttempts = 2
	// DefaultRetryDelay spaces the attempts. 2 × 30 s fits comfortably inside the 90 s boot grace,
	// so a recovery that works is silent and one that does not is honest.
	DefaultRetryDelay = 30 * time.Second
)

// Reconciler performs the start-once sweep. Zero value is not usable — use New.
type Reconciler struct {
	stacks     StackProvider
	logger     *log.Logger
	attempts   int
	retryDelay time.Duration

	// sleep is the inter-attempt wait; injectable so tests never spend 30 real seconds.
	sleep func(context.Context, time.Duration)
}

// Result is the outcome, returned for logging/testing (the hub learns about failures only through
// the existing app_start_failed alarm — this package deliberately pushes no events of its own).
type Result struct {
	Candidates []string // boot-orphaned apps found
	Recovered  []string // running again by the end
	StillDown  []string // still down after the last attempt — the alarm's problem now
	Attempts   int      // attempts actually made (0 when there was nothing to do)
}

// New builds a Reconciler with the shipped defaults.
func New(p StackProvider, logger *log.Logger) *Reconciler {
	return &Reconciler{
		stacks:     p,
		logger:     logger,
		attempts:   DefaultAttempts,
		retryDelay: DefaultRetryDelay,
		sleep:      sleepCtx,
	}
}

func sleepCtx(ctx context.Context, d time.Duration) {
	t := time.NewTimer(d)
	defer t.Stop()
	select {
	case <-ctx.Done():
	case <-t.C:
	}
}

// isBootOrphan reports whether a stack is an app the boot left behind.
//
// The gate, term by term:
//   - Deployed — an app the customer asked to have running.
//   - not Protected — traefik/cloudflared/felhom-controller have their own supervision; this must
//     never race the base-stack self-heal.
//   - not Deploying — mid-deploy is not a fault.
//   - has containers — the D-case guard: a UI Stop removes them, and a deliberate stop must survive
//     a reboot.
//   - IsDownState — stopped/exited/degraded (R-51 included: a boot that half-started a stack is the
//     same interrupted-boot shape).
func isBootOrphan(s stacks.Stack) bool {
	return s.Deployed && !s.Protected && !s.Deploying &&
		len(s.Containers) > 0 && stacks.IsDownState(s.State)
}

// Run performs the sweep once and returns what happened. It is safe to call with no boot orphans
// (the quiet path logs one DEBUG-free INFO-free line — see below) and it never returns an error:
// a failure to start is an app-level fact the alarm reports, not a controller startup failure.
func (r *Reconciler) Run(ctx context.Context) Result {
	var res Result

	pending := map[string]bool{}
	for _, s := range r.stacks.GetStacks() {
		if isBootOrphan(s) {
			pending[s.Name] = true
			res.Candidates = append(res.Candidates, s.Name)
		}
	}
	sortStrings(res.Candidates)

	if len(pending) == 0 {
		// The healthy path must be observable — "no alarms" and "never ran" have to be
		// distinguishable in a log (the v0.91.2 lesson).
		r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: no boot-orphaned apps (nothing to start)")
		return res
	}

	r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: %d boot-orphaned app(s) found: %v — up to %d attempt(s)",
		len(res.Candidates), res.Candidates, r.attempts)

	for attempt := 1; attempt <= r.attempts && len(pending) > 0; attempt++ {
		res.Attempts = attempt
		for _, name := range sortedKeys(pending) {
			if ctx.Err() != nil {
				break
			}
			start := time.Now()
			if err := r.stacks.StartStack(name); err != nil {
				r.logger.Printf("[WARN] [bootrecon] Boot reconciliation attempt %d/%d: start %q failed after %.1fs: %v",
					attempt, r.attempts, name, time.Since(start).Seconds(), err)
				continue
			}
			r.logger.Printf("[INFO] [bootrecon] Boot reconciliation attempt %d/%d: started %q (took %.1fs)",
				attempt, r.attempts, name, time.Since(start).Seconds())
		}
		if ctx.Err() != nil {
			break
		}

		// Re-read reality rather than trusting a nil error: `compose up -d` exits 0 on a crash-loop
		// (a session-critical invariant of this repo), so only a fresh docker ps can say whether the
		// app is actually up.
		if err := r.stacks.RefreshStatus(); err != nil {
			r.logger.Printf("[WARN] [bootrecon] Boot reconciliation: status refresh failed: %v", err)
		}
		for _, s := range r.stacks.GetStacks() {
			if pending[s.Name] && !stacks.IsDownState(s.State) {
				delete(pending, s.Name)
				res.Recovered = append(res.Recovered, s.Name)
			}
		}

		if len(pending) > 0 && attempt < r.attempts {
			r.sleep(ctx, r.retryDelay)
		}
	}

	res.StillDown = sortedKeys(pending)
	sortStrings(res.Recovered)

	if len(res.StillDown) == 0 {
		r.logger.Printf("[INFO] [bootrecon] Boot reconciliation complete: %d app(s) recovered in %d attempt(s): %v",
			len(res.Recovered), res.Attempts, res.Recovered)
	} else {
		// Deliberately no hub event here: the app_start_failed alarm fires on its own once the boot
		// grace expires, and two events for one dead app is how an operator inbox becomes noise.
		r.logger.Printf("[WARN] [bootrecon] Boot reconciliation gave up after %d attempt(s): recovered=%v still down=%v (the dead-app alarm now owns these)",
			res.Attempts, res.Recovered, res.StillDown)
	}
	return res
}

// --- tiny local helpers (no dependency on sort ordering semantics elsewhere) ---

func sortedKeys(m map[string]bool) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sortStrings(out)
	return out
}

func sortStrings(s []string) {
	for i := 1; i < len(s); i++ {
		for j := i; j > 0 && s[j] < s[j-1]; j-- {
			s[j], s[j-1] = s[j-1], s[j]
		}
	}
}