felhom-controller/controller/internal/aftercrash/aftercrash.go

// Package aftercrash runs the apps' after_unclean_boot blocks ONCE per controller start that followed an unclean
// host boot (R-1007, `09` §3 decision 236).
//
// The fact is crashboot's (the host crash guard, read through the agent): the same verdict that holds the app mails
// for CrashGrace. Unknown is a normal boot — a repair never runs on a guess. A normal boot never runs it. After a crash
// boot it waits CrashGrace from the controller's start (the apps come up slowly after a crash), then runs each app's
// block one after the other and reports each app's counts. It never stops an app; a failed run is reported, not
// retried. Pinned by aftercrash_test.go.
package aftercrash

import (
	"log"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/crashboot"
	"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)

// Verdict is crashboot.Grace's Decided (a seam for tests).
type Verdict func(now time.Time) (resolved, crash bool, bootAt time.Time)

// Runner does the one run. Every func field is required; Logger may be nil.
type Runner struct {
	Start   time.Time
	Verdict Verdict
	Apps    func() []string
	Run     func(name string, bootAt time.Time) stacks.AfterUncleanBootResult
	Report  func(name string, bootAt time.Time, res stacks.AfterUncleanBootResult)
	Logger  *log.Logger
	// Wait is how long after Start a crash-boot run waits (default crashboot.CrashGrace).
	Wait time.Duration

	mu   sync.Mutex
	done bool
}

// New builds the runner on the controller's crash-boot grace.
func New(g *crashboot.Grace, apps func() []string,
	run func(string, time.Time) stacks.AfterUncleanBootResult,
	report func(string, time.Time, stacks.AfterUncleanBootResult), logger *log.Logger) *Runner {
	return &Runner{Start: g.Start(), Verdict: g.Decided, Apps: apps, Run: run, Report: report, Logger: logger}
}

// Done reports whether this start's decision has been taken (ran, or decided not to).
func (r *Runner) Done() bool {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.done
}

// Tick is called periodically (the scheduler). It returns true once the run has happened or been ruled out.
func (r *Runner) Tick(now time.Time) bool {
	r.mu.Lock()
	if r.done {
		r.mu.Unlock()
		return true
	}
	resolved, crash, bootAt := r.Verdict(now)
	if !resolved {
		r.mu.Unlock()
		return false
	}
	if !crash {
		r.done = true
		r.mu.Unlock()
		r.logf("[INFO] [aftercrash] a normal (or unknown) boot — no app repair after an unclean boot (R-1007)")
		return true
	}
	wait := r.Wait
	if wait <= 0 {
		wait = crashboot.CrashGrace
	}
	if now.Sub(r.Start) < wait {
		r.mu.Unlock()
		return false
	}
	r.done = true // set BEFORE the run: a slow or failing run is never started twice
	r.mu.Unlock()

	apps := r.Apps()
	r.logf("[INFO] [aftercrash] the host's boot at %s followed an unclean stop — %d app(s) with an after_unclean_boot block (R-1007)",
		bootAt.UTC().Format(time.RFC3339), len(apps))
	for _, name := range apps {
		res := r.Run(name, bootAt)
		if res.Skipped != "" {
			r.logf("[WARN] [aftercrash] %s: skipped — %s", name, res.Skipped)
		} else {
			r.logf("[INFO] [aftercrash] %s: found %d empty file(s), repaired %d, still broken %d, lost %d (checked=%t) in %s%s",
				name, res.Found, res.Repaired, res.StillBroken, res.Lost, res.LostChecked, res.Duration.Round(time.Millisecond), problem(res.Problem))
		}
		r.Report(name, bootAt, res)
	}
	return true
}

func problem(p string) string {
	if p == "" {
		return ""
	}
	return " — problem: " + p
}

func (r *Runner) logf(format string, args ...any) {
	if r.Logger != nil {
		r.Logger.Printf(format, args...)
	}
}