// Package aftercrash runs the apps' after_unclean_boot blocks ONCE per controller start that followed an unclean // host boot (R-1007, `09` §3 decision 236). // // The fact is crashboot's (the host crash guard, read through the agent): the same verdict that holds the app mails // for CrashGrace. Unknown is a normal boot — a repair never runs on a guess. A normal boot never runs it. After a crash // boot it waits CrashGrace from the controller's start (the apps come up slowly after a crash), then runs each app's // block one after the other and reports each app's counts. It never stops an app; a failed run is reported, not // retried. Pinned by aftercrash_test.go. package aftercrash import ( "log" "sync" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/crashboot" "gitea.dooplex.hu/admin/felhom-controller/internal/stacks" ) // Verdict is crashboot.Grace's Decided (a seam for tests). type Verdict func(now time.Time) (resolved, crash bool, bootAt time.Time) // Runner does the one run. Every func field is required; Logger may be nil. type Runner struct { Start time.Time Verdict Verdict Apps func() []string Run func(name string, bootAt time.Time) stacks.AfterUncleanBootResult Report func(name string, bootAt time.Time, res stacks.AfterUncleanBootResult) Logger *log.Logger // Wait is how long after Start a crash-boot run waits (default crashboot.CrashGrace). Wait time.Duration mu sync.Mutex done bool } // New builds the runner on the controller's crash-boot grace. func New(g *crashboot.Grace, apps func() []string, run func(string, time.Time) stacks.AfterUncleanBootResult, report func(string, time.Time, stacks.AfterUncleanBootResult), logger *log.Logger) *Runner { return &Runner{Start: g.Start(), Verdict: g.Decided, Apps: apps, Run: run, Report: report, Logger: logger} } // Done reports whether this start's decision has been taken (ran, or decided not to). func (r *Runner) Done() bool { r.mu.Lock() defer r.mu.Unlock() return r.done } // Tick is called periodically (the scheduler). It returns true once the run has happened or been ruled out. func (r *Runner) Tick(now time.Time) bool { r.mu.Lock() if r.done { r.mu.Unlock() return true } resolved, crash, bootAt := r.Verdict(now) if !resolved { r.mu.Unlock() return false } if !crash { r.done = true r.mu.Unlock() r.logf("[INFO] [aftercrash] a normal (or unknown) boot — no app repair after an unclean boot (R-1007)") return true } wait := r.Wait if wait <= 0 { wait = crashboot.CrashGrace } if now.Sub(r.Start) < wait { r.mu.Unlock() return false } r.done = true // set BEFORE the run: a slow or failing run is never started twice r.mu.Unlock() apps := r.Apps() r.logf("[INFO] [aftercrash] the host's boot at %s followed an unclean stop — %d app(s) with an after_unclean_boot block (R-1007)", bootAt.UTC().Format(time.RFC3339), len(apps)) for _, name := range apps { res := r.Run(name, bootAt) if res.Skipped != "" { r.logf("[WARN] [aftercrash] %s: skipped — %s", name, res.Skipped) } else { r.logf("[INFO] [aftercrash] %s: found %d empty file(s), repaired %d, still broken %d, lost %d (checked=%t) in %s%s", name, res.Found, res.Repaired, res.StillBroken, res.Lost, res.LostChecked, res.Duration.Round(time.Millisecond), problem(res.Problem)) } r.Report(name, bootAt, res) } return true } func problem(p string) string { if p == "" { return "" } return " — problem: " + p } func (r *Runner) logf(format string, args ...any) { if r.Logger != nil { r.Logger.Printf(format, args...) } }