feat(v0.156.0): dead-primary alerting (R-51) + boot desired-state reconciliation (R-52)

R-51: aggregateState's mixed branch returned StateRunning ("partial"), so a stack whose
MAIN container was dead behind live helpers alerted on nothing — immich-server sat Exited
for 18 h, 100 % unreachable, no banner and no app_start_failed (audit F4). New
StateDegraded: a DOWN member whose docker restart policy is always/unless-stopped is a
fault (degraded, a down state); no/on-failure is a finished one-shot and stays benign; an
unreadable policy fails CLOSED. The unhealthy/restarting/paused/unknown exclusions are
byte-identical — folding unhealthy into down is the flapping fix-3 avoided.

R-52: new internal/bootrecon — one bounded start-once sweep at startup (2 attempts, 30 s
apart) for apps an interrupted boot left behind, inside the 90 s boot grace so a success
is silent and a failure still alerts. A zero-container stack is NEVER touched: the UI's
Stop is compose down, so a deliberate stop survives a reboot.

Both features carry a production-path wiring test (the v0.154.0 / v0.91.0 inert-seam
class). The main() assertion is an AST walk, not strings.Contains — the substring version
passed its own red-proof, because a commented-out call still contains the string.

Red-proofs run and restored: mix branch reverted -> "running" on the immich fixture;
boot hook commented out -> wiring test fails; zero-container gate dropped -> the
user-stopped app gets started.

NOTE: controller/cmd/controller/ is matched by .gitignore's `controller` entry, so new
files there need `git add -f` (and ripgrep silently skips main.go without --no-ignore).
This commit is contained in:
2026-07-21 12:27:33 +02:00
parent 0f9b29a19a
commit 285dd1032f
14 changed files with 1167 additions and 22 deletions
+197
View File
@@ -0,0 +1,197 @@
// Package bootrecon implements R-52: the bounded, start-ONCE recovery of apps that were left
// behind by an interrupted boot.
//
// The live failure it closes (AUDIT-vacation-remote-ops-2026-07-20, finding F5): a pre-transport
// shutdown left `immich-server` and `calibre-web` Exited; ten sibling containers came back and
// those two did not, and they were still down ~18 hours later. The controller REPORTED them (the
// 30 s deadapp-check) but never started them — deployed-but-stopped was an alarm with no recovery.
//
// Two deliberate boundaries, both load-bearing:
//
// - **Bounded, never a loop.** At most `attempts` tries, `retryDelay` apart, then it stops and the
// alarm owns the problem. A restart loop would paper over a genuinely broken app forever and
// hammer docker while doing it.
// - **A user's Stop survives a reboot.** The UI's Stop is `docker compose down`, which REMOVES the
// containers; a boot interruption leaves them behind as Exited. So "has containers on disk that
// are down" is the boot-orphan signature, and a stack with ZERO containers is deliberately never
// touched. This distinction is the whole safety argument — see TestReconcile_UserStoppedAppIsNeverStarted.
//
// It runs inside the notifier's boot grace (cmd/controller/main.go `deadAppBootGrace`), so a
// successful recovery never fires an alert and a failed one alerts honestly once the grace expires.
package bootrecon
import (
"context"
"log"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)
// StackProvider is the slice of *stacks.Manager this package needs. Declared consumer-side so the
// tests can count StartStack calls without a docker anywhere near them.
type StackProvider interface {
GetStacks() []stacks.Stack
StartStack(name string) error
RefreshStatus() error
}
const (
// DefaultAttempts is the total number of start attempts per boot (not per app per retry-forever).
DefaultAttempts = 2
// DefaultRetryDelay spaces the attempts. 2 × 30 s fits comfortably inside the 90 s boot grace,
// so a recovery that works is silent and one that does not is honest.
DefaultRetryDelay = 30 * time.Second
)
// Reconciler performs the start-once sweep. Zero value is not usable — use New.
type Reconciler struct {
stacks StackProvider
logger *log.Logger
attempts int
retryDelay time.Duration
// sleep is the inter-attempt wait; injectable so tests never spend 30 real seconds.
sleep func(context.Context, time.Duration)
}
// Result is the outcome, returned for logging/testing (the hub learns about failures only through
// the existing app_start_failed alarm — this package deliberately pushes no events of its own).
type Result struct {
Candidates []string // boot-orphaned apps found
Recovered []string // running again by the end
StillDown []string // still down after the last attempt — the alarm's problem now
Attempts int // attempts actually made (0 when there was nothing to do)
}
// New builds a Reconciler with the shipped defaults.
func New(p StackProvider, logger *log.Logger) *Reconciler {
return &Reconciler{
stacks: p,
logger: logger,
attempts: DefaultAttempts,
retryDelay: DefaultRetryDelay,
sleep: sleepCtx,
}
}
func sleepCtx(ctx context.Context, d time.Duration) {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
case <-t.C:
}
}
// isBootOrphan reports whether a stack is an app the boot left behind.
//
// The gate, term by term:
// - Deployed — an app the customer asked to have running.
// - not Protected — traefik/cloudflared/felhom-controller have their own supervision; this must
// never race the base-stack self-heal.
// - not Deploying — mid-deploy is not a fault.
// - has containers — the D-case guard: a UI Stop removes them, and a deliberate stop must survive
// a reboot.
// - IsDownState — stopped/exited/degraded (R-51 included: a boot that half-started a stack is the
// same interrupted-boot shape).
func isBootOrphan(s stacks.Stack) bool {
return s.Deployed && !s.Protected && !s.Deploying &&
len(s.Containers) > 0 && stacks.IsDownState(s.State)
}
// Run performs the sweep once and returns what happened. It is safe to call with no boot orphans
// (the quiet path logs one DEBUG-free INFO-free line — see below) and it never returns an error:
// a failure to start is an app-level fact the alarm reports, not a controller startup failure.
func (r *Reconciler) Run(ctx context.Context) Result {
var res Result
pending := map[string]bool{}
for _, s := range r.stacks.GetStacks() {
if isBootOrphan(s) {
pending[s.Name] = true
res.Candidates = append(res.Candidates, s.Name)
}
}
sortStrings(res.Candidates)
if len(pending) == 0 {
// The healthy path must be observable — "no alarms" and "never ran" have to be
// distinguishable in a log (the v0.91.2 lesson).
r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: no boot-orphaned apps (nothing to start)")
return res
}
r.logger.Printf("[INFO] [bootrecon] Boot reconciliation: %d boot-orphaned app(s) found: %v — up to %d attempt(s)",
len(res.Candidates), res.Candidates, r.attempts)
for attempt := 1; attempt <= r.attempts && len(pending) > 0; attempt++ {
res.Attempts = attempt
for _, name := range sortedKeys(pending) {
if ctx.Err() != nil {
break
}
start := time.Now()
if err := r.stacks.StartStack(name); err != nil {
r.logger.Printf("[WARN] [bootrecon] Boot reconciliation attempt %d/%d: start %q failed after %.1fs: %v",
attempt, r.attempts, name, time.Since(start).Seconds(), err)
continue
}
r.logger.Printf("[INFO] [bootrecon] Boot reconciliation attempt %d/%d: started %q (took %.1fs)",
attempt, r.attempts, name, time.Since(start).Seconds())
}
if ctx.Err() != nil {
break
}
// Re-read reality rather than trusting a nil error: `compose up -d` exits 0 on a crash-loop
// (a session-critical invariant of this repo), so only a fresh docker ps can say whether the
// app is actually up.
if err := r.stacks.RefreshStatus(); err != nil {
r.logger.Printf("[WARN] [bootrecon] Boot reconciliation: status refresh failed: %v", err)
}
for _, s := range r.stacks.GetStacks() {
if pending[s.Name] && !stacks.IsDownState(s.State) {
delete(pending, s.Name)
res.Recovered = append(res.Recovered, s.Name)
}
}
if len(pending) > 0 && attempt < r.attempts {
r.sleep(ctx, r.retryDelay)
}
}
res.StillDown = sortedKeys(pending)
sortStrings(res.Recovered)
if len(res.StillDown) == 0 {
r.logger.Printf("[INFO] [bootrecon] Boot reconciliation complete: %d app(s) recovered in %d attempt(s): %v",
len(res.Recovered), res.Attempts, res.Recovered)
} else {
// Deliberately no hub event here: the app_start_failed alarm fires on its own once the boot
// grace expires, and two events for one dead app is how an operator inbox becomes noise.
r.logger.Printf("[WARN] [bootrecon] Boot reconciliation gave up after %d attempt(s): recovered=%v still down=%v (the dead-app alarm now owns these)",
res.Attempts, res.Recovered, res.StillDown)
}
return res
}
// --- tiny local helpers (no dependency on sort ordering semantics elsewhere) ---
func sortedKeys(m map[string]bool) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sortStrings(out)
return out
}
func sortStrings(s []string) {
for i := 1; i < len(s); i++ {
for j := i; j > 0 && s[j] < s[j-1]; j-- {
s[j], s[j-1] = s[j-1], s[j]
}
}
}
@@ -0,0 +1,280 @@
package bootrecon
import (
"context"
"errors"
"io"
"log"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)
// fakeStacks counts StartStack calls per app — the assertion that matters in BOTH directions:
// scenario C needs exactly-bounded starts, scenario D needs a start count of ZERO.
type fakeStacks struct {
list []stacks.Stack
starts map[string]int
failWith map[string]error
// onStart mutates the world the way a real successful start would (fresh docker ps).
onStart func(f *fakeStacks, name string)
refreshN int
}
func (f *fakeStacks) GetStacks() []stacks.Stack { return f.list }
func (f *fakeStacks) RefreshStatus() error { f.refreshN++; return nil }
func (f *fakeStacks) StartStack(name string) error {
if f.starts == nil {
f.starts = map[string]int{}
}
f.starts[name]++
if err := f.failWith[name]; err != nil {
return err
}
if f.onStart != nil {
f.onStart(f, name)
}
return nil
}
func (f *fakeStacks) setState(name string, st stacks.ContainerState) {
for i := range f.list {
if f.list[i].Name == name {
f.list[i].State = st
}
}
}
// comesUp is the ordinary success behaviour: the app is running after the start.
func comesUp(f *fakeStacks, name string) { f.setState(name, stacks.StateRunning) }
func newTestReconciler(f *fakeStacks) (*Reconciler, *int) {
slept := 0
r := New(f, log.New(io.Discard, "", 0))
r.sleep = func(context.Context, time.Duration) { slept++ }
return r, &slept
}
// bootOrphan is the F5 shape: deployed, containers still present on disk, all Exited — the boot
// interrupted them, nobody stopped them.
func bootOrphan(name string) stacks.Stack {
return stacks.Stack{
Name: name,
Deployed: true,
State: stacks.StateExited,
Containers: []stacks.ContainerInfo{
{Name: name + "-app", State: stacks.StateExited, Status: "Exited (0) 3 minutes ago"},
},
}
}
// userStopped is the UI-Stop shape: `docker compose down` REMOVED the containers.
func userStopped(name string) stacks.Stack {
return stacks.Stack{Name: name, Deployed: true, State: stacks.StateStopped, Containers: nil}
}
// --- Scenario C -------------------------------------------------------------------------------
func TestReconcile_BootOrphanGetsExactlyOneRecovery(t *testing.T) {
f := &fakeStacks{
list: []stacks.Stack{bootOrphan("immich"), bootOrphan("calibre-web")},
onStart: comesUp,
}
r, slept := newTestReconciler(f)
res := r.Run(context.Background())
for _, name := range []string{"immich", "calibre-web"} {
if f.starts[name] != 1 {
t.Fatalf("StartStack(%q) called %d times, want exactly 1", name, f.starts[name])
}
}
if len(res.Recovered) != 2 || len(res.StillDown) != 0 {
t.Fatalf("recovered=%v stillDown=%v, want both apps recovered", res.Recovered, res.StillDown)
}
if res.Attempts != 1 {
t.Fatalf("attempts = %d, want 1 — a success must not retry", res.Attempts)
}
if *slept != 0 {
t.Fatalf("slept %d times after a first-attempt success, want 0", *slept)
}
}
// --- Scenario D (the WRONG case: this must assert the negative) ---------------------------------
func TestReconcile_UserStoppedAppIsNeverStarted(t *testing.T) {
f := &fakeStacks{
list: []stacks.Stack{userStopped("jellyfin"), bootOrphan("immich")},
onStart: comesUp,
}
r, _ := newTestReconciler(f)
res := r.Run(context.Background())
if n := f.starts["jellyfin"]; n != 0 {
t.Fatalf("StartStack(\"jellyfin\") called %d times, want 0 — a deliberate Stop must survive a reboot", n)
}
if f.starts["immich"] != 1 {
t.Fatalf("the real boot orphan was not started: %v", f.starts)
}
for _, c := range res.Candidates {
if c == "jellyfin" {
t.Fatalf("a zero-container stack must never be a reconciliation candidate: %v", res.Candidates)
}
}
}
// --- Bounded, never a loop ----------------------------------------------------------------------
func TestReconcile_StopsAfterTwoAttemptsAndHandsOverToTheAlarm(t *testing.T) {
f := &fakeStacks{
list: []stacks.Stack{bootOrphan("immich")},
failWith: map[string]error{"immich": errors.New("compose up: exit status 1")},
}
r, slept := newTestReconciler(f)
res := r.Run(context.Background())
if f.starts["immich"] != DefaultAttempts {
t.Fatalf("StartStack called %d times, want exactly %d (bounded, never a loop)", f.starts["immich"], DefaultAttempts)
}
if *slept != DefaultAttempts-1 {
t.Fatalf("slept %d times, want %d (one wait BETWEEN attempts, never after the last)", *slept, DefaultAttempts-1)
}
if len(res.StillDown) != 1 || res.StillDown[0] != "immich" {
t.Fatalf("stillDown = %v, want [immich] — the alarm must inherit the failure", res.StillDown)
}
}
// `compose up -d` exits 0 on a crash-loop, so a nil error is not proof the app is up. Only a
// re-read of docker ps can retire a candidate.
func TestReconcile_NilErrorIsNotProofTheAppCameUp(t *testing.T) {
f := &fakeStacks{list: []stacks.Stack{bootOrphan("immich")}} // onStart nil → stays Exited
r, _ := newTestReconciler(f)
res := r.Run(context.Background())
if f.starts["immich"] != DefaultAttempts {
t.Fatalf("StartStack called %d times, want %d — a still-down app must be retried", f.starts["immich"], DefaultAttempts)
}
if len(res.StillDown) != 1 {
t.Fatalf("stillDown = %v, want the app still listed despite StartStack returning nil", res.StillDown)
}
if f.refreshN < 1 {
t.Fatalf("RefreshStatus was never called — the outcome was taken on trust")
}
}
// A second-attempt success must still end clean (and must not alert).
func TestReconcile_SecondAttemptSucceeds(t *testing.T) {
f := &fakeStacks{list: []stacks.Stack{bootOrphan("immich")}}
f.onStart = func(fs *fakeStacks, name string) {
if fs.starts[name] >= 2 {
comesUp(fs, name)
}
}
r, slept := newTestReconciler(f)
res := r.Run(context.Background())
if f.starts["immich"] != 2 {
t.Fatalf("StartStack called %d times, want 2", f.starts["immich"])
}
if *slept != 1 {
t.Fatalf("slept %d times, want 1", *slept)
}
if len(res.StillDown) != 0 || len(res.Recovered) != 1 {
t.Fatalf("recovered=%v stillDown=%v, want a clean recovery on attempt 2", res.Recovered, res.StillDown)
}
}
// --- The gate, term by term ---------------------------------------------------------------------
func TestIsBootOrphan_Gate(t *testing.T) {
base := bootOrphan("app")
cases := []struct {
name string
mut func(s *stacks.Stack)
want bool
}{
{"boot orphan", func(*stacks.Stack) {}, true},
{"degraded counts (R-51 half-started boot)", func(s *stacks.Stack) { s.State = stacks.StateDegraded }, true},
{"not deployed", func(s *stacks.Stack) { s.Deployed = false }, false},
{"protected infra", func(s *stacks.Stack) { s.Protected = true }, false},
{"mid-deploy", func(s *stacks.Stack) { s.Deploying = true }, false},
{"no containers (UI Stop)", func(s *stacks.Stack) { s.Containers = nil }, false},
{"running", func(s *stacks.Stack) { s.State = stacks.StateRunning }, false},
{"unhealthy is not down", func(s *stacks.Stack) { s.State = stacks.StateUnhealthy }, false},
{"restarting recovers itself", func(s *stacks.Stack) { s.State = stacks.StateRestarting }, false},
{"paused is deliberate", func(s *stacks.Stack) { s.State = stacks.StatePaused }, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
s := base
s.Containers = append([]stacks.ContainerInfo(nil), base.Containers...)
tc.mut(&s)
if got := isBootOrphan(s); got != tc.want {
t.Fatalf("isBootOrphan = %v, want %v", got, tc.want)
}
})
}
}
// A cancelled context (controller shutting down mid-boot) must abandon the sweep, not soldier on.
func TestReconcile_ContextCancellationStops(t *testing.T) {
f := &fakeStacks{list: []stacks.Stack{bootOrphan("immich")}}
r, _ := newTestReconciler(f)
ctx, cancel := context.WithCancel(context.Background())
cancel()
res := r.Run(ctx)
if f.starts["immich"] != 0 {
t.Fatalf("StartStack called %d times on a cancelled context, want 0", f.starts["immich"])
}
if len(res.Candidates) != 1 {
t.Fatalf("candidates = %v, want the app still identified", res.Candidates)
}
}
// The quiet path must be observable — "nothing to do" and "never ran" must not look identical.
func TestReconcile_QuietPathLogsAndStartsNothing(t *testing.T) {
f := &fakeStacks{list: []stacks.Stack{{Name: "immich", Deployed: true, State: stacks.StateRunning,
Containers: []stacks.ContainerInfo{{Name: "immich-app", State: stacks.StateRunning}}}}}
var buf logCapture
r := New(f, log.New(&buf, "", 0))
r.sleep = func(context.Context, time.Duration) {}
res := r.Run(context.Background())
if len(f.starts) != 0 {
t.Fatalf("a healthy box must produce zero starts, got %v", f.starts)
}
if len(res.Candidates) != 0 {
t.Fatalf("candidates = %v, want none", res.Candidates)
}
if !buf.contains("no boot-orphaned apps") {
t.Fatalf("the quiet path logged nothing identifiable: %q", buf.String())
}
}
type logCapture struct{ b []byte }
func (l *logCapture) Write(p []byte) (int, error) { l.b = append(l.b, p...); return len(p), nil }
func (l *logCapture) String() string { return string(l.b) }
func (l *logCapture) contains(s string) bool {
return len(l.b) > 0 && bytesContains(l.b, []byte(s))
}
func bytesContains(hay, needle []byte) bool {
for i := 0; i+len(needle) <= len(hay); i++ {
if string(hay[i:i+len(needle)]) == string(needle) {
return true
}
}
return false
}
var _ io.Writer = (*logCapture)(nil)