package bootrecon import ( "context" "errors" "io" "log" "testing" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/stacks" ) // fakeStacks counts StartStack calls per app — the assertion that matters in BOTH directions: // scenario C needs exactly-bounded starts, scenario D needs a start count of ZERO. type fakeStacks struct { list []stacks.Stack starts map[string]int failWith map[string]error // onStart mutates the world the way a real successful start would (fresh docker ps). onStart func(f *fakeStacks, name string) refreshN int } func (f *fakeStacks) GetStacks() []stacks.Stack { return f.list } func (f *fakeStacks) RefreshStatus() error { f.refreshN++; return nil } func (f *fakeStacks) StartStack(name string) error { if f.starts == nil { f.starts = map[string]int{} } f.starts[name]++ if err := f.failWith[name]; err != nil { return err } if f.onStart != nil { f.onStart(f, name) } return nil } func (f *fakeStacks) setState(name string, st stacks.ContainerState) { for i := range f.list { if f.list[i].Name == name { f.list[i].State = st } } } // comesUp is the ordinary success behaviour: the app is running after the start. func comesUp(f *fakeStacks, name string) { f.setState(name, stacks.StateRunning) } func newTestReconciler(f *fakeStacks) (*Reconciler, *int) { slept := 0 r := New(f, log.New(io.Discard, "", 0)) r.sleep = func(context.Context, time.Duration) { slept++ } return r, &slept } // bootOrphan is the F5 shape: deployed, containers still present on disk, all Exited — the boot // interrupted them, nobody stopped them. func bootOrphan(name string) stacks.Stack { return stacks.Stack{ Name: name, Deployed: true, State: stacks.StateExited, Containers: []stacks.ContainerInfo{ {Name: name + "-app", State: stacks.StateExited, Status: "Exited (0) 3 minutes ago"}, }, } } // userStopped is the UI-Stop shape: `docker compose down` REMOVED the containers. func userStopped(name string) stacks.Stack { return stacks.Stack{Name: name, Deployed: true, State: stacks.StateStopped, Containers: nil} } // --- Scenario C ------------------------------------------------------------------------------- func TestReconcile_BootOrphanGetsExactlyOneRecovery(t *testing.T) { f := &fakeStacks{ list: []stacks.Stack{bootOrphan("immich"), bootOrphan("calibre-web")}, onStart: comesUp, } r, slept := newTestReconciler(f) res := r.Run(context.Background()) for _, name := range []string{"immich", "calibre-web"} { if f.starts[name] != 1 { t.Fatalf("StartStack(%q) called %d times, want exactly 1", name, f.starts[name]) } } if len(res.Recovered) != 2 || len(res.StillDown) != 0 { t.Fatalf("recovered=%v stillDown=%v, want both apps recovered", res.Recovered, res.StillDown) } if res.Attempts != 1 { t.Fatalf("attempts = %d, want 1 — a success must not retry", res.Attempts) } if *slept != 0 { t.Fatalf("slept %d times after a first-attempt success, want 0", *slept) } } // --- Scenario D (the WRONG case: this must assert the negative) --------------------------------- func TestReconcile_UserStoppedAppIsNeverStarted(t *testing.T) { f := &fakeStacks{ list: []stacks.Stack{userStopped("jellyfin"), bootOrphan("immich")}, onStart: comesUp, } r, _ := newTestReconciler(f) res := r.Run(context.Background()) if n := f.starts["jellyfin"]; n != 0 { t.Fatalf("StartStack(\"jellyfin\") called %d times, want 0 — a deliberate Stop must survive a reboot", n) } if f.starts["immich"] != 1 { t.Fatalf("the real boot orphan was not started: %v", f.starts) } for _, c := range res.Candidates { if c == "jellyfin" { t.Fatalf("a zero-container stack must never be a reconciliation candidate: %v", res.Candidates) } } } // --- Bounded, never a loop ---------------------------------------------------------------------- func TestReconcile_StopsAfterTwoAttemptsAndHandsOverToTheAlarm(t *testing.T) { f := &fakeStacks{ list: []stacks.Stack{bootOrphan("immich")}, failWith: map[string]error{"immich": errors.New("compose up: exit status 1")}, } r, slept := newTestReconciler(f) res := r.Run(context.Background()) if f.starts["immich"] != DefaultAttempts { t.Fatalf("StartStack called %d times, want exactly %d (bounded, never a loop)", f.starts["immich"], DefaultAttempts) } if *slept != DefaultAttempts-1 { t.Fatalf("slept %d times, want %d (one wait BETWEEN attempts, never after the last)", *slept, DefaultAttempts-1) } if len(res.StillDown) != 1 || res.StillDown[0] != "immich" { t.Fatalf("stillDown = %v, want [immich] — the alarm must inherit the failure", res.StillDown) } } // `compose up -d` exits 0 on a crash-loop, so a nil error is not proof the app is up. Only a // re-read of docker ps can retire a candidate. func TestReconcile_NilErrorIsNotProofTheAppCameUp(t *testing.T) { f := &fakeStacks{list: []stacks.Stack{bootOrphan("immich")}} // onStart nil → stays Exited r, _ := newTestReconciler(f) res := r.Run(context.Background()) if f.starts["immich"] != DefaultAttempts { t.Fatalf("StartStack called %d times, want %d — a still-down app must be retried", f.starts["immich"], DefaultAttempts) } if len(res.StillDown) != 1 { t.Fatalf("stillDown = %v, want the app still listed despite StartStack returning nil", res.StillDown) } if f.refreshN < 1 { t.Fatalf("RefreshStatus was never called — the outcome was taken on trust") } } // A second-attempt success must still end clean (and must not alert). func TestReconcile_SecondAttemptSucceeds(t *testing.T) { f := &fakeStacks{list: []stacks.Stack{bootOrphan("immich")}} f.onStart = func(fs *fakeStacks, name string) { if fs.starts[name] >= 2 { comesUp(fs, name) } } r, slept := newTestReconciler(f) res := r.Run(context.Background()) if f.starts["immich"] != 2 { t.Fatalf("StartStack called %d times, want 2", f.starts["immich"]) } if *slept != 1 { t.Fatalf("slept %d times, want 1", *slept) } if len(res.StillDown) != 0 || len(res.Recovered) != 1 { t.Fatalf("recovered=%v stillDown=%v, want a clean recovery on attempt 2", res.Recovered, res.StillDown) } } // --- The gate, term by term --------------------------------------------------------------------- func TestIsBootOrphan_Gate(t *testing.T) { base := bootOrphan("app") cases := []struct { name string mut func(s *stacks.Stack) want bool }{ {"boot orphan", func(*stacks.Stack) {}, true}, {"degraded counts (R-51 half-started boot)", func(s *stacks.Stack) { s.State = stacks.StateDegraded }, true}, {"not deployed", func(s *stacks.Stack) { s.Deployed = false }, false}, {"protected infra", func(s *stacks.Stack) { s.Protected = true }, false}, {"mid-deploy", func(s *stacks.Stack) { s.Deploying = true }, false}, {"no containers (UI Stop)", func(s *stacks.Stack) { s.Containers = nil }, false}, {"running", func(s *stacks.Stack) { s.State = stacks.StateRunning }, false}, {"unhealthy is not down", func(s *stacks.Stack) { s.State = stacks.StateUnhealthy }, false}, {"restarting recovers itself", func(s *stacks.Stack) { s.State = stacks.StateRestarting }, false}, {"paused is deliberate", func(s *stacks.Stack) { s.State = stacks.StatePaused }, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { s := base s.Containers = append([]stacks.ContainerInfo(nil), base.Containers...) tc.mut(&s) if got := isBootOrphan(s); got != tc.want { t.Fatalf("isBootOrphan = %v, want %v", got, tc.want) } }) } } // A cancelled context (controller shutting down mid-boot) must abandon the sweep, not soldier on. func TestReconcile_ContextCancellationStops(t *testing.T) { f := &fakeStacks{list: []stacks.Stack{bootOrphan("immich")}} r, _ := newTestReconciler(f) ctx, cancel := context.WithCancel(context.Background()) cancel() res := r.Run(ctx) if f.starts["immich"] != 0 { t.Fatalf("StartStack called %d times on a cancelled context, want 0", f.starts["immich"]) } if len(res.Candidates) != 1 { t.Fatalf("candidates = %v, want the app still identified", res.Candidates) } } // The quiet path must be observable — "nothing to do" and "never ran" must not look identical. func TestReconcile_QuietPathLogsAndStartsNothing(t *testing.T) { f := &fakeStacks{list: []stacks.Stack{{Name: "immich", Deployed: true, State: stacks.StateRunning, Containers: []stacks.ContainerInfo{{Name: "immich-app", State: stacks.StateRunning}}}}} var buf logCapture r := New(f, log.New(&buf, "", 0)) r.sleep = func(context.Context, time.Duration) {} res := r.Run(context.Background()) if len(f.starts) != 0 { t.Fatalf("a healthy box must produce zero starts, got %v", f.starts) } if len(res.Candidates) != 0 { t.Fatalf("candidates = %v, want none", res.Candidates) } if !buf.contains("no boot-orphaned apps") { t.Fatalf("the quiet path logged nothing identifiable: %q", buf.String()) } } type logCapture struct{ b []byte } func (l *logCapture) Write(p []byte) (int, error) { l.b = append(l.b, p...); return len(p), nil } func (l *logCapture) String() string { return string(l.b) } func (l *logCapture) contains(s string) bool { return len(l.b) > 0 && bytesContains(l.b, []byte(s)) } func bytesContains(hay, needle []byte) bool { for i := 0; i+len(needle) <= len(hay); i++ { if string(hay[i:i+len(needle)]) == string(needle) { return true } } return false } var _ io.Writer = (*logCapture)(nil)