package stacks import ( "fmt" "io" "log" "strings" "sync" "testing" "gitea.dooplex.hu/admin/felhom-controller/internal/config" ) // R-51 (v0.156.0). The live defect these tests pin: on 2026-07-20 `immich-server` sat Exited for // 18 hours behind three running helpers, the stack aggregated to StateRunning ("partial"), and // because StateRunning is not a down state NOTHING fired — no dashboard banner, no // `app_start_failed` hub event — while single-container Calibre-Web, down for the same reason, // alerted in 90 s. Evidence: felhom.eu/documentation/audits/AUDIT-vacation-remote-ops-2026-07-20.md // finding F4. // // RED-PROOF (recorded in REPORT.md): with the mix branch reverted to its pre-v0.156.0 body // // if running > 0 { return StateRunning } // // TestAggregateState_DeadSupervisedMemberIsDegraded fails with // "aggregateState = running, want degraded", which is exactly the shape the audit observed. // immichLike is the F4 fixture: the primary Exited, the helpers up. func immichLike() []ContainerInfo { return []ContainerInfo{ {Name: "immich-server", State: StateExited, Status: "Exited (137) 18 hours ago"}, {Name: "immich-machine-learning", State: StateRunning, Status: "Up 18 hours"}, {Name: "immich-redis", State: StateRunning, Status: "Up 18 hours (healthy)"}, {Name: "immich-postgres", State: StateRunning, Status: "Up 18 hours (healthy)"}, } } // policyMap builds a lookup over a name→policy table; an unlisted name reads as UNKNOWN (""). func policyMap(t *testing.T, m map[string]string) restartPolicyLookup { t.Helper() return func(name string) string { return m[name] } } func TestAggregateState_DeadSupervisedMemberIsDegraded(t *testing.T) { got := aggregateState(immichLike(), policyMap(t, map[string]string{ "immich-server": "unless-stopped", "immich-machine-learning": "unless-stopped", "immich-redis": "unless-stopped", "immich-postgres": "unless-stopped", })) if got != StateDegraded { t.Fatalf("aggregateState = %q, want %q (a dead supervised primary must not read as running)", got, StateDegraded) } if !IsDownState(got) { t.Fatalf("IsDownState(%q) = false — the whole point of R-51 is that this state alerts", got) } } // Scenario B: a one-shot init/migrate container that has legitimately finished must NOT alarm. func TestAggregateState_OneShotExitedMemberIsBenign(t *testing.T) { for _, policy := range []string{"no", "on-failure", ""} { name := policy if name == "" { name = "(absent)" } t.Run(name, func(t *testing.T) { containers := []ContainerInfo{ {Name: "app-migrate", State: StateExited, Status: "Exited (0) 2 minutes ago"}, {Name: "app-web", State: StateRunning, Status: "Up 2 minutes"}, } got := aggregateState(containers, policyMap(t, map[string]string{ "app-migrate": policy, "app-web": "unless-stopped", })) want := StateRunning if policy == "" { // UNKNOWN is deliberately fail-CLOSED — see supervisedPolicy. An absent policy in // the compose file resolves to Docker's "no" at inspect time, so the "" case here // is the INSPECT-FAILED case, not the no-restart-policy case. want = StateDegraded } if got != want { t.Fatalf("policy %q: aggregateState = %q, want %q", policy, got, want) } }) } } // The unchanged branches: R-51 must not move any state the pre-existing aggregation produced. func TestAggregateState_UnchangedBranches(t *testing.T) { all := policyMap(t, map[string]string{"a": "unless-stopped", "b": "unless-stopped"}) cases := []struct { name string containers []ContainerInfo want ContainerState }{ {"no containers", nil, StateNotDeployed}, {"all running", []ContainerInfo{{Name: "a", State: StateRunning}, {Name: "b", State: StateRunning}}, StateRunning}, {"all stopped", []ContainerInfo{{Name: "a", State: StateExited}, {Name: "b", State: StateStopped}}, StateStopped}, {"any unhealthy wins", []ContainerInfo{{Name: "a", State: StateUnhealthy}, {Name: "b", State: StateExited}}, StateUnhealthy}, {"any starting wins over exited", []ContainerInfo{{Name: "a", State: StateStarting}, {Name: "b", State: StateExited}}, StateStarting}, {"any restarting wins over exited", []ContainerInfo{{Name: "a", State: StateRestarting}, {Name: "b", State: StateExited}}, StateRestarting}, {"single container exited", []ContainerInfo{{Name: "a", State: StateExited}}, StateStopped}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { if got := aggregateState(tc.containers, all); got != tc.want { t.Fatalf("aggregateState = %q, want %q", got, tc.want) } }) } } // The unhealthy/restarting/paused/unknown exclusions are the fix-3 contract (downstate_test.go owns // them). This asserts the ONE addition, so a future reader can see R-51 widened the set by exactly // one state and by nothing else. func TestIsDownState_DegradedIsTheOnlyAddition(t *testing.T) { if !IsDownState(StateDegraded) { t.Fatalf("IsDownState(degraded) = false, want true") } for _, s := range []ContainerState{StateUnhealthy, StateRestarting, StatePaused, StateUnknown} { if IsDownState(s) { t.Fatalf("IsDownState(%q) = true — R-51 must not touch the fix-3 exclusions", s) } } } // --- production-path wiring test (§9 rule 6) --------------------------------------------------- // // Proves the chain the box actually runs: RefreshStatus → docker ps → aggregateState → docker // inspect. An aggregateState-only test proves the function, not the caller. type scriptedDocker struct { mu sync.Mutex ps string policies map[string]string inspects []string // every container name inspected, in order } func (s *scriptedDocker) exec(name string, args ...string) (string, error) { s.mu.Lock() defer s.mu.Unlock() if name != "docker" { return "", fmt.Errorf("unexpected command %q", name) } switch { case len(args) > 0 && args[0] == "ps": return s.ps, nil case len(args) > 0 && args[0] == "inspect": target := args[len(args)-1] s.inspects = append(s.inspects, target) p, ok := s.policies[target] if !ok { return "", fmt.Errorf("no such container: %s", target) } return p + "\n", nil } return "", fmt.Errorf("unexpected docker args %v", args) } func (s *scriptedDocker) inspectCount() int { s.mu.Lock() defer s.mu.Unlock() return len(s.inspects) } func psLine(name, state, status, project string) string { return strings.Join([]string{name, "img:1", state, status, project}, "\t") } func TestRefreshStatus_WiresDegradedThroughTheRealPath(t *testing.T) { dock := &scriptedDocker{ ps: strings.Join([]string{ psLine("immich-server", "exited", "Exited (137) 18 hours ago", "immich"), psLine("immich-redis", "running", "Up 18 hours (healthy)", "immich"), psLine("calibre-web", "running", "Up 18 hours", "calibre-web"), }, "\n"), policies: map[string]string{"immich-server": "unless-stopped"}, } m := &Manager{ cfg: &config.Config{}, logger: log.New(io.Discard, "", 0), execFn: dock.exec, stacks: map[string]*Stack{ "immich": {Name: "immich", Deployed: true}, "calibre-web": {Name: "calibre-web", Deployed: true}, }, } if err := m.RefreshStatus(); err != nil { t.Fatalf("RefreshStatus: %v", err) } if got := m.stacks["immich"].State; got != StateDegraded { t.Fatalf("immich state = %q, want %q (the F4 shape must reach the stack map)", got, StateDegraded) } if got := m.stacks["calibre-web"].State; got != StateRunning { t.Fatalf("calibre-web state = %q, want %q — a healthy app must be untouched", got, StateRunning) } // Only the DOWN member of the MIXED stack is inspected: never the running members, never the // healthy stack. An inspect per container per 10 s refresh would be a real docker load. if n := dock.inspectCount(); n != 1 { t.Fatalf("docker inspect called %d times, want exactly 1 (%v)", n, dock.inspects) } // Second refresh: the answer comes from the cache, so the inspect count must NOT move. if err := m.RefreshStatus(); err != nil { t.Fatalf("RefreshStatus (2nd): %v", err) } if n := dock.inspectCount(); n != 1 { t.Fatalf("docker inspect called %d times after a second refresh, want 1 — the cache is not being used", n) } if got := m.stacks["immich"].State; got != StateDegraded { t.Fatalf("immich state after 2nd refresh = %q, want %q", got, StateDegraded) } } // A container that vanishes must not leave its policy behind — an unbounded cache in a process that // runs for months is a slow leak, and a stale entry would answer for a recreated container. func TestRestartPolicyCache_PrunesVanishedContainers(t *testing.T) { dock := &scriptedDocker{ ps: strings.Join([]string{ psLine("app-init", "exited", "Exited (0) 1 minute ago", "app"), psLine("app-web", "running", "Up 1 minute", "app"), }, "\n"), policies: map[string]string{"app-init": "no"}, } m := &Manager{ cfg: &config.Config{}, logger: log.New(io.Discard, "", 0), execFn: dock.exec, stacks: map[string]*Stack{"app": {Name: "app", Deployed: true}}, } if err := m.RefreshStatus(); err != nil { t.Fatalf("RefreshStatus: %v", err) } if got := m.stacks["app"].State; got != StateRunning { t.Fatalf("state = %q, want running (a finished one-shot must not alarm)", got) } if len(m.restartPolicyCache) != 1 { t.Fatalf("cache size = %d, want 1", len(m.restartPolicyCache)) } // The one-shot container is reaped; only the web container remains. dock.ps = psLine("app-web", "running", "Up 5 minutes", "app") if err := m.RefreshStatus(); err != nil { t.Fatalf("RefreshStatus (2nd): %v", err) } if len(m.restartPolicyCache) != 0 { t.Fatalf("cache size = %d after the container vanished, want 0: %v", len(m.restartPolicyCache), m.restartPolicyCache) } } // An inspect failure must not silence the alarm — see supervisedPolicy's fail-closed rationale. func TestRefreshStatus_InspectFailureStillDegrades(t *testing.T) { dock := &scriptedDocker{ ps: strings.Join([]string{ psLine("immich-server", "exited", "Exited (137) 1 hour ago", "immich"), psLine("immich-redis", "running", "Up 1 hour", "immich"), }, "\n"), policies: map[string]string{}, // every inspect fails } m := &Manager{ cfg: &config.Config{}, logger: log.New(io.Discard, "", 0), execFn: dock.exec, stacks: map[string]*Stack{"immich": {Name: "immich", Deployed: true}}, } if err := m.RefreshStatus(); err != nil { t.Fatalf("RefreshStatus: %v", err) } if got := m.stacks["immich"].State; got != StateDegraded { t.Fatalf("state = %q, want %q — an unreadable policy must not lose the alarm", got, StateDegraded) } // A failed inspect is deliberately NOT cached, so the next cycle retries. if len(m.restartPolicyCache) != 0 { t.Fatalf("failed inspect was cached: %v", m.restartPolicyCache) } }