felhom-controller/controller/internal/stacks/degraded_test.go

package stacks

import (
	"fmt"
	"io"
	"log"
	"strings"
	"sync"
	"testing"

	"gitea.dooplex.hu/admin/felhom-controller/internal/config"
)

// R-51 (v0.156.0). The live defect these tests pin: on 2026-07-20 `immich-server` sat Exited for
// 18 hours behind three running helpers, the stack aggregated to StateRunning ("partial"), and
// because StateRunning is not a down state NOTHING fired — no dashboard banner, no
// `app_start_failed` hub event — while single-container Calibre-Web, down for the same reason,
// alerted in 90 s. Evidence: felhom.eu/documentation/audits/AUDIT-vacation-remote-ops-2026-07-20.md
// finding F4.
//
// RED-PROOF (recorded in REPORT.md): with the mix branch reverted to its pre-v0.156.0 body
//
//	if running > 0 { return StateRunning }
//
// TestAggregateState_DeadSupervisedMemberIsDegraded fails with
// "aggregateState = running, want degraded", which is exactly the shape the audit observed.

// immichLike is the F4 fixture: the primary Exited, the helpers up.
func immichLike() []ContainerInfo {
	return []ContainerInfo{
		{Name: "immich-server", State: StateExited, Status: "Exited (137) 18 hours ago"},
		{Name: "immich-machine-learning", State: StateRunning, Status: "Up 18 hours"},
		{Name: "immich-redis", State: StateRunning, Status: "Up 18 hours (healthy)"},
		{Name: "immich-postgres", State: StateRunning, Status: "Up 18 hours (healthy)"},
	}
}

// policyMap builds a lookup over a name→policy table; an unlisted name reads as UNKNOWN ("").
func policyMap(t *testing.T, m map[string]string) restartPolicyLookup {
	t.Helper()
	return func(name string) string { return m[name] }
}

func TestAggregateState_DeadSupervisedMemberIsDegraded(t *testing.T) {
	got := aggregateState(immichLike(), policyMap(t, map[string]string{
		"immich-server":           "unless-stopped",
		"immich-machine-learning": "unless-stopped",
		"immich-redis":            "unless-stopped",
		"immich-postgres":         "unless-stopped",
	}))
	if got != StateDegraded {
		t.Fatalf("aggregateState = %q, want %q (a dead supervised primary must not read as running)", got, StateDegraded)
	}
	if !IsDownState(got) {
		t.Fatalf("IsDownState(%q) = false — the whole point of R-51 is that this state alerts", got)
	}
}

// Scenario B: a one-shot init/migrate container that has legitimately finished must NOT alarm.
func TestAggregateState_OneShotExitedMemberIsBenign(t *testing.T) {
	for _, policy := range []string{"no", "on-failure", ""} {
		name := policy
		if name == "" {
			name = "(absent)"
		}
		t.Run(name, func(t *testing.T) {
			containers := []ContainerInfo{
				{Name: "app-migrate", State: StateExited, Status: "Exited (0) 2 minutes ago"},
				{Name: "app-web", State: StateRunning, Status: "Up 2 minutes"},
			}
			got := aggregateState(containers, policyMap(t, map[string]string{
				"app-migrate": policy,
				"app-web":     "unless-stopped",
			}))
			want := StateRunning
			if policy == "" {
				// UNKNOWN is deliberately fail-CLOSED — see supervisedPolicy. An absent policy in
				// the compose file resolves to Docker's "no" at inspect time, so the "" case here
				// is the INSPECT-FAILED case, not the no-restart-policy case.
				want = StateDegraded
			}
			if got != want {
				t.Fatalf("policy %q: aggregateState = %q, want %q", policy, got, want)
			}
		})
	}
}

// The unchanged branches: R-51 must not move any state the pre-existing aggregation produced.
func TestAggregateState_UnchangedBranches(t *testing.T) {
	all := policyMap(t, map[string]string{"a": "unless-stopped", "b": "unless-stopped"})
	cases := []struct {
		name       string
		containers []ContainerInfo
		want       ContainerState
	}{
		{"no containers", nil, StateNotDeployed},
		{"all running", []ContainerInfo{{Name: "a", State: StateRunning}, {Name: "b", State: StateRunning}}, StateRunning},
		{"all stopped", []ContainerInfo{{Name: "a", State: StateExited}, {Name: "b", State: StateStopped}}, StateStopped},
		{"any unhealthy wins", []ContainerInfo{{Name: "a", State: StateUnhealthy}, {Name: "b", State: StateExited}}, StateUnhealthy},
		{"any starting wins over exited", []ContainerInfo{{Name: "a", State: StateStarting}, {Name: "b", State: StateExited}}, StateStarting},
		{"any restarting wins over exited", []ContainerInfo{{Name: "a", State: StateRestarting}, {Name: "b", State: StateExited}}, StateRestarting},
		{"single container exited", []ContainerInfo{{Name: "a", State: StateExited}}, StateStopped},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			if got := aggregateState(tc.containers, all); got != tc.want {
				t.Fatalf("aggregateState = %q, want %q", got, tc.want)
			}
		})
	}
}

// The unhealthy/restarting/paused/unknown exclusions are the fix-3 contract (downstate_test.go owns
// them). This asserts the ONE addition, so a future reader can see R-51 widened the set by exactly
// one state and by nothing else.
func TestIsDownState_DegradedIsTheOnlyAddition(t *testing.T) {
	if !IsDownState(StateDegraded) {
		t.Fatalf("IsDownState(degraded) = false, want true")
	}
	for _, s := range []ContainerState{StateUnhealthy, StateRestarting, StatePaused, StateUnknown} {
		if IsDownState(s) {
			t.Fatalf("IsDownState(%q) = true — R-51 must not touch the fix-3 exclusions", s)
		}
	}
}

// --- production-path wiring test (§9 rule 6) ---------------------------------------------------
//
// Proves the chain the box actually runs: RefreshStatus → docker ps → aggregateState → docker
// inspect. An aggregateState-only test proves the function, not the caller.

type scriptedDocker struct {
	mu       sync.Mutex
	ps       string
	policies map[string]string
	inspects []string // every container name inspected, in order
}

func (s *scriptedDocker) exec(name string, args ...string) (string, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if name != "docker" {
		return "", fmt.Errorf("unexpected command %q", name)
	}
	switch {
	case len(args) > 0 && args[0] == "ps":
		return s.ps, nil
	case len(args) > 0 && args[0] == "inspect":
		target := args[len(args)-1]
		s.inspects = append(s.inspects, target)
		p, ok := s.policies[target]
		if !ok {
			return "", fmt.Errorf("no such container: %s", target)
		}
		return p + "\n", nil
	}
	return "", fmt.Errorf("unexpected docker args %v", args)
}

func (s *scriptedDocker) inspectCount() int {
	s.mu.Lock()
	defer s.mu.Unlock()
	return len(s.inspects)
}

func psLine(name, state, status, project string) string {
	return strings.Join([]string{name, "img:1", state, status, project}, "\t")
}

func TestRefreshStatus_WiresDegradedThroughTheRealPath(t *testing.T) {
	dock := &scriptedDocker{
		ps: strings.Join([]string{
			psLine("immich-server", "exited", "Exited (137) 18 hours ago", "immich"),
			psLine("immich-redis", "running", "Up 18 hours (healthy)", "immich"),
			psLine("calibre-web", "running", "Up 18 hours", "calibre-web"),
		}, "\n"),
		policies: map[string]string{"immich-server": "unless-stopped"},
	}
	m := &Manager{
		cfg:    &config.Config{},
		logger: log.New(io.Discard, "", 0),
		execFn: dock.exec,
		stacks: map[string]*Stack{
			"immich":      {Name: "immich", Deployed: true},
			"calibre-web": {Name: "calibre-web", Deployed: true},
		},
	}

	if err := m.RefreshStatus(); err != nil {
		t.Fatalf("RefreshStatus: %v", err)
	}
	if got := m.stacks["immich"].State; got != StateDegraded {
		t.Fatalf("immich state = %q, want %q (the F4 shape must reach the stack map)", got, StateDegraded)
	}
	if got := m.stacks["calibre-web"].State; got != StateRunning {
		t.Fatalf("calibre-web state = %q, want %q — a healthy app must be untouched", got, StateRunning)
	}

	// Only the DOWN member of the MIXED stack is inspected: never the running members, never the
	// healthy stack. An inspect per container per 10 s refresh would be a real docker load.
	if n := dock.inspectCount(); n != 1 {
		t.Fatalf("docker inspect called %d times, want exactly 1 (%v)", n, dock.inspects)
	}

	// Second refresh: the answer comes from the cache, so the inspect count must NOT move.
	if err := m.RefreshStatus(); err != nil {
		t.Fatalf("RefreshStatus (2nd): %v", err)
	}
	if n := dock.inspectCount(); n != 1 {
		t.Fatalf("docker inspect called %d times after a second refresh, want 1 — the cache is not being used", n)
	}
	if got := m.stacks["immich"].State; got != StateDegraded {
		t.Fatalf("immich state after 2nd refresh = %q, want %q", got, StateDegraded)
	}
}

// A container that vanishes must not leave its policy behind — an unbounded cache in a process that
// runs for months is a slow leak, and a stale entry would answer for a recreated container.
func TestRestartPolicyCache_PrunesVanishedContainers(t *testing.T) {
	dock := &scriptedDocker{
		ps: strings.Join([]string{
			psLine("app-init", "exited", "Exited (0) 1 minute ago", "app"),
			psLine("app-web", "running", "Up 1 minute", "app"),
		}, "\n"),
		policies: map[string]string{"app-init": "no"},
	}
	m := &Manager{
		cfg:    &config.Config{},
		logger: log.New(io.Discard, "", 0),
		execFn: dock.exec,
		stacks: map[string]*Stack{"app": {Name: "app", Deployed: true}},
	}
	if err := m.RefreshStatus(); err != nil {
		t.Fatalf("RefreshStatus: %v", err)
	}
	if got := m.stacks["app"].State; got != StateRunning {
		t.Fatalf("state = %q, want running (a finished one-shot must not alarm)", got)
	}
	if len(m.restartPolicyCache) != 1 {
		t.Fatalf("cache size = %d, want 1", len(m.restartPolicyCache))
	}

	// The one-shot container is reaped; only the web container remains.
	dock.ps = psLine("app-web", "running", "Up 5 minutes", "app")
	if err := m.RefreshStatus(); err != nil {
		t.Fatalf("RefreshStatus (2nd): %v", err)
	}
	if len(m.restartPolicyCache) != 0 {
		t.Fatalf("cache size = %d after the container vanished, want 0: %v", len(m.restartPolicyCache), m.restartPolicyCache)
	}
}

// An inspect failure must not silence the alarm — see supervisedPolicy's fail-closed rationale.
func TestRefreshStatus_InspectFailureStillDegrades(t *testing.T) {
	dock := &scriptedDocker{
		ps: strings.Join([]string{
			psLine("immich-server", "exited", "Exited (137) 1 hour ago", "immich"),
			psLine("immich-redis", "running", "Up 1 hour", "immich"),
		}, "\n"),
		policies: map[string]string{}, // every inspect fails
	}
	m := &Manager{
		cfg:    &config.Config{},
		logger: log.New(io.Discard, "", 0),
		execFn: dock.exec,
		stacks: map[string]*Stack{"immich": {Name: "immich", Deployed: true}},
	}
	if err := m.RefreshStatus(); err != nil {
		t.Fatalf("RefreshStatus: %v", err)
	}
	if got := m.stacks["immich"].State; got != StateDegraded {
		t.Fatalf("state = %q, want %q — an unreadable policy must not lose the alarm", got, StateDegraded)
	}
	// A failed inspect is deliberately NOT cached, so the next cycle retries.
	if len(m.restartPolicyCache) != 0 {
		t.Fatalf("failed inspect was cached: %v", m.restartPolicyCache)
	}
}