felhom-controller/controller/cmd/controller/crashloop_classify_test.go

package main

import (
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)

// C9-F2 — a SUSTAINED `restarting` is a crash loop and must alarm; a BRIEF one must not.
//
// The defect: `IsDownState` excludes `restarting` as "self-recovering", but for the catalog's
// standard `restart: unless-stopped` Docker retries forever, so a crash loop sat in `restarting`
// indefinitely and was counted as working. Campaign 9 watched docmost loop for nine minutes
// (restartcount 18) while the F-OBS heartbeat printed "4 deployed app(s) evaluated, 0 currently down".
//
// The whole design tension is that B must keep passing while A does: an alarm that fires on every
// deploy is one the operator learns to ignore.

// restartingSince builds a deployed stack that has been restarting since `since`.
func restartingSince(name string, since time.Time) stacks.Stack {
	s := stack(name, stacks.StateRestarting, true, false)
	s.RestartingSince = since
	return s
}

// SCENARIO A — a crash loop alarms. A stack restarting for longer than the threshold enters BOTH the
// banner dead-list and the notifier Down-set, so app_start_failed can fire.
//
// RED-PROOF (observed): drop `|| crashLooping` from the `down` expression in classifyRunStates →
//
//	crashloop_classify_test.go:52: docmost is NOT in the Down-set — a crash loop is silent (this is C9-F2)
//	crashloop_classify_test.go:55: docmost is NOT in the banner dead-list
func TestClassifyRunStates_SustainedRestartingAlarms(t *testing.T) {
	now := time.Now()
	sts := []stacks.Stack{
		stack("paperless-ngx", stacks.StateRunning, true, false),
		restartingSince("docmost", now.Add(-9*time.Minute)), // the Campaign 9 observation, exactly
	}

	dead, states := classifyRunStates(sts, nil, nil, now)

	if !downByName(states)["docmost"] {
		t.Errorf("docmost is NOT in the Down-set — a crash loop is silent (this is C9-F2)")
	}
	if !deadNames(dead)["docmost"] {
		t.Errorf("docmost is NOT in the banner dead-list")
	}
	if downByName(states)["paperless-ngx"] {
		t.Errorf("a healthy app was dragged down with it")
	}
}

// SCENARIO B — a normal deploy or update does NOT alarm. `docker compose up -d` passes through
// restarting; alarming there would page the operator on every routine operation, fleet-wide.
//
// This is the test that must fail against the naive fix. RED-PROOF (observed): add StateRestarting
// to IsDownState instead of using the threshold →
//
//	crashloop_classify_test.go:78: a BRIEFLY restarting app alarms — every deploy and update would page the operator
func TestClassifyRunStates_BriefRestartingIsSilent(t *testing.T) {
	now := time.Now()
	sts := []stacks.Stack{
		restartingSince("mealie", now.Add(-30*time.Second)), // mid-deploy
		restartingSince("ghost", now.Add(-2*time.Minute)),   // slow image pull, still normal
	}

	dead, states := classifyRunStates(sts, nil, nil, now)

	for _, name := range []string{"mealie", "ghost"} {
		if downByName(states)[name] {
			t.Errorf("a BRIEFLY restarting app alarms (%s) — every deploy and update would page the operator", name)
		}
	}
	if len(dead) != 0 {
		t.Errorf("banner dead-list should be empty during normal restarts, got %v", deadNames(dead))
	}
}

// The boundary itself, asserted from both sides so the threshold cannot drift silently.
func TestCrashLooping_ThresholdBoundary(t *testing.T) {
	now := time.Now()
	for _, tc := range []struct {
		name string
		age  time.Duration
		want bool
	}{
		{"just under the threshold", 4*time.Minute + 59*time.Second, false},
		{"exactly at the threshold", 5 * time.Minute, true},
		{"well past it", 30 * time.Minute, true},
	} {
		s := restartingSince("app", now.Add(-tc.age))
		if got := s.CrashLooping(now); got != tc.want {
			t.Errorf("%s: CrashLooping(age=%s) = %v, want %v", tc.name, tc.age, got, tc.want)
		}
	}

	// A stack that is not restarting is never a crash loop, however old the stamp.
	s := stack("app", stacks.StateRunning, true, false)
	s.RestartingSince = now.Add(-time.Hour)
	if s.CrashLooping(now) {
		t.Error("a RUNNING stack reported as crash-looping — the state test is missing")
	}

	// A zero stamp is "not yet observed restarting", never a crash loop — this is what makes the
	// first scan after a controller restart silent instead of alarming on everything at once.
	z := stack("app", stacks.StateRestarting, true, false)
	if z.CrashLooping(now) {
		t.Error("a zero RestartingSince reported as crash-looping — a controller restart would alarm fleet-wide")
	}
}

// SCENARIO C — R-97b's quiesce suppression still wins inside its window. A stack the backup stopped
// and is restarting must stay silent while suppressed, even if its restarting run is old enough to
// qualify. The window EXPIRES, so a genuinely dead app still alarms afterwards — proven by the
// second half of this test.
//
// RED-PROOF (observed): drop `&& !quiesced[st.Name]` from the `down` expression →
//
//	crashloop_classify_test.go:129: a quiesced stack alarms — every backup would page the customer
func TestClassifyRunStates_QuiesceSuppressionBeatsCrashLoop(t *testing.T) {
	now := time.Now()
	sts := []stacks.Stack{restartingSince("docmost", now.Add(-9*time.Minute))}

	// Inside the R-97b window.
	_, states := classifyRunStates(sts, map[string]bool{"docmost": true}, nil, now)
	if downByName(states)["docmost"] {
		t.Errorf("a quiesced stack alarms — every backup would page the customer")
	}

	// Window expired (the stack is no longer reported as suppressed): the same stack must now alarm.
	dead, states := classifyRunStates(sts, nil, nil, now)
	if !downByName(states)["docmost"] {
		t.Errorf("suppression outlived its window — a genuinely dead app stayed silent (R-97b's own warning)")
	}
	if !deadNames(dead)["docmost"] {
		t.Errorf("suppression outlived its window for the banner too")
	}
}