felhom-controller/controller/internal/quiesce/failed_restart_test.go

package quiesce

import (
	"context"
	"errors"
	"sync"
	"testing"
)

// F-CRIT-1 cause 1 (Campaign 8): `restartAll` used to return nothing, so a failed restart was
// logged and dropped and no caller could ever learn a customer's app had not come back.
//
// Scenario D — the outcome must reach the caller.

// stacksWithStartFailures is a Stacks whose StartStack fails for named stacks.
type stacksWithStartFailures struct {
	mu       sync.Mutex
	running  []string
	startErr map[string]error
	started  []string
	stopped  []string
}

func (f *stacksWithStartFailures) RunningAppStacks() []string {
	f.mu.Lock()
	defer f.mu.Unlock()
	return append([]string(nil), f.running...)
}
func (f *stacksWithStartFailures) StopStack(name string) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.stopped = append(f.stopped, name)
	return nil
}
func (f *stacksWithStartFailures) StartStack(name string) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.started = append(f.started, name)
	if f.startErr != nil {
		return f.startErr[name]
	}
	return nil
}

// Scenario D — restartAll RETURNS the stacks that would not start.
//
// RED-PROOF: revert restartAll to `func (l *Loop) restartAll(stacks []string)` with the error only
// logged → this file does not compile ("l.restartAll(...) used as value"), which is the loudest
// possible form of "the caller cannot learn".
func TestRestartAll_ReturnsTheStacksThatFailed(t *testing.T) {
	boom := errors.New("compose up: exit code 1")
	st := &stacksWithStartFailures{startErr: map[string]error{"immich": boom}}
	l := testLoop(t, &fakeBackend{}, st)

	failed := l.restartAll([]string{"bookstack", "immich", "docmost"})

	if len(failed) != 1 || failed[0] != "immich" {
		t.Fatalf("restartAll returned %v, want exactly [immich]", failed)
	}
	// every stack must still have been ATTEMPTED — one failure must not abort the rest
	if len(st.started) != 3 {
		t.Errorf("attempted %d restarts, want 3 — a failure must not stop the loop restarting the others", len(st.started))
	}
}

// Scenario A (cause 1) — a failed restart is REMEMBERED on the Loop, so the classifier can see it.
//
// RED-PROOF: drop the `l.noteRestartOutcome(...)` call at the unquiesce site → FailedRestarts() is
// empty and this fails with "want immich to be recorded as a failed restart".
func TestFailedRestarts_RecordedAfterACycle(t *testing.T) {
	boom := errors.New("compose up: exit code 1")
	st := &stacksWithStartFailures{
		running:  []string{"bookstack", "immich"},
		startErr: map[string]error{"immich": boom},
	}
	be := &fakeBackend{due: true, phases: []string{"done"}}
	l := testLoop(t, be, st)

	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("runOnce: %v", err)
	}

	failed := l.FailedRestarts()
	if !failed["immich"] {
		t.Errorf("want immich to be recorded as a failed restart, got %v", failed)
	}
	if failed["bookstack"] {
		t.Errorf("bookstack restarted fine and must NOT be recorded as failed, got %v", failed)
	}
}

// The flag must CLEAR when a later cycle restarts the stack successfully — otherwise a fixed app
// carries its old failure forever and would alarm the next time it is legitimately stopped.
func TestFailedRestarts_ClearedOnALaterSuccess(t *testing.T) {
	boom := errors.New("compose up: exit code 1")
	st := &stacksWithStartFailures{
		running:  []string{"immich"},
		startErr: map[string]error{"immich": boom},
	}
	be := &fakeBackend{due: true, phases: []string{"done"}}
	l := testLoop(t, be, st)

	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("cycle 1: %v", err)
	}
	if !l.FailedRestarts()["immich"] {
		t.Fatal("precondition: immich should be flagged after the failing cycle")
	}

	// the operator fixes the app; the next cycle restarts it cleanly
	st.mu.Lock()
	st.startErr = nil
	st.mu.Unlock()
	if err := l.runOnce(context.Background()); err != nil {
		t.Fatalf("cycle 2: %v", err)
	}

	if l.FailedRestarts()["immich"] {
		t.Error("the flag survived a SUCCESSFUL restart — a fixed app would carry its old failure forever")
	}
}

// FailedRestarts must be nil-safe on a nil *Loop, for the same reason SuppressedStacks is: an
// unprovisioned guest has no loop and the caller must not need a branch.
func TestFailedRestarts_NilSafe(t *testing.T) {
	var l *Loop
	if got := l.FailedRestarts(); got != nil {
		t.Errorf("nil Loop returned %v, want nil", got)
	}
}