felhom-controller/controller/internal/stacks/unattended_test.go

package stacks

import (
	"bytes"
	"context"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"
)

// v0.271.0 — the automatic update leg (`09` §3 decisions 11–15, 20; §6.4 part 7). Every test runs the
// REAL leg over the REAL guarded update job with the process boundaries faked (ladderManager: nextcloud
// pinned at A, the catalog at C, a two-step ladder A→B→C, B's own definition in steps/), and reads the
// EFFECT back: the pin in app.yaml, the definitions `up` ran on, app.yaml's records, the summary.

// legNow is inside the night of window 02:30 (the leg starts at W+105m = 04:15).
var legNow = time.Date(2026, 9, 25, 4, 15, 0, 0, time.UTC)

func legOpts(m *Manager, mut func(o *UpdateLegOptions)) {
	o := UpdateLegOptions{
		Enabled:     func() bool { return true },
		WindowStart: func() string { return "02:30" },
		Location:    time.UTC,
		Poll:        2 * time.Millisecond,
		RetryWait:   time.Millisecond,
		Now:         func() time.Time { return legNow },
	}
	if mut != nil {
		mut(&o)
	}
	m.SetUpdateLeg(o)
}

// writeLadder replaces the catalog's .felhom.yml with the given ladder lines.
func writeLadder(t *testing.T, m *Manager, lines ...string) {
	t.Helper()
	catDir := filepath.Dir(m.CatalogTemplatePath("nextcloud", "docker-compose.yml"))
	mustWrite(t, filepath.Join(catDir, ".felhom.yml"), "display_name: Nextcloud\nupdate_ladder:\n"+strings.Join(lines, ""))
}

// ladderLineMarks is ladderLine with a verdict and a marks object.
func ladderLineMarks(from, to, verdict, marks string) string {
	return `  - {"from": {"web": "` + from + `"}, "to": {"web": "` + to + `"}, "digest": {"web": "sha256:` +
		strings.Repeat("a", 64) + `"}, "verdict": "` + verdict + `", "marks": ` + marks + `}` + "\n"
}

func legStepFor(s *UpdateLegSummary, app string) LegStep {
	for _, st := range s.Steps {
		if st.App == app {
			return st
		}
	}
	return LegStep{}
}

// legManager is ladderManager plus what a real box has and the fakes do not produce: an installed-image
// record (CatalogOrder answers Unknown without one, and the leg never presses Unknown). Every `up` records
// the image it brought up, as recordInstalledImages does on a box from the running container.
func legManager(t *testing.T) (*Manager, string, *fakeGuards, *[]string, *bytes.Buffer) {
	t.Helper()
	m, dir, g, ups, logBuf := ladderManager(t, true)
	setInstalled := func(img string) {
		cfg := LoadAppConfig(dir)
		cfg.InstalledImages = map[string]InstalledImage{"web": {Ref: img}}
		if err := SaveAppConfig(dir, cfg, m.encKey, nil); err != nil {
			t.Fatal(err)
		}
	}
	setInstalled(ladderA)
	inner := m.updateComposeFn
	m.updateComposeFn = func(d string, env []string, args ...string) (string, error) {
		out, err := inner(d, env, args...)
		if err == nil && args[0] == "up" {
			if imgs, perr := ParseComposeImages(ComposePathIn(d)); perr == nil {
				setInstalled(strings.SplitN(imgs["web"], "@", 2)[0])
			}
		}
		return out, err
	}
	return m, dir, g, ups, logBuf
}

func mustLeg(t *testing.T, m *Manager) *UpdateLegSummary {
	t.Helper()
	if err := m.ScanStacks(); err != nil {
		t.Fatal(err)
	}
	s := m.RunUpdateLeg(context.Background(), "test")
	if s == nil {
		t.Fatal("the leg did not run")
	}
	return s
}

// TestLeg_OneStepPerAppPerNight — a box two steps behind takes ONE step a night (decision 14 + the
// brief: "one tested step per app"), with the step's own definition, and the summary says done=1.
//
// COMPANION RED-PROOF (REPORT.md): make the leg loop while the app stays behind — this test fails at
// "the leg took 2 steps in one night".
func TestLeg_OneStepPerAppPerNight(t *testing.T) {
	m, dir, _, ups, _ := legManager(t)
	legOpts(m, nil)
	s := mustLeg(t, m)
	if len(*ups) != 1 {
		t.Fatalf("the leg took %d steps in one night (ups=%v), want exactly one", len(*ups), *ups)
	}
	if pinOf(t, dir) != ladderB {
		t.Fatalf("pinned %s, want the first tested step B", pinOf(t, dir))
	}
	if s.Done != 1 || legStepFor(s, "nextcloud").Outcome != LegOutcomeDone {
		t.Fatalf("summary %+v, want done=1 for nextcloud", s)
	}
	cfg := LoadAppConfig(dir)
	if cfg.LastAutoUpdate == nil || cfg.LastAutoUpdate.Outcome != LegOutcomeDone || cfg.LastAutoUpdate.To["web"] != ladderB {
		t.Fatalf("app.yaml must carry the page's record of the automatic step, got %+v", cfg.LastAutoUpdate)
	}
	// the next night climbs the next step
	s2 := mustLeg(t, m)
	if pinOf(t, dir) != ladderC || s2.Done != 1 {
		t.Fatalf("night 2 ended on %s (done=%d), want C", pinOf(t, dir), s2.Done)
	}
	// and the third night finds nothing to do — no press, not even counted as a skip
	s3 := mustLeg(t, m)
	if len(*ups) != 2 || s3.Done+s3.Skipped != 0 {
		t.Fatalf("night 3 at the head pressed or counted something: ups=%v summary=%+v", *ups, s3)
	}
}

// TestR678_StepsLeftFreshAtDone — the page's steps-left count and the pin are current the moment the
// update says `done`, with no scan in between. MEASURED 2026-09-24: ~50 s stale, six re-presses.
//
// COMPANION RED-PROOF (REPORT.md): drop the ScanStacks call in verifyAndConclude — this test fails at
// "steps left read 2 right after the step ended done".
func TestR678_StepsLeftFreshAtDone(t *testing.T) {
	m, _, _, _, _ := ladderManager(t, true)
	if err := m.ScanStacks(); err != nil {
		t.Fatal(err)
	}
	if st, _ := m.GetStack("nextcloud"); st.LadderStepsLeft != 2 {
		t.Fatalf("before: steps left %d, want 2", st.LadderStepsLeft)
	}
	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
		t.Fatal(err)
	}
	st := waitUpdateDone(t, m, "nextcloud")
	if st.UpdatePhase != UpdatePhaseDone {
		t.Fatalf("ended %q", st.UpdatePhase)
	}
	if st.LadderStepsLeft != 1 {
		t.Fatalf("steps left read %d right after the step ended done, want 1", st.LadderStepsLeft)
	}
	if st.AppConfig == nil || st.AppConfig.PinnedImages["web"] != ladderB {
		t.Fatalf("the in-memory pin is stale after done: %+v", st.AppConfig)
	}
}

// TestR680_FailedStepIsNotPressedAgain — B fails and is undone: the box records the failed step, the
// next night's leg skips it (failed_before) and nothing is brought up; a PERSON can still press it; and
// when the catalog's ladder changes, the leg may try again.
//
// COMPANION RED-PROOF (REPORT.md): drop the FailedStep check in legCandidate — this test fails at
// "night 2 pressed the step that was undone on night 1".
func TestR680_FailedStepIsNotPressedAgain(t *testing.T) {
	m, dir, _, ups, _ := legManager(t)
	m.undoCopier = newFakeCopier(map[string]string{undoVol: "OLD"})
	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) { return false, "B unhealthy" }
	m.updateUndoHealthFn = func(context.Context, string, time.Duration, *Metadata) (bool, string) { return true, "A answers" }
	legOpts(m, nil)

	s1 := mustLeg(t, m)
	if s1.Undone != 1 || pinOf(t, dir) != ladderA {
		t.Fatalf("night 1: %+v on %s, want undone back on A", s1, pinOf(t, dir))
	}
	cfg := LoadAppConfig(dir)
	if cfg.FailedStep == nil || cfg.FailedStep.To["web"] != ladderB || cfg.FailedStep.Outcome != "undone" || cfg.FailedStep.Ladder == "" {
		t.Fatalf("the failed step must be recorded on the box, got %+v", cfg.FailedStep)
	}
	upsAfter1 := len(*ups)

	s2 := mustLeg(t, m)
	if len(*ups) != upsAfter1 {
		t.Fatalf("night 2 pressed the step that was undone on night 1 (ups=%v)", *ups)
	}
	if st := legStepFor(s2, "nextcloud"); st.Outcome != LegOutcomeSkipped || st.Reason != LegSkipFailedBefore {
		t.Fatalf("night 2 must skip with %q, got %+v", LegSkipFailedBefore, st)
	}

	// a person still can — the record binds the leg only
	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
		t.Fatalf("a person's press must not be refused by the failed-step record: %v", err)
	}
	waitUpdateDone(t, m, "nextcloud")

	// the catalog re-tests the step (a new tested_at): the ladder's print changes → the leg tries again
	writeLadder(t, m,
		`  - {"from": {"web": "`+ladderA+`"}, "to": {"web": "`+ladderB+`"}, "digest": {"web": "sha256:`+strings.Repeat("a", 64)+`"}, "verdict": "proven", "tested_at": "2026-09-26T01:00:00Z"}`+"\n",
		ladderLine(ladderB, ladderC))
	before := len(*ups)
	mustLeg(t, m)
	if len(*ups) <= before || (*ups)[before] != ladderB {
		t.Fatalf("after the catalog changed the ladder the leg must try the step again (ups=%v)", *ups)
	}
}

// TestLeg_NeedsPersonIsNeverTaken — decision 13's mark: the leg never takes a needs_person step.
//
// COMPANION RED-PROOF (REPORT.md): drop the NeedsPerson check — this test fails at "a needs_person step
// was pressed by the leg".
func TestLeg_NeedsPersonIsNeverTaken(t *testing.T) {
	m, dir, _, ups, _ := legManager(t)
	writeLadder(t, m,
		ladderLineMarks(ladderA, ladderB, "proven", `{"files_may_change": false, "needs_person": "the admin must re-login after this step", "memory_tight": false}`),
		ladderLine(ladderB, ladderC))
	legOpts(m, nil)
	s := mustLeg(t, m)
	if len(*ups) != 0 || pinOf(t, dir) != ladderA {
		t.Fatalf("a needs_person step was pressed by the leg (ups=%v)", *ups)
	}
	if st := legStepFor(s, "nextcloud"); st.Reason != LegSkipNeedsPerson {
		t.Fatalf("skip reason %+v, want %q", st, LegSkipNeedsPerson)
	}
}

// TestLeg_FilesMayChangeNeedsAWholeCopy — decision 13's other mark: taken only when a fresh WHOLE copy
// exists (the backup side's truth table, asked through FreshWholeCopy).
func TestLeg_FilesMayChangeNeedsAWholeCopy(t *testing.T) {
	m, dir, _, ups, _ := legManager(t)
	writeLadder(t, m,
		ladderLineMarks(ladderA, ladderB, "proven", `{"files_may_change": true, "needs_person": null, "memory_tight": false}`),
		ladderLine(ladderB, ladderC))
	whole := false
	legOpts(m, func(o *UpdateLegOptions) {
		o.FreshWholeCopy = func(context.Context, string) (bool, string) { return whole, "fake" }
	})
	s := mustLeg(t, m)
	if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != LegSkipFilesNoCopy {
		t.Fatalf("without a whole copy the step must be skipped: ups=%v summary=%+v", *ups, s)
	}
	whole = true
	s = mustLeg(t, m)
	if len(*ups) != 1 || pinOf(t, dir) != ladderB || s.Done != 1 {
		t.Fatalf("with a fresh whole copy the step must be taken: ups=%v pin=%s", *ups, pinOf(t, dir))
	}
	// unwired check = no whole copy (fail closed)
	m2, _, _, ups2, _ := legManager(t)
	writeLadder(t, m2,
		ladderLineMarks(ladderA, ladderB, "proven", `{"files_may_change": true, "needs_person": null, "memory_tight": false}`),
		ladderLine(ladderB, ladderC))
	legOpts(m2, nil)
	mustLeg(t, m2)
	if len(*ups2) != 0 {
		t.Fatal("with no whole-copy check wired a files_may_change step must never be taken")
	}
}

// TestLeg_OlderThanLadderIsNotTouched — an installed version matching no ladder entry: the leg leaves it
// alone and says so by name (a person's press still jumps, TestLadder_UnknownInstalledJumpsAndSaysSo).
func TestLeg_OlderThanLadderIsNotTouched(t *testing.T) {
	m, dir, _, ups, logBuf := legManager(t)
	old := "services:\n  web:\n    image: nextcloud:30.0.0-apache\nvolumes:\n  db:\n"
	mustWrite(t, ComposePathIn(dir), old)
	mustWrite(t, AppliedComposePath(dir), old)
	mustWrite(t, filepath.Join(dir, "app.yaml"), "deployed: true\nenv: {}\npinned_images:\n    web: nextcloud:30.0.0-apache\ninstalled_images:\n    web:\n        ref: nextcloud:30.0.0-apache\n")
	legOpts(m, nil)
	s := mustLeg(t, m)
	if len(*ups) != 0 {
		t.Fatalf("an app older than the ladder was pressed: %v", *ups)
	}
	if legStepFor(s, "nextcloud").Reason != LegSkipOlderThanLadder || !strings.Contains(logBuf.String(), "nextcloud:30.0.0-apache") {
		t.Fatalf("skip must be older_than_ladder and logged by name; summary %+v", s)
	}
}

// TestLeg_OnlyProvenStepsAreTaken — an `unrecorded` (backfilled, never tested) entry is not a test.
func TestLeg_OnlyProvenStepsAreTaken(t *testing.T) {
	m, _, _, ups, _ := legManager(t)
	writeLadder(t, m,
		ladderLineMarks(ladderA, ladderB, "unrecorded", `{"files_may_change": false, "needs_person": null, "memory_tight": false}`),
		ladderLine(ladderB, ladderC))
	legOpts(m, nil)
	s := mustLeg(t, m)
	if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != LegSkipNotProven {
		t.Fatalf("an unproven step was taken or mis-reasoned: ups=%v %+v", *ups, s)
	}
}

// TestLeg_NoStepAtOrAfterW5h — decision 20: the leg starts nothing at W+5h.
//
// COMPANION RED-PROOF (REPORT.md): drop the deadline arm of legStop — this test fails at "a step was
// started at W+5h".
func TestLeg_NoStepAtOrAfterW5h(t *testing.T) {
	m, _, _, ups, _ := legManager(t)
	legOpts(m, func(o *UpdateLegOptions) {
		o.Now = func() time.Time { return time.Date(2026, 9, 25, 7, 30, 0, 0, time.UTC) } // W 02:30 + 5h
	})
	s := mustLeg(t, m)
	if len(*ups) != 0 {
		t.Fatalf("a step was started at W+5h: %v", *ups)
	}
	if s.Stopped != LegSkipWindowEnd || legStepFor(s, "nextcloud").Reason != LegSkipWindowEnd {
		t.Fatalf("summary must say window_end: %+v", s)
	}
}

// TestLeg_SwitchOffPressesNothing — decision 12: the switch off means no press at all.
func TestLeg_SwitchOffPressesNothing(t *testing.T) {
	m, _, _, ups, _ := legManager(t)
	legOpts(m, func(o *UpdateLegOptions) { o.Enabled = func() bool { return false } })
	s := mustLeg(t, m)
	if len(*ups) != 0 || s.Enabled || s.Stopped != LegSkipSwitchedOff {
		t.Fatalf("switch off: ups=%v summary=%+v", *ups, s)
	}
}

// TestLeg_TransientRefusalIsRetriedOnce — `busy` is a passing reason: one retry, then a named skip.
func TestLeg_TransientRefusalIsRetriedOnce(t *testing.T) {
	m, _, g, ups, logBuf := legManager(t)
	g.busy = true
	legOpts(m, nil)
	s := mustLeg(t, m)
	if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != "refused:busy" {
		t.Fatalf("busy: ups=%v summary=%+v", *ups, s)
	}
	if n := strings.Count(logBuf.String(), "REFUSED (busy)"); n != 2 {
		t.Fatalf("a busy refusal must be pressed exactly twice (once + one retry), saw %d", n)
	}
}

// TestLeg_HeldAppIsNotPressed — a held app is terminal for the leg.
func TestLeg_HeldAppIsNotPressed(t *testing.T) {
	m, _, g, ups, _ := legManager(t)
	g.held, g.holdWhy = true, "HELD"
	legOpts(m, nil)
	s := mustLeg(t, m)
	if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != LegSkipHeld {
		t.Fatalf("held: ups=%v summary=%+v", *ups, s)
	}
}

// TestLeg_GateSeesTheLegAndItsStep — UpdateLegState answers active while the leg runs and stepRunning
// while its step is in flight (the full-system backup's gate reads exactly this), and both go false after.
func TestLeg_GateSeesTheLegAndItsStep(t *testing.T) {
	m, _, _, _, _ := legManager(t)
	release := make(chan struct{})
	var mu sync.Mutex
	var seen [][2]bool
	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) {
		a, s := m.UpdateLegState()
		mu.Lock()
		seen = append(seen, [2]bool{a, s})
		mu.Unlock()
		<-release
		return true, "ok"
	}
	legOpts(m, nil)
	done := make(chan struct{})
	go func() { mustLeg(t, m); close(done) }()
	deadline := time.Now().Add(3 * time.Second)
	for {
		mu.Lock()
		n := len(seen)
		mu.Unlock()
		if n > 0 || time.Now().After(deadline) {
			break
		}
		time.Sleep(2 * time.Millisecond)
	}
	close(release)
	<-done
	if len(seen) == 0 || !seen[0][0] || !seen[0][1] {
		t.Fatalf("during the step's verify the gate must see (active, stepRunning) = (true, true), saw %v", seen)
	}
	if a, s := m.UpdateLegState(); a || s {
		t.Fatalf("after the leg the gate must see (false, false), got (%v, %v)", a, s)
	}
}

// TestD28_NoCrashLoopStopDuringAnAutomaticStep — decision 28's stop must not fire while the leg's step
// restarts the app (pull, up, verify, undo): an app that is Updating is not sampled and its history is
// dropped, so restarts climbing during the step never make a verdict.
//
// COMPANION RED-PROOF (REPORT.md): drop `st.Updating` from ObserveUnhealthy's skip — this test fails at
// "a crash-loop verdict fired during an automatic step".
func TestD28_NoCrashLoopStopDuringAnAutomaticStep(t *testing.T) {
	m, _, _, _, _ := legManager(t)
	if err := m.ScanStacks(); err != nil {
		t.Fatal(err)
	}
	var cmu sync.Mutex
	restarts := 0
	m.execFn = func(name string, args ...string) (string, error) {
		if name == "docker" && len(args) > 0 && args[0] == "inspect" {
			cmu.Lock()
			defer cmu.Unlock()
			return "/nextcloud-web-1|" + strconv.Itoa(restarts) + "\n", nil
		}
		return "", nil
	}
	release := make(chan struct{})
	inVerify := make(chan struct{}, 1)
	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) {
		inVerify <- struct{}{}
		<-release
		return true, "ok"
	}
	legOpts(m, nil)
	done := make(chan struct{})
	go func() { m.RunUpdateLeg(context.Background(), "test"); close(done) }()
	select {
	case <-inVerify:
	case <-time.After(3 * time.Second):
		t.Fatal("the step never reached its verify")
	}
	// the container the samples read (set now: the leg's rescan rebuilt the stack before the step)
	m.mu.Lock()
	m.stacks["nextcloud"].Containers = []ContainerInfo{{Name: "nextcloud-web-1"}}
	m.mu.Unlock()
	t0 := time.Date(2026, 9, 25, 4, 20, 0, 0, time.UTC)
	for i := 0; i < 10; i++ { // ten samples a minute apart, +3 restarts each: 27 restarts in 9 minutes
		cmu.Lock()
		restarts += 3
		cmu.Unlock()
		if v := m.ObserveUnhealthy(t0.Add(time.Duration(i)*time.Minute), nil); len(v) > 0 {
			close(release)
			<-done
			t.Fatalf("a crash-loop verdict fired during an automatic step: %+v", v)
		}
	}
	close(release)
	<-done
}

// TestLegDeadline — W+5h of the night that contains now, across midnight too.
func TestLegDeadline(t *testing.T) {
	loc := time.UTC
	cases := []struct {
		now    time.Time
		window string
		want   time.Time
	}{
		{time.Date(2026, 9, 25, 4, 15, 0, 0, loc), "02:30", time.Date(2026, 9, 25, 7, 30, 0, 0, loc)},
		{time.Date(2026, 9, 25, 1, 0, 0, 0, loc), "23:00", time.Date(2026, 9, 25, 4, 0, 0, 0, loc)},
		{time.Date(2026, 9, 25, 2, 0, 0, 0, loc), "02:30", time.Date(2026, 9, 24, 7, 30, 0, 0, loc)}, // before W: last night's
		{time.Date(2026, 9, 25, 4, 15, 0, 0, loc), "junk", time.Date(2026, 9, 25, 7, 30, 0, 0, loc)}, // default 02:30
	}
	for _, c := range cases {
		if got := LegDeadline(c.now, c.window, loc); !got.Equal(c.want) {
			t.Errorf("LegDeadline(%s, %q) = %s, want %s", c.now.Format("15:04"), c.window, got, c.want)
		}
	}
}