controller v0.271.0: automatic app updates — the update leg after the off-site copy, the backup gate waits, the switch (09 6.4 part 7; R-680, R-678, R-643)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-09-24 22:07:24 +02:00
parent 1495ca43fb
commit 9cf13a3add
33 changed files with 1672 additions and 14 deletions
@@ -0,0 +1,454 @@
+package stacks
+
+import (
+	"bytes"
+	"context"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// v0.271.0 — the automatic update leg (`09` §3 decisions 11–15, 20; §6.4 part 7). Every test runs the
+// REAL leg over the REAL guarded update job with the process boundaries faked (ladderManager: nextcloud
+// pinned at A, the catalog at C, a two-step ladder A→B→C, B's own definition in steps/), and reads the
+// EFFECT back: the pin in app.yaml, the definitions `up` ran on, app.yaml's records, the summary.
+
+// legNow is inside the night of window 02:30 (the leg starts at W+105m = 04:15).
+var legNow = time.Date(2026, 9, 25, 4, 15, 0, 0, time.UTC)
+
+func legOpts(m *Manager, mut func(o *UpdateLegOptions)) {
+	o := UpdateLegOptions{
+		Enabled:     func() bool { return true },
+		WindowStart: func() string { return "02:30" },
+		Location:    time.UTC,
+		Poll:        2 * time.Millisecond,
+		RetryWait:   time.Millisecond,
+		Now:         func() time.Time { return legNow },
+	}
+	if mut != nil {
+		mut(&o)
+	}
+	m.SetUpdateLeg(o)
+}
+
+// writeLadder replaces the catalog's .felhom.yml with the given ladder lines.
+func writeLadder(t *testing.T, m *Manager, lines ...string) {
+	t.Helper()
+	catDir := filepath.Dir(m.CatalogTemplatePath("nextcloud", "docker-compose.yml"))
+	mustWrite(t, filepath.Join(catDir, ".felhom.yml"), "display_name: Nextcloud\nupdate_ladder:\n"+strings.Join(lines, ""))
+}
+
+// ladderLineMarks is ladderLine with a verdict and a marks object.
+func ladderLineMarks(from, to, verdict, marks string) string {
+	return `  - {"from": {"web": "` + from + `"}, "to": {"web": "` + to + `"}, "digest": {"web": "sha256:` +
+		strings.Repeat("a", 64) + `"}, "verdict": "` + verdict + `", "marks": ` + marks + `}` + "\n"
+}
+
+func legStepFor(s *UpdateLegSummary, app string) LegStep {
+	for _, st := range s.Steps {
+		if st.App == app {
+			return st
+		}
+	}
+	return LegStep{}
+}
+
+// legManager is ladderManager plus what a real box has and the fakes do not produce: an installed-image
+// record (CatalogOrder answers Unknown without one, and the leg never presses Unknown). Every `up` records
+// the image it brought up, as recordInstalledImages does on a box from the running container.
+func legManager(t *testing.T) (*Manager, string, *fakeGuards, *[]string, *bytes.Buffer) {
+	t.Helper()
+	m, dir, g, ups, logBuf := ladderManager(t, true)
+	setInstalled := func(img string) {
+		cfg := LoadAppConfig(dir)
+		cfg.InstalledImages = map[string]InstalledImage{"web": {Ref: img}}
+		if err := SaveAppConfig(dir, cfg, m.encKey, nil); err != nil {
+			t.Fatal(err)
+		}
+	}
+	setInstalled(ladderA)
+	inner := m.updateComposeFn
+	m.updateComposeFn = func(d string, env []string, args ...string) (string, error) {
+		out, err := inner(d, env, args...)
+		if err == nil && args[0] == "up" {
+			if imgs, perr := ParseComposeImages(ComposePathIn(d)); perr == nil {
+				setInstalled(strings.SplitN(imgs["web"], "@", 2)[0])
+			}
+		}
+		return out, err
+	}
+	return m, dir, g, ups, logBuf
+}
+
+func mustLeg(t *testing.T, m *Manager) *UpdateLegSummary {
+	t.Helper()
+	if err := m.ScanStacks(); err != nil {
+		t.Fatal(err)
+	}
+	s := m.RunUpdateLeg(context.Background(), "test")
+	if s == nil {
+		t.Fatal("the leg did not run")
+	}
+	return s
+}
+
+// TestLeg_OneStepPerAppPerNight — a box two steps behind takes ONE step a night (decision 14 + the
+// brief: "one tested step per app"), with the step's own definition, and the summary says done=1.
+//
+// COMPANION RED-PROOF (REPORT.md): make the leg loop while the app stays behind — this test fails at
+// "the leg took 2 steps in one night".
+func TestLeg_OneStepPerAppPerNight(t *testing.T) {
+	m, dir, _, ups, _ := legManager(t)
+	legOpts(m, nil)
+	s := mustLeg(t, m)
+	if len(*ups) != 1 {
+		t.Fatalf("the leg took %d steps in one night (ups=%v), want exactly one", len(*ups), *ups)
+	}
+	if pinOf(t, dir) != ladderB {
+		t.Fatalf("pinned %s, want the first tested step B", pinOf(t, dir))
+	}
+	if s.Done != 1 || legStepFor(s, "nextcloud").Outcome != LegOutcomeDone {
+		t.Fatalf("summary %+v, want done=1 for nextcloud", s)
+	}
+	cfg := LoadAppConfig(dir)
+	if cfg.LastAutoUpdate == nil || cfg.LastAutoUpdate.Outcome != LegOutcomeDone || cfg.LastAutoUpdate.To["web"] != ladderB {
+		t.Fatalf("app.yaml must carry the page's record of the automatic step, got %+v", cfg.LastAutoUpdate)
+	}
+	// the next night climbs the next step
+	s2 := mustLeg(t, m)
+	if pinOf(t, dir) != ladderC || s2.Done != 1 {
+		t.Fatalf("night 2 ended on %s (done=%d), want C", pinOf(t, dir), s2.Done)
+	}
+	// and the third night finds nothing to do — no press, not even counted as a skip
+	s3 := mustLeg(t, m)
+	if len(*ups) != 2 || s3.Done+s3.Skipped != 0 {
+		t.Fatalf("night 3 at the head pressed or counted something: ups=%v summary=%+v", *ups, s3)
+	}
+}
+
+// TestR678_StepsLeftFreshAtDone — the page's steps-left count and the pin are current the moment the
+// update says `done`, with no scan in between. MEASURED 2026-09-24: ~50 s stale, six re-presses.
+//
+// COMPANION RED-PROOF (REPORT.md): drop the ScanStacks call in verifyAndConclude — this test fails at
+// "steps left read 2 right after the step ended done".
+func TestR678_StepsLeftFreshAtDone(t *testing.T) {
+	m, _, _, _, _ := ladderManager(t, true)
+	if err := m.ScanStacks(); err != nil {
+		t.Fatal(err)
+	}
+	if st, _ := m.GetStack("nextcloud"); st.LadderStepsLeft != 2 {
+		t.Fatalf("before: steps left %d, want 2", st.LadderStepsLeft)
+	}
+	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
+		t.Fatal(err)
+	}
+	st := waitUpdateDone(t, m, "nextcloud")
+	if st.UpdatePhase != UpdatePhaseDone {
+		t.Fatalf("ended %q", st.UpdatePhase)
+	}
+	if st.LadderStepsLeft != 1 {
+		t.Fatalf("steps left read %d right after the step ended done, want 1", st.LadderStepsLeft)
+	}
+	if st.AppConfig == nil || st.AppConfig.PinnedImages["web"] != ladderB {
+		t.Fatalf("the in-memory pin is stale after done: %+v", st.AppConfig)
+	}
+}
+
+// TestR680_FailedStepIsNotPressedAgain — B fails and is undone: the box records the failed step, the
+// next night's leg skips it (failed_before) and nothing is brought up; a PERSON can still press it; and
+// when the catalog's ladder changes, the leg may try again.
+//
+// COMPANION RED-PROOF (REPORT.md): drop the FailedStep check in legCandidate — this test fails at
+// "night 2 pressed the step that was undone on night 1".
+func TestR680_FailedStepIsNotPressedAgain(t *testing.T) {
+	m, dir, _, ups, _ := legManager(t)
+	m.undoCopier = newFakeCopier(map[string]string{undoVol: "OLD"})
+	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) { return false, "B unhealthy" }
+	m.updateUndoHealthFn = func(context.Context, string, time.Duration, *Metadata) (bool, string) { return true, "A answers" }
+	legOpts(m, nil)
+
+	s1 := mustLeg(t, m)
+	if s1.Undone != 1 || pinOf(t, dir) != ladderA {
+		t.Fatalf("night 1: %+v on %s, want undone back on A", s1, pinOf(t, dir))
+	}
+	cfg := LoadAppConfig(dir)
+	if cfg.FailedStep == nil || cfg.FailedStep.To["web"] != ladderB || cfg.FailedStep.Outcome != "undone" || cfg.FailedStep.Ladder == "" {
+		t.Fatalf("the failed step must be recorded on the box, got %+v", cfg.FailedStep)
+	}
+	upsAfter1 := len(*ups)
+
+	s2 := mustLeg(t, m)
+	if len(*ups) != upsAfter1 {
+		t.Fatalf("night 2 pressed the step that was undone on night 1 (ups=%v)", *ups)
+	}
+	if st := legStepFor(s2, "nextcloud"); st.Outcome != LegOutcomeSkipped || st.Reason != LegSkipFailedBefore {
+		t.Fatalf("night 2 must skip with %q, got %+v", LegSkipFailedBefore, st)
+	}
+
+	// a person still can — the record binds the leg only
+	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
+		t.Fatalf("a person's press must not be refused by the failed-step record: %v", err)
+	}
+	waitUpdateDone(t, m, "nextcloud")
+
+	// the catalog re-tests the step (a new tested_at): the ladder's print changes → the leg tries again
+	writeLadder(t, m,
+		`  - {"from": {"web": "`+ladderA+`"}, "to": {"web": "`+ladderB+`"}, "digest": {"web": "sha256:`+strings.Repeat("a", 64)+`"}, "verdict": "proven", "tested_at": "2026-09-26T01:00:00Z"}`+"\n",
+		ladderLine(ladderB, ladderC))
+	before := len(*ups)
+	mustLeg(t, m)
+	if len(*ups) <= before || (*ups)[before] != ladderB {
+		t.Fatalf("after the catalog changed the ladder the leg must try the step again (ups=%v)", *ups)
+	}
+}
+
+// TestLeg_NeedsPersonIsNeverTaken — decision 13's mark: the leg never takes a needs_person step.
+//
+// COMPANION RED-PROOF (REPORT.md): drop the NeedsPerson check — this test fails at "a needs_person step
+// was pressed by the leg".
+func TestLeg_NeedsPersonIsNeverTaken(t *testing.T) {
+	m, dir, _, ups, _ := legManager(t)
+	writeLadder(t, m,
+		ladderLineMarks(ladderA, ladderB, "proven", `{"files_may_change": false, "needs_person": "the admin must re-login after this step", "memory_tight": false}`),
+		ladderLine(ladderB, ladderC))
+	legOpts(m, nil)
+	s := mustLeg(t, m)
+	if len(*ups) != 0 || pinOf(t, dir) != ladderA {
+		t.Fatalf("a needs_person step was pressed by the leg (ups=%v)", *ups)
+	}
+	if st := legStepFor(s, "nextcloud"); st.Reason != LegSkipNeedsPerson {
+		t.Fatalf("skip reason %+v, want %q", st, LegSkipNeedsPerson)
+	}
+}
+
+// TestLeg_FilesMayChangeNeedsAWholeCopy — decision 13's other mark: taken only when a fresh WHOLE copy
+// exists (the backup side's truth table, asked through FreshWholeCopy).
+func TestLeg_FilesMayChangeNeedsAWholeCopy(t *testing.T) {
+	m, dir, _, ups, _ := legManager(t)
+	writeLadder(t, m,
+		ladderLineMarks(ladderA, ladderB, "proven", `{"files_may_change": true, "needs_person": null, "memory_tight": false}`),
+		ladderLine(ladderB, ladderC))
+	whole := false
+	legOpts(m, func(o *UpdateLegOptions) {
+		o.FreshWholeCopy = func(context.Context, string) (bool, string) { return whole, "fake" }
+	})
+	s := mustLeg(t, m)
+	if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != LegSkipFilesNoCopy {
+		t.Fatalf("without a whole copy the step must be skipped: ups=%v summary=%+v", *ups, s)
+	}
+	whole = true
+	s = mustLeg(t, m)
+	if len(*ups) != 1 || pinOf(t, dir) != ladderB || s.Done != 1 {
+		t.Fatalf("with a fresh whole copy the step must be taken: ups=%v pin=%s", *ups, pinOf(t, dir))
+	}
+	// unwired check = no whole copy (fail closed)
+	m2, _, _, ups2, _ := legManager(t)
+	writeLadder(t, m2,
+		ladderLineMarks(ladderA, ladderB, "proven", `{"files_may_change": true, "needs_person": null, "memory_tight": false}`),
+		ladderLine(ladderB, ladderC))
+	legOpts(m2, nil)
+	mustLeg(t, m2)
+	if len(*ups2) != 0 {
+		t.Fatal("with no whole-copy check wired a files_may_change step must never be taken")
+	}
+}
+
+// TestLeg_OlderThanLadderIsNotTouched — an installed version matching no ladder entry: the leg leaves it
+// alone and says so by name (a person's press still jumps, TestLadder_UnknownInstalledJumpsAndSaysSo).
+func TestLeg_OlderThanLadderIsNotTouched(t *testing.T) {
+	m, dir, _, ups, logBuf := legManager(t)
+	old := "services:\n  web:\n    image: nextcloud:30.0.0-apache\nvolumes:\n  db:\n"
+	mustWrite(t, ComposePathIn(dir), old)
+	mustWrite(t, AppliedComposePath(dir), old)
+	mustWrite(t, filepath.Join(dir, "app.yaml"), "deployed: true\nenv: {}\npinned_images:\n    web: nextcloud:30.0.0-apache\ninstalled_images:\n    web:\n        ref: nextcloud:30.0.0-apache\n")
+	legOpts(m, nil)
+	s := mustLeg(t, m)
+	if len(*ups) != 0 {
+		t.Fatalf("an app older than the ladder was pressed: %v", *ups)
+	}
+	if legStepFor(s, "nextcloud").Reason != LegSkipOlderThanLadder || !strings.Contains(logBuf.String(), "nextcloud:30.0.0-apache") {
+		t.Fatalf("skip must be older_than_ladder and logged by name; summary %+v", s)
+	}
+}
+
+// TestLeg_OnlyProvenStepsAreTaken — an `unrecorded` (backfilled, never tested) entry is not a test.
+func TestLeg_OnlyProvenStepsAreTaken(t *testing.T) {
+	m, _, _, ups, _ := legManager(t)
+	writeLadder(t, m,
+		ladderLineMarks(ladderA, ladderB, "unrecorded", `{"files_may_change": false, "needs_person": null, "memory_tight": false}`),
+		ladderLine(ladderB, ladderC))
+	legOpts(m, nil)
+	s := mustLeg(t, m)
+	if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != LegSkipNotProven {
+		t.Fatalf("an unproven step was taken or mis-reasoned: ups=%v %+v", *ups, s)
+	}
+}
+
+// TestLeg_NoStepAtOrAfterW5h — decision 20: the leg starts nothing at W+5h.
+//
+// COMPANION RED-PROOF (REPORT.md): drop the deadline arm of legStop — this test fails at "a step was
+// started at W+5h".
+func TestLeg_NoStepAtOrAfterW5h(t *testing.T) {
+	m, _, _, ups, _ := legManager(t)
+	legOpts(m, func(o *UpdateLegOptions) {
+		o.Now = func() time.Time { return time.Date(2026, 9, 25, 7, 30, 0, 0, time.UTC) } // W 02:30 + 5h
+	})
+	s := mustLeg(t, m)
+	if len(*ups) != 0 {
+		t.Fatalf("a step was started at W+5h: %v", *ups)
+	}
+	if s.Stopped != LegSkipWindowEnd || legStepFor(s, "nextcloud").Reason != LegSkipWindowEnd {
+		t.Fatalf("summary must say window_end: %+v", s)
+	}
+}
+
+// TestLeg_SwitchOffPressesNothing — decision 12: the switch off means no press at all.
+func TestLeg_SwitchOffPressesNothing(t *testing.T) {
+	m, _, _, ups, _ := legManager(t)
+	legOpts(m, func(o *UpdateLegOptions) { o.Enabled = func() bool { return false } })
+	s := mustLeg(t, m)
+	if len(*ups) != 0 || s.Enabled || s.Stopped != LegSkipSwitchedOff {
+		t.Fatalf("switch off: ups=%v summary=%+v", *ups, s)
+	}
+}
+
+// TestLeg_TransientRefusalIsRetriedOnce — `busy` is a passing reason: one retry, then a named skip.
+func TestLeg_TransientRefusalIsRetriedOnce(t *testing.T) {
+	m, _, g, ups, logBuf := legManager(t)
+	g.busy = true
+	legOpts(m, nil)
+	s := mustLeg(t, m)
+	if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != "refused:busy" {
+		t.Fatalf("busy: ups=%v summary=%+v", *ups, s)
+	}
+	if n := strings.Count(logBuf.String(), "REFUSED (busy)"); n != 2 {
+		t.Fatalf("a busy refusal must be pressed exactly twice (once + one retry), saw %d", n)
+	}
+}
+
+// TestLeg_HeldAppIsNotPressed — a held app is terminal for the leg.
+func TestLeg_HeldAppIsNotPressed(t *testing.T) {
+	m, _, g, ups, _ := legManager(t)
+	g.held, g.holdWhy = true, "HELD"
+	legOpts(m, nil)
+	s := mustLeg(t, m)
+	if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != LegSkipHeld {
+		t.Fatalf("held: ups=%v summary=%+v", *ups, s)
+	}
+}
+
+// TestLeg_GateSeesTheLegAndItsStep — UpdateLegState answers active while the leg runs and stepRunning
+// while its step is in flight (the full-system backup's gate reads exactly this), and both go false after.
+func TestLeg_GateSeesTheLegAndItsStep(t *testing.T) {
+	m, _, _, _, _ := legManager(t)
+	release := make(chan struct{})
+	var mu sync.Mutex
+	var seen [][2]bool
+	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) {
+		a, s := m.UpdateLegState()
+		mu.Lock()
+		seen = append(seen, [2]bool{a, s})
+		mu.Unlock()
+		<-release
+		return true, "ok"
+	}
+	legOpts(m, nil)
+	done := make(chan struct{})
+	go func() { mustLeg(t, m); close(done) }()
+	deadline := time.Now().Add(3 * time.Second)
+	for {
+		mu.Lock()
+		n := len(seen)
+		mu.Unlock()
+		if n > 0 || time.Now().After(deadline) {
+			break
+		}
+		time.Sleep(2 * time.Millisecond)
+	}
+	close(release)
+	<-done
+	if len(seen) == 0 || !seen[0][0] || !seen[0][1] {
+		t.Fatalf("during the step's verify the gate must see (active, stepRunning) = (true, true), saw %v", seen)
+	}
+	if a, s := m.UpdateLegState(); a || s {
+		t.Fatalf("after the leg the gate must see (false, false), got (%v, %v)", a, s)
+	}
+}
+
+// TestD28_NoCrashLoopStopDuringAnAutomaticStep — decision 28's stop must not fire while the leg's step
+// restarts the app (pull, up, verify, undo): an app that is Updating is not sampled and its history is
+// dropped, so restarts climbing during the step never make a verdict.
+//
+// COMPANION RED-PROOF (REPORT.md): drop `st.Updating` from ObserveUnhealthy's skip — this test fails at
+// "a crash-loop verdict fired during an automatic step".
+func TestD28_NoCrashLoopStopDuringAnAutomaticStep(t *testing.T) {
+	m, _, _, _, _ := legManager(t)
+	if err := m.ScanStacks(); err != nil {
+		t.Fatal(err)
+	}
+	var cmu sync.Mutex
+	restarts := 0
+	m.execFn = func(name string, args ...string) (string, error) {
+		if name == "docker" && len(args) > 0 && args[0] == "inspect" {
+			cmu.Lock()
+			defer cmu.Unlock()
+			return "/nextcloud-web-1|" + strconv.Itoa(restarts) + "\n", nil
+		}
+		return "", nil
+	}
+	release := make(chan struct{})
+	inVerify := make(chan struct{}, 1)
+	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) {
+		inVerify <- struct{}{}
+		<-release
+		return true, "ok"
+	}
+	legOpts(m, nil)
+	done := make(chan struct{})
+	go func() { m.RunUpdateLeg(context.Background(), "test"); close(done) }()
+	select {
+	case <-inVerify:
+	case <-time.After(3 * time.Second):
+		t.Fatal("the step never reached its verify")
+	}
+	// the container the samples read (set now: the leg's rescan rebuilt the stack before the step)
+	m.mu.Lock()
+	m.stacks["nextcloud"].Containers = []ContainerInfo{{Name: "nextcloud-web-1"}}
+	m.mu.Unlock()
+	t0 := time.Date(2026, 9, 25, 4, 20, 0, 0, time.UTC)
+	for i := 0; i < 10; i++ { // ten samples a minute apart, +3 restarts each: 27 restarts in 9 minutes
+		cmu.Lock()
+		restarts += 3
+		cmu.Unlock()
+		if v := m.ObserveUnhealthy(t0.Add(time.Duration(i)*time.Minute), nil); len(v) > 0 {
+			close(release)
+			<-done
+			t.Fatalf("a crash-loop verdict fired during an automatic step: %+v", v)
+		}
+	}
+	close(release)
+	<-done
+}
+
+// TestLegDeadline — W+5h of the night that contains now, across midnight too.
+func TestLegDeadline(t *testing.T) {
+	loc := time.UTC
+	cases := []struct {
+		now    time.Time
+		window string
+		want   time.Time
+	}{
+		{time.Date(2026, 9, 25, 4, 15, 0, 0, loc), "02:30", time.Date(2026, 9, 25, 7, 30, 0, 0, loc)},
+		{time.Date(2026, 9, 25, 1, 0, 0, 0, loc), "23:00", time.Date(2026, 9, 25, 4, 0, 0, 0, loc)},
+		{time.Date(2026, 9, 25, 2, 0, 0, 0, loc), "02:30", time.Date(2026, 9, 24, 7, 30, 0, 0, loc)}, // before W: last night's
+		{time.Date(2026, 9, 25, 4, 15, 0, 0, loc), "junk", time.Date(2026, 9, 25, 7, 30, 0, 0, loc)}, // default 02:30
+	}
+	for _, c := range cases {
+		if got := LegDeadline(c.now, c.window, loc); !got.Equal(c.want) {
+			t.Errorf("LegDeadline(%s, %q) = %s, want %s", c.now.Format("15:04"), c.window, got, c.want)
+		}
+	}
+}