package stacks import ( "bytes" "context" "encoding/json" "path/filepath" "strconv" "strings" "sync" "testing" "time" ) // v0.271.0 — the automatic update leg (`09` §3 decisions 11–15, 20; §6.4 part 7). Every test runs the // REAL leg over the REAL guarded update job with the process boundaries faked (ladderManager: nextcloud // pinned at A, the catalog at C, a two-step ladder A→B→C, B's own definition in steps/), and reads the // EFFECT back: the pin in app.yaml, the definitions `up` ran on, app.yaml's records, the summary. // legNow is inside the night of window 02:30 (the leg starts at W+105m = 04:15). var legNow = time.Date(2026, 9, 25, 4, 15, 0, 0, time.UTC) func legOpts(m *Manager, mut func(o *UpdateLegOptions)) { o := UpdateLegOptions{ Enabled: func() bool { return true }, WindowStart: func() string { return "02:30" }, Location: time.UTC, Poll: 2 * time.Millisecond, RetryWait: time.Millisecond, Now: func() time.Time { return legNow }, } if mut != nil { mut(&o) } m.SetUpdateLeg(o) } // writeLadder replaces the catalog's .felhom.yml with the given ladder lines. func writeLadder(t *testing.T, m *Manager, lines ...string) { t.Helper() catDir := filepath.Dir(m.CatalogTemplatePath("nextcloud", "docker-compose.yml")) mustWrite(t, filepath.Join(catDir, ".felhom.yml"), "display_name: Nextcloud\nupdate_ladder:\n"+strings.Join(lines, "")) } // ladderLineMarks is ladderLine with a verdict and a marks object. func ladderLineMarks(from, to, verdict, marks string) string { return ` - {"from": {"web": "` + from + `"}, "to": {"web": "` + to + `"}, "digest": {"web": "sha256:` + strings.Repeat("a", 64) + `"}, "verdict": "` + verdict + `", "marks": ` + marks + `}` + "\n" } func legStepFor(s *UpdateLegSummary, app string) LegStep { for _, st := range s.Steps { if st.App == app { return st } } return LegStep{} } // legManager is ladderManager plus what a real box has and the fakes do not produce: an installed-image // record (CatalogOrder answers Unknown without one, and the leg never presses Unknown). Every `up` records // the image it brought up, as recordInstalledImages does on a box from the running container. func legManager(t *testing.T) (*Manager, string, *fakeGuards, *[]string, *bytes.Buffer) { t.Helper() m, dir, g, ups, logBuf := ladderManager(t, true) setInstalled := func(img string) { cfg := LoadAppConfig(dir) cfg.InstalledImages = map[string]InstalledImage{"web": {Ref: img}} if err := SaveAppConfig(dir, cfg, m.encKey, nil); err != nil { t.Fatal(err) } } setInstalled(ladderA) inner := m.updateComposeFn m.updateComposeFn = func(d string, env []string, args ...string) (string, error) { out, err := inner(d, env, args...) if err == nil && args[0] == "up" { if imgs, perr := ParseComposeImages(ComposePathIn(d)); perr == nil { setInstalled(strings.SplitN(imgs["web"], "@", 2)[0]) } } return out, err } return m, dir, g, ups, logBuf } func mustLeg(t *testing.T, m *Manager) *UpdateLegSummary { t.Helper() if err := m.ScanStacks(); err != nil { t.Fatal(err) } s := m.RunUpdateLeg(context.Background(), "test") if s == nil { t.Fatal("the leg did not run") } return s } // TestLeg_OneStepPerAppPerNight — a box two steps behind takes ONE step a night (decision 14 + the // brief: "one tested step per app"), with the step's own definition, and the summary says done=1. // // COMPANION RED-PROOF (REPORT.md): make the leg loop while the app stays behind — this test fails at // "the leg took 2 steps in one night". func TestLeg_OneStepPerAppPerNight(t *testing.T) { m, dir, _, ups, _ := legManager(t) legOpts(m, nil) s := mustLeg(t, m) if len(*ups) != 1 { t.Fatalf("the leg took %d steps in one night (ups=%v), want exactly one", len(*ups), *ups) } if pinOf(t, dir) != ladderB { t.Fatalf("pinned %s, want the first tested step B", pinOf(t, dir)) } if s.Done != 1 || legStepFor(s, "nextcloud").Outcome != LegOutcomeDone { t.Fatalf("summary %+v, want done=1 for nextcloud", s) } cfg := LoadAppConfig(dir) if cfg.LastAutoUpdate == nil || cfg.LastAutoUpdate.Outcome != LegOutcomeDone || cfg.LastAutoUpdate.To["web"] != ladderB { t.Fatalf("app.yaml must carry the page's record of the automatic step, got %+v", cfg.LastAutoUpdate) } // the next night climbs the next step s2 := mustLeg(t, m) if pinOf(t, dir) != ladderC || s2.Done != 1 { t.Fatalf("night 2 ended on %s (done=%d), want C", pinOf(t, dir), s2.Done) } // and the third night finds nothing to do — no press, not even counted as a skip s3 := mustLeg(t, m) if len(*ups) != 2 || s3.Done+s3.Skipped != 0 { t.Fatalf("night 3 at the head pressed or counted something: ups=%v summary=%+v", *ups, s3) } } // TestR678_StepsLeftFreshAtDone — the page's steps-left count and the pin are current the moment the // update says `done`, with no scan in between. MEASURED 2026-09-24: ~50 s stale, six re-presses. // // COMPANION RED-PROOF (REPORT.md): drop the ScanStacks call in verifyAndConclude — this test fails at // "steps left read 2 right after the step ended done". func TestR678_StepsLeftFreshAtDone(t *testing.T) { m, _, _, _, _ := ladderManager(t, true) if err := m.ScanStacks(); err != nil { t.Fatal(err) } if st, _ := m.GetStack("nextcloud"); st.LadderStepsLeft != 2 { t.Fatalf("before: steps left %d, want 2", st.LadderStepsLeft) } if err := m.StartGuardedUpdate("nextcloud"); err != nil { t.Fatal(err) } st := waitUpdateDone(t, m, "nextcloud") if st.UpdatePhase != UpdatePhaseDone { t.Fatalf("ended %q", st.UpdatePhase) } if st.LadderStepsLeft != 1 { t.Fatalf("steps left read %d right after the step ended done, want 1", st.LadderStepsLeft) } if st.AppConfig == nil || st.AppConfig.PinnedImages["web"] != ladderB { t.Fatalf("the in-memory pin is stale after done: %+v", st.AppConfig) } } // TestR680_FailedStepIsNotPressedAgain — B fails and is undone: the box records the failed step, the // next night's leg skips it (failed_before) and nothing is brought up; a PERSON can still press it; and // when the catalog's ladder changes, the leg may try again. // // COMPANION RED-PROOF (REPORT.md): drop the FailedStep check in legCandidate — this test fails at // "night 2 pressed the step that was undone on night 1". func TestR680_FailedStepIsNotPressedAgain(t *testing.T) { m, dir, _, ups, _ := legManager(t) m.undoCopier = newFakeCopier(map[string]string{undoVol: "OLD"}) m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) { return false, "B unhealthy" } m.updateUndoHealthFn = func(context.Context, string, time.Duration, *Metadata) (bool, string) { return true, "A answers" } legOpts(m, nil) s1 := mustLeg(t, m) if s1.Undone != 1 || pinOf(t, dir) != ladderA { t.Fatalf("night 1: %+v on %s, want undone back on A", s1, pinOf(t, dir)) } cfg := LoadAppConfig(dir) if cfg.FailedStep == nil || cfg.FailedStep.To["web"] != ladderB || cfg.FailedStep.Outcome != "undone" || cfg.FailedStep.Ladder == "" { t.Fatalf("the failed step must be recorded on the box, got %+v", cfg.FailedStep) } upsAfter1 := len(*ups) s2 := mustLeg(t, m) if len(*ups) != upsAfter1 { t.Fatalf("night 2 pressed the step that was undone on night 1 (ups=%v)", *ups) } if st := legStepFor(s2, "nextcloud"); st.Outcome != LegOutcomeSkipped || st.Reason != LegSkipFailedBefore { t.Fatalf("night 2 must skip with %q, got %+v", LegSkipFailedBefore, st) } // a person still can — the record binds the leg only if err := m.StartGuardedUpdate("nextcloud"); err != nil { t.Fatalf("a person's press must not be refused by the failed-step record: %v", err) } waitUpdateDone(t, m, "nextcloud") // the catalog re-tests the step (a new tested_at): the ladder's print changes → the leg tries again writeLadder(t, m, ` - {"from": {"web": "`+ladderA+`"}, "to": {"web": "`+ladderB+`"}, "digest": {"web": "sha256:`+strings.Repeat("a", 64)+`"}, "verdict": "proven", "tested_at": "2026-09-26T01:00:00Z"}`+"\n", ladderLine(ladderB, ladderC)) before := len(*ups) mustLeg(t, m) if len(*ups) <= before || (*ups)[before] != ladderB { t.Fatalf("after the catalog changed the ladder the leg must try the step again (ups=%v)", *ups) } } // TestLeg_NeedsPersonIsNeverTaken — decision 13's mark: the leg never takes a needs_person step. // // COMPANION RED-PROOF (REPORT.md): drop the NeedsPerson check — this test fails at "a needs_person step // was pressed by the leg". func TestLeg_NeedsPersonIsNeverTaken(t *testing.T) { m, dir, _, ups, _ := legManager(t) writeLadder(t, m, ladderLineMarks(ladderA, ladderB, "proven", `{"files_may_change": false, "needs_person": "the admin must re-login after this step", "memory_tight": false}`), ladderLine(ladderB, ladderC)) legOpts(m, nil) s := mustLeg(t, m) if len(*ups) != 0 || pinOf(t, dir) != ladderA { t.Fatalf("a needs_person step was pressed by the leg (ups=%v)", *ups) } if st := legStepFor(s, "nextcloud"); st.Reason != LegSkipNeedsPerson { t.Fatalf("skip reason %+v, want %q", st, LegSkipNeedsPerson) } } // TestLeg_FilesMayChangeNeedsAWholeCopy — decision 13's other mark: taken only when a fresh WHOLE copy // exists (the backup side's truth table, asked through FreshWholeCopy). func TestLeg_FilesMayChangeNeedsAWholeCopy(t *testing.T) { m, dir, _, ups, logBuf := legManager(t) writeLadder(t, m, ladderLineMarks(ladderA, ladderB, "proven", `{"files_may_change": true, "needs_person": null, "memory_tight": false}`), ladderLine(ladderB, ladderC)) whole := false legOpts(m, func(o *UpdateLegOptions) { o.FreshWholeCopy = func(context.Context, string) (bool, string) { return whole, "tier 2 copy from FAKE-TIME" } }) s := mustLeg(t, m) if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != LegSkipFilesNoCopy { t.Fatalf("without a whole copy the step must be skipped: ups=%v summary=%+v", *ups, s) } whole = true s = mustLeg(t, m) if len(*ups) != 1 || pinOf(t, dir) != ladderB || s.Done != 1 { t.Fatalf("with a fresh whole copy the step must be taken: ups=%v pin=%s", *ups, pinOf(t, dir)) } // R-687 (v0.273.0): the taken step names the copy that allowed it. COMPANION RED-PROOF: drop the log line. if !strings.Contains(logBuf.String(), "taken, a fresh whole copy exists: tier 2 copy from FAKE-TIME") { t.Fatal("a taken files_may_change step must log which whole copy allowed it") } // unwired check = no whole copy (fail closed) m2, _, _, ups2, _ := legManager(t) writeLadder(t, m2, ladderLineMarks(ladderA, ladderB, "proven", `{"files_may_change": true, "needs_person": null, "memory_tight": false}`), ladderLine(ladderB, ladderC)) legOpts(m2, nil) mustLeg(t, m2) if len(*ups2) != 0 { t.Fatal("with no whole-copy check wired a files_may_change step must never be taken") } } // TestLeg_OlderThanLadderIsNotTouched — an installed version matching no ladder entry: the leg leaves it // alone and says so by name (a person's press still jumps, TestLadder_UnknownInstalledJumpsAndSaysSo). func TestLeg_OlderThanLadderIsNotTouched(t *testing.T) { m, dir, _, ups, logBuf := legManager(t) old := "services:\n web:\n image: nextcloud:30.0.0-apache\nvolumes:\n db:\n" mustWrite(t, ComposePathIn(dir), old) mustWrite(t, AppliedComposePath(dir), old) mustWrite(t, filepath.Join(dir, "app.yaml"), "deployed: true\nenv: {}\npinned_images:\n web: nextcloud:30.0.0-apache\ninstalled_images:\n web:\n ref: nextcloud:30.0.0-apache\n") legOpts(m, nil) s := mustLeg(t, m) if len(*ups) != 0 { t.Fatalf("an app older than the ladder was pressed: %v", *ups) } if legStepFor(s, "nextcloud").Reason != LegSkipOlderThanLadder || !strings.Contains(logBuf.String(), "nextcloud:30.0.0-apache") { t.Fatalf("skip must be older_than_ladder and logged by name; summary %+v", s) } } // TestLeg_OnlyProvenStepsAreTaken — an `unrecorded` (backfilled, never tested) entry is not a test. func TestLeg_OnlyProvenStepsAreTaken(t *testing.T) { m, _, _, ups, _ := legManager(t) writeLadder(t, m, ladderLineMarks(ladderA, ladderB, "unrecorded", `{"files_may_change": false, "needs_person": null, "memory_tight": false}`), ladderLine(ladderB, ladderC)) legOpts(m, nil) s := mustLeg(t, m) if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != LegSkipNotProven { t.Fatalf("an unproven step was taken or mis-reasoned: ups=%v %+v", *ups, s) } } // TestLeg_NoStepAtOrAfterW5h — decision 20: the leg starts nothing at W+5h. // // COMPANION RED-PROOF (REPORT.md): drop the deadline arm of legStop — this test fails at "a step was // started at W+5h". func TestLeg_NoStepAtOrAfterW5h(t *testing.T) { m, _, _, ups, _ := legManager(t) legOpts(m, func(o *UpdateLegOptions) { o.Now = func() time.Time { return time.Date(2026, 9, 25, 7, 30, 0, 0, time.UTC) } // W 02:30 + 5h }) s := mustLeg(t, m) if len(*ups) != 0 { t.Fatalf("a step was started at W+5h: %v", *ups) } if s.Stopped != LegSkipWindowEnd || legStepFor(s, "nextcloud").Reason != LegSkipWindowEnd { t.Fatalf("summary must say window_end: %+v", s) } } // TestLeg_SwitchOffPressesNothing — decision 12: the switch off means no press at all. func TestLeg_SwitchOffPressesNothing(t *testing.T) { m, _, _, ups, _ := legManager(t) legOpts(m, func(o *UpdateLegOptions) { o.Enabled = func() bool { return false } }) s := mustLeg(t, m) if len(*ups) != 0 || s.Enabled || s.Stopped != LegSkipSwitchedOff { t.Fatalf("switch off: ups=%v summary=%+v", *ups, s) } } // TestLeg_TransientRefusalIsRetriedOnce — `busy` is a passing reason: one retry, then a named skip. func TestLeg_TransientRefusalIsRetriedOnce(t *testing.T) { m, _, g, ups, logBuf := legManager(t) g.busy = true legOpts(m, nil) s := mustLeg(t, m) if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != "refused:busy" { t.Fatalf("busy: ups=%v summary=%+v", *ups, s) } if n := strings.Count(logBuf.String(), "REFUSED (busy)"); n != 2 { t.Fatalf("a busy refusal must be pressed exactly twice (once + one retry), saw %d", n) } } // TestLeg_HeldAppIsNotPressed — a held app is terminal for the leg. func TestLeg_HeldAppIsNotPressed(t *testing.T) { m, _, g, ups, _ := legManager(t) g.held, g.holdWhy = true, "HELD" legOpts(m, nil) s := mustLeg(t, m) if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != LegSkipHeld { t.Fatalf("held: ups=%v summary=%+v", *ups, s) } } // TestLeg_GateSeesTheLegAndItsStep — UpdateLegState answers active while the leg runs and stepRunning // while its step is in flight (the full-system backup's gate reads exactly this), and both go false after. func TestLeg_GateSeesTheLegAndItsStep(t *testing.T) { m, _, _, _, _ := legManager(t) release := make(chan struct{}) var mu sync.Mutex var seen [][2]bool m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) { a, s := m.UpdateLegState() mu.Lock() seen = append(seen, [2]bool{a, s}) mu.Unlock() <-release return true, "ok" } legOpts(m, nil) done := make(chan struct{}) go func() { mustLeg(t, m); close(done) }() deadline := time.Now().Add(3 * time.Second) for { mu.Lock() n := len(seen) mu.Unlock() if n > 0 || time.Now().After(deadline) { break } time.Sleep(2 * time.Millisecond) } close(release) <-done if len(seen) == 0 || !seen[0][0] || !seen[0][1] { t.Fatalf("during the step's verify the gate must see (active, stepRunning) = (true, true), saw %v", seen) } if a, s := m.UpdateLegState(); a || s { t.Fatalf("after the leg the gate must see (false, false), got (%v, %v)", a, s) } } // TestD28_NoCrashLoopStopDuringAnAutomaticStep — decision 28's stop must not fire while the leg's step // restarts the app (pull, up, verify, undo): an app that is Updating is not sampled and its history is // dropped, so restarts climbing during the step never make a verdict. // // COMPANION RED-PROOF (REPORT.md): drop `st.Updating` from ObserveUnhealthy's skip — this test fails at // "a crash-loop verdict fired during an automatic step". func TestD28_NoCrashLoopStopDuringAnAutomaticStep(t *testing.T) { m, _, _, _, _ := legManager(t) if err := m.ScanStacks(); err != nil { t.Fatal(err) } var cmu sync.Mutex restarts := 0 m.execFn = func(name string, args ...string) (string, error) { if name == "docker" && len(args) > 0 && args[0] == "inspect" { cmu.Lock() defer cmu.Unlock() return "/nextcloud-web-1|" + strconv.Itoa(restarts) + "\n", nil } return "", nil } release := make(chan struct{}) inVerify := make(chan struct{}, 1) m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) { inVerify <- struct{}{} <-release return true, "ok" } legOpts(m, nil) done := make(chan struct{}) go func() { m.RunUpdateLeg(context.Background(), "test"); close(done) }() select { case <-inVerify: case <-time.After(3 * time.Second): t.Fatal("the step never reached its verify") } // the container the samples read (set now: the leg's rescan rebuilt the stack before the step) m.mu.Lock() m.stacks["nextcloud"].Containers = []ContainerInfo{{Name: "nextcloud-web-1"}} m.mu.Unlock() t0 := time.Date(2026, 9, 25, 4, 20, 0, 0, time.UTC) for i := 0; i < 10; i++ { // ten samples a minute apart, +3 restarts each: 27 restarts in 9 minutes cmu.Lock() restarts += 3 cmu.Unlock() if v := m.ObserveUnhealthy(t0.Add(time.Duration(i)*time.Minute), nil); len(v) > 0 { close(release) <-done t.Fatalf("a crash-loop verdict fired during an automatic step: %+v", v) } } close(release) <-done } // TestLegDeadline — W+5h of the night that contains now, across midnight too. func TestLegDeadline(t *testing.T) { loc := time.UTC cases := []struct { now time.Time window string want time.Time }{ {time.Date(2026, 9, 25, 4, 15, 0, 0, loc), "02:30", time.Date(2026, 9, 25, 7, 30, 0, 0, loc)}, {time.Date(2026, 9, 25, 1, 0, 0, 0, loc), "23:00", time.Date(2026, 9, 25, 4, 0, 0, 0, loc)}, {time.Date(2026, 9, 25, 2, 0, 0, 0, loc), "02:30", time.Date(2026, 9, 24, 7, 30, 0, 0, loc)}, // before W: last night's {time.Date(2026, 9, 25, 4, 15, 0, 0, loc), "junk", time.Date(2026, 9, 25, 7, 30, 0, 0, loc)}, // default 02:30 } for _, c := range cases { if got := LegDeadline(c.now, c.window, loc); !got.Equal(c.want) { t.Errorf("LegDeadline(%s, %q) = %s, want %s", c.now.Format("15:04"), c.window, got, c.want) } } } // TestR687_EmptyLegReportsEmptySteps — a leg that pressed nothing reports `"steps": []` to the hub, not // `null` (read live 2026-09-25 from demo-hp's report). COMPANION RED-PROOF: copy with append(nil, …). func TestR687_EmptyLegReportsEmptySteps(t *testing.T) { m, _, _, _, _ := legManager(t) m.leg.mu.Lock() m.leg.last = &UpdateLegSummary{Trigger: "after-offsite", Steps: []LegStep{}} m.leg.mu.Unlock() b, err := json.Marshal(m.LastUpdateLegSummary()) if err != nil { t.Fatal(err) } if !strings.Contains(string(b), `"steps":[]`) { t.Fatalf("an empty leg must report steps as [], got %s", b) } }