diff --git a/controller/cmd/controller/bootwindow_test.go b/controller/cmd/controller/bootwindow_test.go index 15f7799..3f080f7 100644 --- a/controller/cmd/controller/bootwindow_test.go +++ b/controller/cmd/controller/bootwindow_test.go @@ -23,10 +23,12 @@ import ( // the whole point: the pre-v0.190.0 sweep sampled once and could not see a late settler. type windowStacks struct { // frames is the fleet as seen on each successive GetStacks() call; the last frame repeats. - frames [][]stacks.Stack - calls int - starts map[string]int - onStart func(*windowStacks, string) + frames [][]stacks.Stack + calls int + starts map[string]int + onStart func(*windowStacks, string) + refreshes int + refreshErr error // cycle makes the fleet NEVER settle: frames repeat forever instead of the last one sticking. // Required by the budget test — with frames that eventually stop changing, the window terminates // by SETTLING even with the budget removed, so the red-proof would not reach the hang it exists @@ -47,7 +49,13 @@ func (w *windowStacks) GetStacks() []stacks.Stack { return w.frames[i] } -func (w *windowStacks) RefreshStatus() error { return nil } +func (w *windowStacks) RefreshStatus() error { + w.refreshes++ + if w.refreshErr != nil { + return w.refreshErr + } + return nil +} func (w *windowStacks) StartStack(name string) error { if w.starts == nil { @@ -309,3 +317,50 @@ func TestBootWindow_CommonCaseFitsInsideTheDeadAppGrace(t *testing.T) { "between two docker events'", bootReconcileSample, bootReconcileStableFor) } } + +// --- the sample must observe REALITY, not the Manager's cache ------------------------------------ + +func TestBootWindow_EverySampleRefreshesTheStatus(t *testing.T) { + // FOUND BY LIVE VALIDATION, not review. GetStacks() returns the Manager's in-memory map, which + // the scheduler refreshes on its own 10 s cadence. Sampling every 5 s WITHOUT refreshing means two + // consecutive samples can be identical because the cache did not update — so the window declares + // "settled" on stale data and sweeps on a picture of the box from up to 10 s ago. On 9201 a + // container removed ~5 s before the window closed was still in the sampled fleet, and the sweep + // logged "no boot-orphaned apps" for an app that had none. + // + // RED-PROOF: delete the `_ = mgr.RefreshStatus()` line from sampleBootFleet and this test fails + // with refreshes=0. Demonstrated in REPORT.md §4. + shrinkWindow(t, time.Millisecond, 3, 500*time.Millisecond) + captureSweep(t) + w := &windowStacks{frames: [][]stacks.Stack{{upStack("immich")}}} + + runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0)) + + if w.refreshes < 3 { + t.Fatalf("the window refreshed %d time(s) for %d samples — every sample must observe reality, "+ + "or 'settled' can mean 'the cache did not update'", w.refreshes, w.calls) + } + // calls includes ONE extra GetStacks from the captured sweep itself, which does not sample. + if w.refreshes != w.calls-1 { + t.Fatalf("refreshes=%d but samples=%d — each sample must refresh exactly once before reading", + w.refreshes, w.calls-1) + } +} + +func TestBootWindow_RefreshErrorDoesNotStopTheWindow(t *testing.T) { + // A boot window that cannot reach docker is exactly when a stale verdict is most dangerous, but + // giving up entirely would leave the sweep un-run. Degrade, do not abort. + shrinkWindow(t, time.Millisecond, 2, 200*time.Millisecond) + seen := captureSweep(t) + w := &windowStacks{frames: [][]stacks.Stack{{upStack("immich")}}, refreshErr: errRefresh{}} + + runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0)) + + if len(*seen) != 1 { + t.Fatalf("sweeps=%d, want 1 — a refresh error must not abort the window", len(*seen)) + } +} + +type errRefresh struct{} + +func (errRefresh) Error() string { return "docker unreachable" } diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index 9448dd4..d677b46 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -1406,8 +1406,25 @@ type bootFleetSample struct { containers int } -// sampleBootFleet returns the fleet snapshot, sorted, so two samples compare by equality. +// sampleBootFleet REFRESHES, then returns the fleet snapshot, sorted, so two samples compare by +// equality. +// +// THE REFRESH IS LOAD-BEARING, and it was found by live validation, not by review. `GetStacks()` +// returns the Manager's IN-MEMORY map, which the scheduler refreshes on its own 10 s cadence +// (`status-refresh`, main.go). Sampling it every 5 s without refreshing means two consecutive samples +// can straddle one refresh and be identical because THE CACHE DID NOT UPDATE — not because the fleet +// stopped moving. The window would then declare "settled" on stale data and sweep on a picture of the +// box from up to 10 s ago, which is a quieter version of the exact defect R-157 mechanism A is. +// +// Observed on guest 9201 on 2026-08-02: a container removed ~5 s before the window closed was still +// present in the sampled fleet, so the sweep logged "no boot-orphaned apps" for an app that had none. +// +// `RefreshStatus` is a cheap `docker ps`-based refresh of the in-memory map (the same one the +// scheduler runs every 10 s), so at most ~10 extra calls per boot. A refresh error is logged and the +// sample proceeds on whatever the map holds — degraded, but never silently: a boot window that cannot +// see docker is exactly when a stale verdict is most dangerous. func sampleBootFleet(mgr bootrecon.StackProvider) []bootFleetSample { + _ = mgr.RefreshStatus() stacksNow := mgr.GetStacks() out := make([]bootFleetSample, 0, len(stacksNow)) for _, s := range stacksNow {