boot window: sample REFRESHES first — a cached fleet made 'settled' meaningless
gates / gates (push) Successful in 9s

Found by live validation on 9201, not by review. GetStacks() is the Manager's
in-memory map refreshed by the scheduler every 10s; sampling it every 5s without
refreshing means two identical samples can mean the cache did not update rather
than that the fleet settled. A container removed ~5s before the window closed was
still in the sampled fleet and the sweep logged 'no boot-orphaned apps' for an app
that had none. sampleBootFleet now refreshes first; a refresh error degrades
rather than aborting the window.
This commit is contained in:
2026-08-02 20:17:12 +02:00
parent 582135f861
commit dcc3363d2f
2 changed files with 78 additions and 6 deletions
+56 -1
View File
@@ -27,6 +27,8 @@ type windowStacks struct {
calls int
starts map[string]int
onStart func(*windowStacks, string)
refreshes int
refreshErr error
// cycle makes the fleet NEVER settle: frames repeat forever instead of the last one sticking.
// Required by the budget test — with frames that eventually stop changing, the window terminates
// by SETTLING even with the budget removed, so the red-proof would not reach the hang it exists
@@ -47,7 +49,13 @@ func (w *windowStacks) GetStacks() []stacks.Stack {
return w.frames[i]
}
func (w *windowStacks) RefreshStatus() error { return nil }
func (w *windowStacks) RefreshStatus() error {
w.refreshes++
if w.refreshErr != nil {
return w.refreshErr
}
return nil
}
func (w *windowStacks) StartStack(name string) error {
if w.starts == nil {
@@ -309,3 +317,50 @@ func TestBootWindow_CommonCaseFitsInsideTheDeadAppGrace(t *testing.T) {
"between two docker events'", bootReconcileSample, bootReconcileStableFor)
}
}
// --- the sample must observe REALITY, not the Manager's cache ------------------------------------
func TestBootWindow_EverySampleRefreshesTheStatus(t *testing.T) {
// FOUND BY LIVE VALIDATION, not review. GetStacks() returns the Manager's in-memory map, which
// the scheduler refreshes on its own 10 s cadence. Sampling every 5 s WITHOUT refreshing means two
// consecutive samples can be identical because the cache did not update — so the window declares
// "settled" on stale data and sweeps on a picture of the box from up to 10 s ago. On 9201 a
// container removed ~5 s before the window closed was still in the sampled fleet, and the sweep
// logged "no boot-orphaned apps" for an app that had none.
//
// RED-PROOF: delete the `_ = mgr.RefreshStatus()` line from sampleBootFleet and this test fails
// with refreshes=0. Demonstrated in REPORT.md §4.
shrinkWindow(t, time.Millisecond, 3, 500*time.Millisecond)
captureSweep(t)
w := &windowStacks{frames: [][]stacks.Stack{{upStack("immich")}}}
runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0))
if w.refreshes < 3 {
t.Fatalf("the window refreshed %d time(s) for %d samples — every sample must observe reality, "+
"or 'settled' can mean 'the cache did not update'", w.refreshes, w.calls)
}
// calls includes ONE extra GetStacks from the captured sweep itself, which does not sample.
if w.refreshes != w.calls-1 {
t.Fatalf("refreshes=%d but samples=%d — each sample must refresh exactly once before reading",
w.refreshes, w.calls-1)
}
}
func TestBootWindow_RefreshErrorDoesNotStopTheWindow(t *testing.T) {
// A boot window that cannot reach docker is exactly when a stale verdict is most dangerous, but
// giving up entirely would leave the sweep un-run. Degrade, do not abort.
shrinkWindow(t, time.Millisecond, 2, 200*time.Millisecond)
seen := captureSweep(t)
w := &windowStacks{frames: [][]stacks.Stack{{upStack("immich")}}, refreshErr: errRefresh{}}
runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0))
if len(*seen) != 1 {
t.Fatalf("sweeps=%d, want 1 — a refresh error must not abort the window", len(*seen))
}
}
type errRefresh struct{}
func (errRefresh) Error() string { return "docker unreachable" }
+18 -1
View File
@@ -1406,8 +1406,25 @@ type bootFleetSample struct {
containers int
}
// sampleBootFleet returns the fleet snapshot, sorted, so two samples compare by equality.
// sampleBootFleet REFRESHES, then returns the fleet snapshot, sorted, so two samples compare by
// equality.
//
// THE REFRESH IS LOAD-BEARING, and it was found by live validation, not by review. `GetStacks()`
// returns the Manager's IN-MEMORY map, which the scheduler refreshes on its own 10 s cadence
// (`status-refresh`, main.go). Sampling it every 5 s without refreshing means two consecutive samples
// can straddle one refresh and be identical because THE CACHE DID NOT UPDATE — not because the fleet
// stopped moving. The window would then declare "settled" on stale data and sweep on a picture of the
// box from up to 10 s ago, which is a quieter version of the exact defect R-157 mechanism A is.
//
// Observed on guest 9201 on 2026-08-02: a container removed ~5 s before the window closed was still
// present in the sampled fleet, so the sweep logged "no boot-orphaned apps" for an app that had none.
//
// `RefreshStatus` is a cheap `docker ps`-based refresh of the in-memory map (the same one the
// scheduler runs every 10 s), so at most ~10 extra calls per boot. A refresh error is logged and the
// sample proceeds on whatever the map holds — degraded, but never silently: a boot window that cannot
// see docker is exactly when a stale verdict is most dangerous.
func sampleBootFleet(mgr bootrecon.StackProvider) []bootFleetSample {
_ = mgr.RefreshStatus()
stacksNow := mgr.GetStacks()
out := make([]bootFleetSample, 0, len(stacksNow))
for _, s := range stacksNow {