boot window: sample REFRESHES first — a cached fleet made 'settled' meaningless
gates / gates (push) Successful in 9s

Found by live validation on 9201, not by review. GetStacks() is the Manager's
in-memory map refreshed by the scheduler every 10s; sampling it every 5s without
refreshing means two identical samples can mean the cache did not update rather
than that the fleet settled. A container removed ~5s before the window closed was
still in the sampled fleet and the sweep logged 'no boot-orphaned apps' for an app
that had none. sampleBootFleet now refreshes first; a refresh error degrades
rather than aborting the window.
This commit is contained in:
2026-08-02 20:17:12 +02:00
parent 582135f861
commit dcc3363d2f
2 changed files with 78 additions and 6 deletions
+60 -5
View File
@@ -23,10 +23,12 @@ import (
// the whole point: the pre-v0.190.0 sweep sampled once and could not see a late settler.
type windowStacks struct {
// frames is the fleet as seen on each successive GetStacks() call; the last frame repeats.
frames [][]stacks.Stack
calls int
starts map[string]int
onStart func(*windowStacks, string)
frames [][]stacks.Stack
calls int
starts map[string]int
onStart func(*windowStacks, string)
refreshes int
refreshErr error
// cycle makes the fleet NEVER settle: frames repeat forever instead of the last one sticking.
// Required by the budget test — with frames that eventually stop changing, the window terminates
// by SETTLING even with the budget removed, so the red-proof would not reach the hang it exists
@@ -47,7 +49,13 @@ func (w *windowStacks) GetStacks() []stacks.Stack {
return w.frames[i]
}
func (w *windowStacks) RefreshStatus() error { return nil }
func (w *windowStacks) RefreshStatus() error {
w.refreshes++
if w.refreshErr != nil {
return w.refreshErr
}
return nil
}
func (w *windowStacks) StartStack(name string) error {
if w.starts == nil {
@@ -309,3 +317,50 @@ func TestBootWindow_CommonCaseFitsInsideTheDeadAppGrace(t *testing.T) {
"between two docker events'", bootReconcileSample, bootReconcileStableFor)
}
}
// --- the sample must observe REALITY, not the Manager's cache ------------------------------------
func TestBootWindow_EverySampleRefreshesTheStatus(t *testing.T) {
// FOUND BY LIVE VALIDATION, not review. GetStacks() returns the Manager's in-memory map, which
// the scheduler refreshes on its own 10 s cadence. Sampling every 5 s WITHOUT refreshing means two
// consecutive samples can be identical because the cache did not update — so the window declares
// "settled" on stale data and sweeps on a picture of the box from up to 10 s ago. On 9201 a
// container removed ~5 s before the window closed was still in the sampled fleet, and the sweep
// logged "no boot-orphaned apps" for an app that had none.
//
// RED-PROOF: delete the `_ = mgr.RefreshStatus()` line from sampleBootFleet and this test fails
// with refreshes=0. Demonstrated in REPORT.md §4.
shrinkWindow(t, time.Millisecond, 3, 500*time.Millisecond)
captureSweep(t)
w := &windowStacks{frames: [][]stacks.Stack{{upStack("immich")}}}
runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0))
if w.refreshes < 3 {
t.Fatalf("the window refreshed %d time(s) for %d samples — every sample must observe reality, "+
"or 'settled' can mean 'the cache did not update'", w.refreshes, w.calls)
}
// calls includes ONE extra GetStacks from the captured sweep itself, which does not sample.
if w.refreshes != w.calls-1 {
t.Fatalf("refreshes=%d but samples=%d — each sample must refresh exactly once before reading",
w.refreshes, w.calls-1)
}
}
func TestBootWindow_RefreshErrorDoesNotStopTheWindow(t *testing.T) {
// A boot window that cannot reach docker is exactly when a stale verdict is most dangerous, but
// giving up entirely would leave the sweep un-run. Degrade, do not abort.
shrinkWindow(t, time.Millisecond, 2, 200*time.Millisecond)
seen := captureSweep(t)
w := &windowStacks{frames: [][]stacks.Stack{{upStack("immich")}}, refreshErr: errRefresh{}}
runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0))
if len(*seen) != 1 {
t.Fatalf("sweeps=%d, want 1 — a refresh error must not abort the window", len(*seen))
}
}
type errRefresh struct{}
func (errRefresh) Error() string { return "docker unreachable" }