boot window: sample REFRESHES first — a cached fleet made 'settled' meaningless
gates / gates (push) Successful in 9s
gates / gates (push) Successful in 9s
Found by live validation on 9201, not by review. GetStacks() is the Manager's in-memory map refreshed by the scheduler every 10s; sampling it every 5s without refreshing means two identical samples can mean the cache did not update rather than that the fleet settled. A container removed ~5s before the window closed was still in the sampled fleet and the sweep logged 'no boot-orphaned apps' for an app that had none. sampleBootFleet now refreshes first; a refresh error degrades rather than aborting the window.
This commit is contained in:
@@ -23,10 +23,12 @@ import (
|
||||
// the whole point: the pre-v0.190.0 sweep sampled once and could not see a late settler.
|
||||
type windowStacks struct {
|
||||
// frames is the fleet as seen on each successive GetStacks() call; the last frame repeats.
|
||||
frames [][]stacks.Stack
|
||||
calls int
|
||||
starts map[string]int
|
||||
onStart func(*windowStacks, string)
|
||||
frames [][]stacks.Stack
|
||||
calls int
|
||||
starts map[string]int
|
||||
onStart func(*windowStacks, string)
|
||||
refreshes int
|
||||
refreshErr error
|
||||
// cycle makes the fleet NEVER settle: frames repeat forever instead of the last one sticking.
|
||||
// Required by the budget test — with frames that eventually stop changing, the window terminates
|
||||
// by SETTLING even with the budget removed, so the red-proof would not reach the hang it exists
|
||||
@@ -47,7 +49,13 @@ func (w *windowStacks) GetStacks() []stacks.Stack {
|
||||
return w.frames[i]
|
||||
}
|
||||
|
||||
func (w *windowStacks) RefreshStatus() error { return nil }
|
||||
func (w *windowStacks) RefreshStatus() error {
|
||||
w.refreshes++
|
||||
if w.refreshErr != nil {
|
||||
return w.refreshErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *windowStacks) StartStack(name string) error {
|
||||
if w.starts == nil {
|
||||
@@ -309,3 +317,50 @@ func TestBootWindow_CommonCaseFitsInsideTheDeadAppGrace(t *testing.T) {
|
||||
"between two docker events'", bootReconcileSample, bootReconcileStableFor)
|
||||
}
|
||||
}
|
||||
|
||||
// --- the sample must observe REALITY, not the Manager's cache ------------------------------------
|
||||
|
||||
func TestBootWindow_EverySampleRefreshesTheStatus(t *testing.T) {
|
||||
// FOUND BY LIVE VALIDATION, not review. GetStacks() returns the Manager's in-memory map, which
|
||||
// the scheduler refreshes on its own 10 s cadence. Sampling every 5 s WITHOUT refreshing means two
|
||||
// consecutive samples can be identical because the cache did not update — so the window declares
|
||||
// "settled" on stale data and sweeps on a picture of the box from up to 10 s ago. On 9201 a
|
||||
// container removed ~5 s before the window closed was still in the sampled fleet, and the sweep
|
||||
// logged "no boot-orphaned apps" for an app that had none.
|
||||
//
|
||||
// RED-PROOF: delete the `_ = mgr.RefreshStatus()` line from sampleBootFleet and this test fails
|
||||
// with refreshes=0. Demonstrated in REPORT.md §4.
|
||||
shrinkWindow(t, time.Millisecond, 3, 500*time.Millisecond)
|
||||
captureSweep(t)
|
||||
w := &windowStacks{frames: [][]stacks.Stack{{upStack("immich")}}}
|
||||
|
||||
runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0))
|
||||
|
||||
if w.refreshes < 3 {
|
||||
t.Fatalf("the window refreshed %d time(s) for %d samples — every sample must observe reality, "+
|
||||
"or 'settled' can mean 'the cache did not update'", w.refreshes, w.calls)
|
||||
}
|
||||
// calls includes ONE extra GetStacks from the captured sweep itself, which does not sample.
|
||||
if w.refreshes != w.calls-1 {
|
||||
t.Fatalf("refreshes=%d but samples=%d — each sample must refresh exactly once before reading",
|
||||
w.refreshes, w.calls-1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootWindow_RefreshErrorDoesNotStopTheWindow(t *testing.T) {
|
||||
// A boot window that cannot reach docker is exactly when a stale verdict is most dangerous, but
|
||||
// giving up entirely would leave the sweep un-run. Degrade, do not abort.
|
||||
shrinkWindow(t, time.Millisecond, 2, 200*time.Millisecond)
|
||||
seen := captureSweep(t)
|
||||
w := &windowStacks{frames: [][]stacks.Stack{{upStack("immich")}}, refreshErr: errRefresh{}}
|
||||
|
||||
runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0))
|
||||
|
||||
if len(*seen) != 1 {
|
||||
t.Fatalf("sweeps=%d, want 1 — a refresh error must not abort the window", len(*seen))
|
||||
}
|
||||
}
|
||||
|
||||
type errRefresh struct{}
|
||||
|
||||
func (errRefresh) Error() string { return "docker unreachable" }
|
||||
|
||||
Reference in New Issue
Block a user