v0.190.0 — the boot settle window, both gates on intent, and R-171
gates / gates (push) Successful in 8s
gates / gates (push) Successful in 8s
R-171 (a regression v0.189.0 introduced, CONFIRMED on hardware before any fix was written). Replacing isBootOrphan's container-count term with recorded intent made a drive-gate-stopped app read as a boot orphan: the gate stops apps with `compose down` (zero containers) and never touches desired_state, because it is not the customer. Observed on 9201 with the drive held unmounted — the sweep found and started it, burned both attempts, and handed it to the dead-app alarm. The write hazard did not materialise (the unbound mountpoint is host-root-owned and the guest is unprivileged) but that protection is accidental and untested. New consumer-side seam bootrecon.StartGate, fail-safe (cannot determine ⇒ do not start), wired in main.go. The rule is not new: the API's startGatedByMissingDrive already refuses this; the sweep bypassed it. R-157 mechanism A. The sweep looked once at T+5s, deriving candidates from a fleet docker was still restoring — three of six hard resets. Now a settle-then- sweep window: sample every 5s, settled after 3 identical samples, sweep ONCE at the end; ends on settled or a 50s budget, and the log says which. The budget is 50s because settle+budget+one retry must stay under the 90s dead-app grace — a test rejected 60s at 95s. A window that overruns emits a LATE RECOVERY warn rather than the grace being widened to hide it. Widening the window made two more holders reachable, so the one gate covers all three: an absent drive, a quiesce, and an in-flight app-data operation — reusing quiesce.SuppressedStacks() and a new read-only AppStopGuard.HeldStacks(). R-170. shouldRecreateOnBoot now reads desired_state with the identical three-way table; absent keeps the old hasContainers behaviour exactly. Its comment argued for the container count and was rewritten. presentStable is untouched. The two gates' agreement is pinned from both sides against one fixture table. 27/27 packages green; 6 red-proofs observed FAIL then restored.
This commit is contained in:
@@ -0,0 +1,311 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/bootrecon"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
|
||||
// R-157 mechanism A — the sweep that looked once.
|
||||
//
|
||||
// TIMING IS NOT TESTED BY SLEEPING (§10). The window's constants are package vars, so each test
|
||||
// shrinks them to sub-millisecond values: the CONTRACT under test is "how many samples, and what
|
||||
// ends the window", not "how long a second is". A test that waited real seconds would be slow,
|
||||
// flaky, and would still not prove the contract.
|
||||
|
||||
// windowStacks is a StackProvider whose fleet CHANGES over successive GetStacks() calls — which is
|
||||
// the whole point: the pre-v0.190.0 sweep sampled once and could not see a late settler.
|
||||
type windowStacks struct {
|
||||
// frames is the fleet as seen on each successive GetStacks() call; the last frame repeats.
|
||||
frames [][]stacks.Stack
|
||||
calls int
|
||||
starts map[string]int
|
||||
onStart func(*windowStacks, string)
|
||||
// cycle makes the fleet NEVER settle: frames repeat forever instead of the last one sticking.
|
||||
// Required by the budget test — with frames that eventually stop changing, the window terminates
|
||||
// by SETTLING even with the budget removed, so the red-proof would not reach the hang it exists
|
||||
// to demonstrate.
|
||||
cycle bool
|
||||
}
|
||||
|
||||
func (w *windowStacks) GetStacks() []stacks.Stack {
|
||||
i := w.calls
|
||||
w.calls++
|
||||
if i >= len(w.frames) {
|
||||
if w.cycle {
|
||||
i = i % len(w.frames)
|
||||
} else {
|
||||
i = len(w.frames) - 1
|
||||
}
|
||||
}
|
||||
return w.frames[i]
|
||||
}
|
||||
|
||||
func (w *windowStacks) RefreshStatus() error { return nil }
|
||||
|
||||
func (w *windowStacks) StartStack(name string) error {
|
||||
if w.starts == nil {
|
||||
w.starts = map[string]int{}
|
||||
}
|
||||
w.starts[name]++
|
||||
if w.onStart != nil {
|
||||
w.onStart(w, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shrinkWindow makes the window fast and deterministic, and restores the shipped values after.
|
||||
func shrinkWindow(t *testing.T, sample time.Duration, stableFor int, budget time.Duration) {
|
||||
t.Helper()
|
||||
os, ost, ob, osettle := bootReconcileSample, bootReconcileStableFor, bootReconcileBudget, bootReconcileSettle
|
||||
t.Cleanup(func() {
|
||||
bootReconcileSample, bootReconcileStableFor, bootReconcileBudget, bootReconcileSettle = os, ost, ob, osettle
|
||||
})
|
||||
bootReconcileSample, bootReconcileStableFor, bootReconcileBudget = sample, stableFor, budget
|
||||
bootReconcileSettle = time.Millisecond
|
||||
}
|
||||
|
||||
// captureSweep replaces the sweep with a recorder and returns the fleet it was handed.
|
||||
func captureSweep(t *testing.T) *[][]stacks.Stack {
|
||||
t.Helper()
|
||||
orig := bootReconcileFn
|
||||
t.Cleanup(func() { bootReconcileFn = orig })
|
||||
var seen [][]stacks.Stack
|
||||
bootReconcileFn = func(_ context.Context, mgr bootrecon.StackProvider, _ *log.Logger) bootrecon.Result {
|
||||
seen = append(seen, mgr.GetStacks())
|
||||
return bootrecon.Result{}
|
||||
}
|
||||
return &seen
|
||||
}
|
||||
|
||||
func upStack(name string) stacks.Stack {
|
||||
return stacks.Stack{
|
||||
Name: name, Deployed: true, State: stacks.StateRunning,
|
||||
Containers: []stacks.ContainerInfo{{Name: name, State: stacks.StateRunning}},
|
||||
AppConfig: &stacks.AppConfig{Deployed: true, DesiredState: stacks.DesiredStateRunning},
|
||||
}
|
||||
}
|
||||
|
||||
// settlingLate is the R-157-A shape: at T+5s the app is still `starting` with its containers coming
|
||||
// up, and it only comes to rest in a DOWN state later.
|
||||
func settlingLate(name string) stacks.Stack {
|
||||
return stacks.Stack{
|
||||
Name: name, Deployed: true, State: stacks.StateStarting,
|
||||
Containers: []stacks.ContainerInfo{{Name: name, State: stacks.StateStarting}},
|
||||
AppConfig: &stacks.AppConfig{Deployed: true, DesiredState: stacks.DesiredStateRunning},
|
||||
}
|
||||
}
|
||||
|
||||
func settledDown(name string) stacks.Stack {
|
||||
return stacks.Stack{
|
||||
Name: name, Deployed: true, State: stacks.StateExited,
|
||||
Containers: []stacks.ContainerInfo{{Name: name, State: stacks.StateExited}},
|
||||
AppConfig: &stacks.AppConfig{Deployed: true, DesiredState: stacks.DesiredStateRunning},
|
||||
}
|
||||
}
|
||||
|
||||
// --- Group A / Scenario B — a late settler IS swept -----------------------------------------------
|
||||
|
||||
func TestBootWindow_LateSettlerIsSweptOnASettledFleet(t *testing.T) {
|
||||
// The fleet is still moving for the first frames and settles only later. The sweep must run
|
||||
// AFTER it settles and must be handed the SETTLED fleet — because the pre-v0.190.0 defect was a
|
||||
// candidate set derived from a fleet that had not finished moving.
|
||||
//
|
||||
// RED-PROOF: restore the single-sweep shape (delete the sampling loop so runBootReconcile calls
|
||||
// bootReconcileFn straight after the settle delay) and this test fails — the sweep is handed the
|
||||
// `starting` frame, in which the app is not a down-state candidate at all.
|
||||
// Demonstrated in REPORT.md §4.
|
||||
shrinkWindow(t, time.Millisecond, 2, 500*time.Millisecond)
|
||||
seen := captureSweep(t)
|
||||
|
||||
w := &windowStacks{frames: [][]stacks.Stack{
|
||||
{settlingLate("immich")}, // T+5s: still coming up
|
||||
{settlingLate("immich")},
|
||||
{settledDown("immich")}, // settles into a down state only now
|
||||
{settledDown("immich")},
|
||||
{settledDown("immich")},
|
||||
}}
|
||||
|
||||
runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0))
|
||||
|
||||
if len(*seen) != 1 {
|
||||
t.Fatalf("the sweep ran %d times, want exactly 1 — the window samples, it does not sweep per sample", len(*seen))
|
||||
}
|
||||
got := (*seen)[0]
|
||||
if len(got) != 1 || got[0].State != stacks.StateExited {
|
||||
t.Fatalf("the sweep was handed state=%v, want the SETTLED (exited) fleet — a candidate set "+
|
||||
"derived from a still-moving fleet is exactly the R-157 mechanism-A defect", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootWindow_SweepRunsExactlyOnceEvenOnAQuietBoot(t *testing.T) {
|
||||
shrinkWindow(t, time.Millisecond, 2, 500*time.Millisecond)
|
||||
seen := captureSweep(t)
|
||||
w := &windowStacks{frames: [][]stacks.Stack{{upStack("bookstack")}}}
|
||||
|
||||
runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0))
|
||||
|
||||
if len(*seen) != 1 {
|
||||
t.Fatalf("sweeps=%d, want exactly 1 on a quiet boot", len(*seen))
|
||||
}
|
||||
}
|
||||
|
||||
// --- Group B / Scenario C — the window TERMINATES -------------------------------------------------
|
||||
|
||||
func TestBootWindow_BudgetEndsAForeverChangingFleet(t *testing.T) {
|
||||
// A fleet that never stops changing must not sample forever. The budget ends it, the sweep runs
|
||||
// once anyway (a churning box is exactly the box that needs it), and the log SAYS the budget
|
||||
// ended it — "settled and found nothing" and "ran out of time" are different facts.
|
||||
//
|
||||
// RED-PROOF: remove the `time.Since(started) < bootReconcileBudget` loop condition and this test
|
||||
// hangs — the unbounded-loop shape §5 bans. Demonstrated in REPORT.md §4 (observed as a timeout).
|
||||
shrinkWindow(t, time.Millisecond, 3, 30*time.Millisecond)
|
||||
seen := captureSweep(t)
|
||||
var buf strings.Builder
|
||||
|
||||
// Every frame differs, so `stable` can never reach stableFor.
|
||||
frames := make([][]stacks.Stack, 0, 200)
|
||||
for i := 0; i < 200; i++ {
|
||||
s := upStack("immich")
|
||||
s.Containers = make([]stacks.ContainerInfo, i%7) // container count changes every sample
|
||||
frames = append(frames, []stacks.Stack{s})
|
||||
}
|
||||
w := &windowStacks{frames: frames, cycle: true}
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
runBootReconcile(context.Background(), w, log.New(&buf, "", 0))
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("runBootReconcile did not terminate on a forever-changing fleet — this is the " +
|
||||
"unbounded restart-loop shape the package's own boundary forbids")
|
||||
}
|
||||
|
||||
if len(*seen) != 1 {
|
||||
t.Fatalf("sweeps=%d, want exactly 1 after the budget expired", len(*seen))
|
||||
}
|
||||
if out := buf.String(); !strings.Contains(out, "budget") {
|
||||
t.Fatalf("the log does not say the BUDGET ended the window, so a churning boot reads like a "+
|
||||
"quiet one:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootWindow_SettledPathSaysSettled(t *testing.T) {
|
||||
shrinkWindow(t, time.Millisecond, 2, 500*time.Millisecond)
|
||||
captureSweep(t)
|
||||
var buf strings.Builder
|
||||
w := &windowStacks{frames: [][]stacks.Stack{{upStack("docmost")}}}
|
||||
|
||||
runBootReconcile(context.Background(), w, log.New(&buf, "", 0))
|
||||
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "settled") {
|
||||
t.Fatalf("a settled window must say so — otherwise it is indistinguishable from a budget "+
|
||||
"expiry:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "budget") {
|
||||
t.Fatalf("a settled window must NOT claim the budget ended it:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootWindow_CancelledContextStopsImmediately(t *testing.T) {
|
||||
shrinkWindow(t, time.Millisecond, 3, time.Second)
|
||||
seen := captureSweep(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
runBootReconcile(ctx, &windowStacks{frames: [][]stacks.Stack{{upStack("x")}}}, log.New(io.Discard, "", 0))
|
||||
if len(*seen) != 0 {
|
||||
t.Fatalf("the sweep ran %d times on a cancelled context, want 0", len(*seen))
|
||||
}
|
||||
}
|
||||
|
||||
// --- Group C / Scenario D — a customer's Stop survives the WIDENED window -------------------------
|
||||
|
||||
func TestBootWindow_CustomerStoppedAppSurvivesEveryPass(t *testing.T) {
|
||||
// THE REGRESSION THIS TASK COULD INTRODUCE. A longer window means more chances to resurrect an
|
||||
// app the customer deliberately stopped. It must survive the whole window — this drives the REAL
|
||||
// bootrecon sweep (not the captured stub), so the desired-state check is genuinely exercised.
|
||||
//
|
||||
// RED-PROOF: drop the DesiredStateStopped branch from isBootOrphan (make it fall through to the
|
||||
// running case) and this test fails with a start count of 1. Demonstrated in REPORT.md §4.
|
||||
shrinkWindow(t, time.Millisecond, 2, 200*time.Millisecond)
|
||||
|
||||
stopped := stacks.Stack{
|
||||
Name: "nextcloud", Deployed: true, State: stacks.StateStopped, Containers: nil,
|
||||
AppConfig: &stacks.AppConfig{Deployed: true, DesiredState: stacks.DesiredStateStopped},
|
||||
}
|
||||
// The fleet churns around it, so the window runs many passes before settling.
|
||||
frames := [][]stacks.Stack{
|
||||
{stopped, settlingLate("immich")},
|
||||
{stopped, settlingLate("immich")},
|
||||
{stopped, settledDown("immich")},
|
||||
{stopped, upStack("immich")},
|
||||
{stopped, upStack("immich")},
|
||||
{stopped, upStack("immich")},
|
||||
}
|
||||
w := &windowStacks{frames: frames, onStart: func(w *windowStacks, _ string) {}}
|
||||
|
||||
runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0))
|
||||
|
||||
if n := w.starts["nextcloud"]; n != 0 {
|
||||
t.Fatalf("the customer-stopped app was started %d time(s) by the widened window — this is the "+
|
||||
"regression a longer window makes possible and it is the worst outcome available here", n)
|
||||
}
|
||||
}
|
||||
|
||||
// --- §8.3 — a late recovery is REPORTED, never hidden ---------------------------------------------
|
||||
|
||||
func TestRecordLateRecovery_WarnsWhenTheGraceHasAlreadyExpired(t *testing.T) {
|
||||
var buf strings.Builder
|
||||
lg := log.New(&buf, "", 0)
|
||||
// started far enough back that settle + elapsed exceeds the 90 s grace
|
||||
recordLateRecovery(lg, time.Now().Add(-(deadAppBootGrace + 10*time.Second)), bootrecon.Result{Recovered: []string{"immich"}})
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "LATE RECOVERY") || !strings.Contains(out, "immich") {
|
||||
t.Fatalf("a recovery past the dead-app grace must be reported by name — otherwise a stale "+
|
||||
"alarm stands with no counter-evidence (§8.3):\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordLateRecovery_SilentInsideTheGrace(t *testing.T) {
|
||||
var buf strings.Builder
|
||||
recordLateRecovery(log.New(&buf, "", 0), time.Now(), bootrecon.Result{Recovered: []string{"immich"}})
|
||||
if buf.Len() != 0 {
|
||||
t.Fatalf("a recovery INSIDE the grace must stay silent — that is what makes a successful "+
|
||||
"recovery invisible to the customer:\n%s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordLateRecovery_SilentWhenNothingRecovered(t *testing.T) {
|
||||
var buf strings.Builder
|
||||
recordLateRecovery(log.New(&buf, "", 0), time.Now().Add(-time.Hour), bootrecon.Result{})
|
||||
if buf.Len() != 0 {
|
||||
t.Fatalf("nothing was recovered, so there is nothing late to report:\n%s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// --- The window's constants must fit the grace they are justified against -------------------------
|
||||
|
||||
func TestBootWindow_CommonCaseFitsInsideTheDeadAppGrace(t *testing.T) {
|
||||
// The comment on the window constants justifies them against deadAppBootGrace. A comment
|
||||
// asserting an invariant needs a test pinning it, or it is a wish.
|
||||
common := bootReconcileSettle + bootReconcileBudget + bootrecon.DefaultRetryDelay
|
||||
if common > deadAppBootGrace {
|
||||
t.Fatalf("settle(%s) + budget(%s) + one retry(%s) = %s exceeds the %s dead-app grace — the "+
|
||||
"COMMON case must stay silent, or every slow boot alerts",
|
||||
bootReconcileSettle, bootReconcileBudget, bootrecon.DefaultRetryDelay, common, deadAppBootGrace)
|
||||
}
|
||||
if bootReconcileSample <= 0 || bootReconcileStableFor < 2 {
|
||||
t.Fatalf("sample=%s stableFor=%d — one sample cannot distinguish 'settled' from 'sampled "+
|
||||
"between two docker events'", bootReconcileSample, bootReconcileStableFor)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user