v0.190.0 — the boot settle window, both gates on intent, and R-171

R-171 (a regression v0.189.0 introduced, CONFIRMED on hardware before any fix was written). Replacing isBootOrphan's container-count term with recorded intent made a drive-gate-stopped app read as a boot orphan: the gate stops apps with `compose down` (zero containers) and never touches desired_state, because it is not the customer. Observed on 9201 with the drive held unmounted — the sweep found and started it, burned both attempts, and handed it to the dead-app alarm. The write hazard did not materialise (the unbound mountpoint is host-root-owned and the guest is unprivileged) but that protection is accidental and untested. New consumer-side seam bootrecon.StartGate, fail-safe (cannot determine ⇒ do not start), wired in main.go. The rule is not new: the API's startGatedByMissingDrive already refuses this; the sweep bypassed it. R-157 mechanism A. The sweep looked once at T+5s, deriving candidates from a fleet docker was still restoring — three of six hard resets. Now a settle-then- sweep window: sample every 5s, settled after 3 identical samples, sweep ONCE at the end; ends on settled or a 50s budget, and the log says which. The budget is 50s because settle+budget+one retry must stay under the 90s dead-app grace — a test rejected 60s at 95s. A window that overruns emits a LATE RECOVERY warn rather than the grace being widened to hide it. Widening the window made two more holders reachable, so the one gate covers all three: an absent drive, a quiesce, and an in-flight app-data operation — reusing quiesce.SuppressedStacks() and a new read-only AppStopGuard.HeldStacks(). R-170. shouldRecreateOnBoot now reads desired_state with the identical three-way table; absent keeps the old hasContainers behaviour exactly. Its comment argued for the container count and was rewritten. presentStable is untouched. The two gates' agreement is pinned from both sides against one fixture table. 27/27 packages green; 6 red-proofs observed FAIL then restored.
2026-08-02 19:56:20 +02:00
parent 3446609420
commit 582135f861
13 changed files with 1272 additions and 43 deletions
@@ -166,3 +166,67 @@ func TestMainReportsTheInterruptedOperation(t *testing.T) {
 			"healthy boot would page the operator about a backup that was never interrupted")
 	}
 }
+
+// --- R-171 seam: the boot drive gate must be WIRED in production -------------------------------
+
+// TestMainWiresBootDriveGate is the Group-H seam test. An unwired drive gate is not a crash — it is
+// SILENTLY the pre-v0.190.0 behaviour, which started apps onto absent drives (observed live,
+// audits/DIAG-bootrecon-drive-absent-2026-08-02.md). Every behavioural test in internal/bootrecon
+// still passes with the wiring gone, which is exactly the hole this walks the AST to close.
+//
+// AST, not strings.Contains: a commented-out call still contains the string — the distinction that
+// made a previous version of this project's own seam test pass its red-proof (2026-07-21).
+func TestMainWiresBootDriveGate(t *testing.T) {
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "main.go", nil, 0)
+	if err != nil {
+		t.Fatalf("parse main.go: %v", err)
+	}
+
+	// (a) the settings handle the gate reads is assigned somewhere in main().
+	assigned := false
+	for _, name := range assignedIdentsIn(mainBody(t)) {
+		if name == "bootDriveSettings" {
+			assigned = true
+		}
+	}
+	if !assigned {
+		t.Fatal("func main() no longer assigns bootDriveSettings — the boot drive gate would read a " +
+			"nil settings handle and could not see a disconnected drive")
+	}
+
+	// (b) SetDriveGate is actually called where the reconciler is constructed.
+	called := false
+	ast.Inspect(f, func(n ast.Node) bool {
+		call, ok := n.(*ast.CallExpr)
+		if !ok {
+			return true
+		}
+		if sel, ok := call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "SetDriveGate" {
+			called = true
+		}
+		return true
+	})
+	if !called {
+		t.Fatal("main.go no longer calls SetDriveGate on the boot reconciler — the sweep would start " +
+			"apps whose data drive is absent (R-171, a regression observed live on 2026-08-02)")
+	}
+}
+
+// assignedIdentsIn returns the names assigned to in a block (plain `=` and `:=`).
+func assignedIdentsIn(body *ast.BlockStmt) []string {
+	var names []string
+	ast.Inspect(body, func(n ast.Node) bool {
+		as, ok := n.(*ast.AssignStmt)
+		if !ok {
+			return true
+		}
+		for _, lhs := range as.Lhs {
+			if id, ok := lhs.(*ast.Ident); ok {
+				names = append(names, id.Name)
+			}
+		}
+		return true
+	})
+	return names
+}
@@ -0,0 +1,311 @@
+package main
+
+import (
+	"context"
+	"io"
+	"log"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/bootrecon"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
+)
+
+// R-157 mechanism A — the sweep that looked once.
+//
+// TIMING IS NOT TESTED BY SLEEPING (§10). The window's constants are package vars, so each test
+// shrinks them to sub-millisecond values: the CONTRACT under test is "how many samples, and what
+// ends the window", not "how long a second is". A test that waited real seconds would be slow,
+// flaky, and would still not prove the contract.
+
+// windowStacks is a StackProvider whose fleet CHANGES over successive GetStacks() calls — which is
+// the whole point: the pre-v0.190.0 sweep sampled once and could not see a late settler.
+type windowStacks struct {
+	// frames is the fleet as seen on each successive GetStacks() call; the last frame repeats.
+	frames  [][]stacks.Stack
+	calls   int
+	starts  map[string]int
+	onStart func(*windowStacks, string)
+	// cycle makes the fleet NEVER settle: frames repeat forever instead of the last one sticking.
+	// Required by the budget test — with frames that eventually stop changing, the window terminates
+	// by SETTLING even with the budget removed, so the red-proof would not reach the hang it exists
+	// to demonstrate.
+	cycle bool
+}
+
+func (w *windowStacks) GetStacks() []stacks.Stack {
+	i := w.calls
+	w.calls++
+	if i >= len(w.frames) {
+		if w.cycle {
+			i = i % len(w.frames)
+		} else {
+			i = len(w.frames) - 1
+		}
+	}
+	return w.frames[i]
+}
+
+func (w *windowStacks) RefreshStatus() error { return nil }
+
+func (w *windowStacks) StartStack(name string) error {
+	if w.starts == nil {
+		w.starts = map[string]int{}
+	}
+	w.starts[name]++
+	if w.onStart != nil {
+		w.onStart(w, name)
+	}
+	return nil
+}
+
+// shrinkWindow makes the window fast and deterministic, and restores the shipped values after.
+func shrinkWindow(t *testing.T, sample time.Duration, stableFor int, budget time.Duration) {
+	t.Helper()
+	os, ost, ob, osettle := bootReconcileSample, bootReconcileStableFor, bootReconcileBudget, bootReconcileSettle
+	t.Cleanup(func() {
+		bootReconcileSample, bootReconcileStableFor, bootReconcileBudget, bootReconcileSettle = os, ost, ob, osettle
+	})
+	bootReconcileSample, bootReconcileStableFor, bootReconcileBudget = sample, stableFor, budget
+	bootReconcileSettle = time.Millisecond
+}
+
+// captureSweep replaces the sweep with a recorder and returns the fleet it was handed.
+func captureSweep(t *testing.T) *[][]stacks.Stack {
+	t.Helper()
+	orig := bootReconcileFn
+	t.Cleanup(func() { bootReconcileFn = orig })
+	var seen [][]stacks.Stack
+	bootReconcileFn = func(_ context.Context, mgr bootrecon.StackProvider, _ *log.Logger) bootrecon.Result {
+		seen = append(seen, mgr.GetStacks())
+		return bootrecon.Result{}
+	}
+	return &seen
+}
+
+func upStack(name string) stacks.Stack {
+	return stacks.Stack{
+		Name: name, Deployed: true, State: stacks.StateRunning,
+		Containers: []stacks.ContainerInfo{{Name: name, State: stacks.StateRunning}},
+		AppConfig:  &stacks.AppConfig{Deployed: true, DesiredState: stacks.DesiredStateRunning},
+	}
+}
+
+// settlingLate is the R-157-A shape: at T+5s the app is still `starting` with its containers coming
+// up, and it only comes to rest in a DOWN state later.
+func settlingLate(name string) stacks.Stack {
+	return stacks.Stack{
+		Name: name, Deployed: true, State: stacks.StateStarting,
+		Containers: []stacks.ContainerInfo{{Name: name, State: stacks.StateStarting}},
+		AppConfig:  &stacks.AppConfig{Deployed: true, DesiredState: stacks.DesiredStateRunning},
+	}
+}
+
+func settledDown(name string) stacks.Stack {
+	return stacks.Stack{
+		Name: name, Deployed: true, State: stacks.StateExited,
+		Containers: []stacks.ContainerInfo{{Name: name, State: stacks.StateExited}},
+		AppConfig:  &stacks.AppConfig{Deployed: true, DesiredState: stacks.DesiredStateRunning},
+	}
+}
+
+// --- Group A / Scenario B — a late settler IS swept -----------------------------------------------
+
+func TestBootWindow_LateSettlerIsSweptOnASettledFleet(t *testing.T) {
+	// The fleet is still moving for the first frames and settles only later. The sweep must run
+	// AFTER it settles and must be handed the SETTLED fleet — because the pre-v0.190.0 defect was a
+	// candidate set derived from a fleet that had not finished moving.
+	//
+	// RED-PROOF: restore the single-sweep shape (delete the sampling loop so runBootReconcile calls
+	// bootReconcileFn straight after the settle delay) and this test fails — the sweep is handed the
+	// `starting` frame, in which the app is not a down-state candidate at all.
+	// Demonstrated in REPORT.md §4.
+	shrinkWindow(t, time.Millisecond, 2, 500*time.Millisecond)
+	seen := captureSweep(t)
+
+	w := &windowStacks{frames: [][]stacks.Stack{
+		{settlingLate("immich")}, // T+5s: still coming up
+		{settlingLate("immich")},
+		{settledDown("immich")}, // settles into a down state only now
+		{settledDown("immich")},
+		{settledDown("immich")},
+	}}
+
+	runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0))
+
+	if len(*seen) != 1 {
+		t.Fatalf("the sweep ran %d times, want exactly 1 — the window samples, it does not sweep per sample", len(*seen))
+	}
+	got := (*seen)[0]
+	if len(got) != 1 || got[0].State != stacks.StateExited {
+		t.Fatalf("the sweep was handed state=%v, want the SETTLED (exited) fleet — a candidate set "+
+			"derived from a still-moving fleet is exactly the R-157 mechanism-A defect", got)
+	}
+}
+
+func TestBootWindow_SweepRunsExactlyOnceEvenOnAQuietBoot(t *testing.T) {
+	shrinkWindow(t, time.Millisecond, 2, 500*time.Millisecond)
+	seen := captureSweep(t)
+	w := &windowStacks{frames: [][]stacks.Stack{{upStack("bookstack")}}}
+
+	runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0))
+
+	if len(*seen) != 1 {
+		t.Fatalf("sweeps=%d, want exactly 1 on a quiet boot", len(*seen))
+	}
+}
+
+// --- Group B / Scenario C — the window TERMINATES -------------------------------------------------
+
+func TestBootWindow_BudgetEndsAForeverChangingFleet(t *testing.T) {
+	// A fleet that never stops changing must not sample forever. The budget ends it, the sweep runs
+	// once anyway (a churning box is exactly the box that needs it), and the log SAYS the budget
+	// ended it — "settled and found nothing" and "ran out of time" are different facts.
+	//
+	// RED-PROOF: remove the `time.Since(started) < bootReconcileBudget` loop condition and this test
+	// hangs — the unbounded-loop shape §5 bans. Demonstrated in REPORT.md §4 (observed as a timeout).
+	shrinkWindow(t, time.Millisecond, 3, 30*time.Millisecond)
+	seen := captureSweep(t)
+	var buf strings.Builder
+
+	// Every frame differs, so `stable` can never reach stableFor.
+	frames := make([][]stacks.Stack, 0, 200)
+	for i := 0; i < 200; i++ {
+		s := upStack("immich")
+		s.Containers = make([]stacks.ContainerInfo, i%7) // container count changes every sample
+		frames = append(frames, []stacks.Stack{s})
+	}
+	w := &windowStacks{frames: frames, cycle: true}
+
+	done := make(chan struct{})
+	go func() {
+		runBootReconcile(context.Background(), w, log.New(&buf, "", 0))
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("runBootReconcile did not terminate on a forever-changing fleet — this is the " +
+			"unbounded restart-loop shape the package's own boundary forbids")
+	}
+
+	if len(*seen) != 1 {
+		t.Fatalf("sweeps=%d, want exactly 1 after the budget expired", len(*seen))
+	}
+	if out := buf.String(); !strings.Contains(out, "budget") {
+		t.Fatalf("the log does not say the BUDGET ended the window, so a churning boot reads like a "+
+			"quiet one:\n%s", out)
+	}
+}
+
+func TestBootWindow_SettledPathSaysSettled(t *testing.T) {
+	shrinkWindow(t, time.Millisecond, 2, 500*time.Millisecond)
+	captureSweep(t)
+	var buf strings.Builder
+	w := &windowStacks{frames: [][]stacks.Stack{{upStack("docmost")}}}
+
+	runBootReconcile(context.Background(), w, log.New(&buf, "", 0))
+
+	out := buf.String()
+	if !strings.Contains(out, "settled") {
+		t.Fatalf("a settled window must say so — otherwise it is indistinguishable from a budget "+
+			"expiry:\n%s", out)
+	}
+	if strings.Contains(out, "budget") {
+		t.Fatalf("a settled window must NOT claim the budget ended it:\n%s", out)
+	}
+}
+
+func TestBootWindow_CancelledContextStopsImmediately(t *testing.T) {
+	shrinkWindow(t, time.Millisecond, 3, time.Second)
+	seen := captureSweep(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	runBootReconcile(ctx, &windowStacks{frames: [][]stacks.Stack{{upStack("x")}}}, log.New(io.Discard, "", 0))
+	if len(*seen) != 0 {
+		t.Fatalf("the sweep ran %d times on a cancelled context, want 0", len(*seen))
+	}
+}
+
+// --- Group C / Scenario D — a customer's Stop survives the WIDENED window -------------------------
+
+func TestBootWindow_CustomerStoppedAppSurvivesEveryPass(t *testing.T) {
+	// THE REGRESSION THIS TASK COULD INTRODUCE. A longer window means more chances to resurrect an
+	// app the customer deliberately stopped. It must survive the whole window — this drives the REAL
+	// bootrecon sweep (not the captured stub), so the desired-state check is genuinely exercised.
+	//
+	// RED-PROOF: drop the DesiredStateStopped branch from isBootOrphan (make it fall through to the
+	// running case) and this test fails with a start count of 1. Demonstrated in REPORT.md §4.
+	shrinkWindow(t, time.Millisecond, 2, 200*time.Millisecond)
+
+	stopped := stacks.Stack{
+		Name: "nextcloud", Deployed: true, State: stacks.StateStopped, Containers: nil,
+		AppConfig: &stacks.AppConfig{Deployed: true, DesiredState: stacks.DesiredStateStopped},
+	}
+	// The fleet churns around it, so the window runs many passes before settling.
+	frames := [][]stacks.Stack{
+		{stopped, settlingLate("immich")},
+		{stopped, settlingLate("immich")},
+		{stopped, settledDown("immich")},
+		{stopped, upStack("immich")},
+		{stopped, upStack("immich")},
+		{stopped, upStack("immich")},
+	}
+	w := &windowStacks{frames: frames, onStart: func(w *windowStacks, _ string) {}}
+
+	runBootReconcile(context.Background(), w, log.New(io.Discard, "", 0))
+
+	if n := w.starts["nextcloud"]; n != 0 {
+		t.Fatalf("the customer-stopped app was started %d time(s) by the widened window — this is the "+
+			"regression a longer window makes possible and it is the worst outcome available here", n)
+	}
+}
+
+// --- §8.3 — a late recovery is REPORTED, never hidden ---------------------------------------------
+
+func TestRecordLateRecovery_WarnsWhenTheGraceHasAlreadyExpired(t *testing.T) {
+	var buf strings.Builder
+	lg := log.New(&buf, "", 0)
+	// started far enough back that settle + elapsed exceeds the 90 s grace
+	recordLateRecovery(lg, time.Now().Add(-(deadAppBootGrace + 10*time.Second)), bootrecon.Result{Recovered: []string{"immich"}})
+	out := buf.String()
+	if !strings.Contains(out, "LATE RECOVERY") || !strings.Contains(out, "immich") {
+		t.Fatalf("a recovery past the dead-app grace must be reported by name — otherwise a stale "+
+			"alarm stands with no counter-evidence (§8.3):\n%s", out)
+	}
+}
+
+func TestRecordLateRecovery_SilentInsideTheGrace(t *testing.T) {
+	var buf strings.Builder
+	recordLateRecovery(log.New(&buf, "", 0), time.Now(), bootrecon.Result{Recovered: []string{"immich"}})
+	if buf.Len() != 0 {
+		t.Fatalf("a recovery INSIDE the grace must stay silent — that is what makes a successful "+
+			"recovery invisible to the customer:\n%s", buf.String())
+	}
+}
+
+func TestRecordLateRecovery_SilentWhenNothingRecovered(t *testing.T) {
+	var buf strings.Builder
+	recordLateRecovery(log.New(&buf, "", 0), time.Now().Add(-time.Hour), bootrecon.Result{})
+	if buf.Len() != 0 {
+		t.Fatalf("nothing was recovered, so there is nothing late to report:\n%s", buf.String())
+	}
+}
+
+// --- The window's constants must fit the grace they are justified against -------------------------
+
+func TestBootWindow_CommonCaseFitsInsideTheDeadAppGrace(t *testing.T) {
+	// The comment on the window constants justifies them against deadAppBootGrace. A comment
+	// asserting an invariant needs a test pinning it, or it is a wish.
+	common := bootReconcileSettle + bootReconcileBudget + bootrecon.DefaultRetryDelay
+	if common > deadAppBootGrace {
+		t.Fatalf("settle(%s) + budget(%s) + one retry(%s) = %s exceeds the %s dead-app grace — the "+
+			"COMMON case must stay silent, or every slow boot alerts",
+			bootReconcileSettle, bootReconcileBudget, bootrecon.DefaultRetryDelay, common, deadAppBootGrace)
+	}
+	if bootReconcileSample <= 0 || bootReconcileStableFor < 2 {
+		t.Fatalf("sample=%s stableFor=%d — one sample cannot distinguish 'settled' from 'sampled "+
+			"between two docker events'", bootReconcileSample, bootReconcileStableFor)
+	}
+}
@@ -13,6 +13,7 @@ import (
 	"os/exec"
 	"os/signal"
 	"path/filepath"
+	"sort"
 	"syscall"
 	"time"

@@ -258,6 +259,12 @@ func main() {
 	// sweep, deliberately AFTER the quiesce recovery above so the two never race for the same stack,
 	// and entirely inside deadAppBootGrace so a successful recovery is silent and a failed one still
 	// alerts honestly. Never touches an app the customer stopped — see internal/bootrecon.
+	// R-171: hand the boot sweep the settings it needs to answer "is this app's drive live?" BEFORE
+	// the goroutine starts — an unwired gate is silently the pre-v0.190.0 behaviour that started apps
+	// onto absent drives. TestMainWiresBootDriveGate walks this file's AST for the assignment.
+	bootDriveSettings = sett
+	bootQuiesceLoop = quiesceLoop
+	bootAppStopGuard = appStopGuard
 	go runBootReconcile(ctx, stackMgr, logger)

 	// --- Start CPU collector ---
@@ -1258,27 +1265,247 @@ func noteDeadAppScan(logger *log.Logger, scans, evaluated, down int) {
 		scans, evaluated, down)
 }

-// bootReconcileSettle lets the initial scan, the first status refresh and the quiesce recovery
-// settle before the R-52 sweep decides what "down" means. 5 s + at most one 30 s retry gap keeps
-// the whole sweep inside deadAppBootGrace (90 s), which is what makes a successful recovery silent.
+// bootReconcileSettle is the delay before the FIRST observation. It lets the initial scan, the first
+// status refresh and the two crash recoveries land before the R-52 sweep decides what "down" means.
 var bootReconcileSettle = 5 * time.Second

+// ── R-157 mechanism A: the window, and why these three numbers ───────────────────────────────────
+//
+// Until v0.190.0 the sweep looked exactly ONCE, at T+5 s, and returned. At T+5 s docker is still
+// restoring containers after a hard reset, so an app that has not yet settled into a down state is
+// not a candidate — and because the sweep never looked again, it stayed down. Measured failing on
+// THREE OF SIX hard resets (CAMPAIGN-10). The predicate was never the problem; the single
+// observation was.
+//
+// The fix is a window that SETTLES rather than a timer that runs forever (§5: an unbounded loop
+// papers over a genuinely broken app and hammers docker). Three constants, each chosen against the
+// 90 s dead-app boot grace:
+//
+//   - bootReconcileSample = 5 s. Fine enough that a container settling at T+40 s is seen within one
+//     sample, coarse enough that a quiet boot costs ~12 cheap GetStacks() calls, not hundreds.
+//   - bootReconcileStableFor = 3 consecutive identical samples (15 s of no change) before the fleet
+//     is called settled. One sample cannot distinguish "settled" from "sampled between two docker
+//     events"; three spans the gap between a container exiting and its restart policy re-creating it.
+//   - bootReconcileBudget = 50 s. THE BINDING CONSTRAINT, and it is arithmetic, not taste:
+//     bootReconcileSettle (5 s) + budget (50 s) + ONE DefaultRetryDelay (30 s) inside the final
+//     sweep = 85 s, which must stay under deadAppBootGrace (90 s) so a recovery that works is
+//     SILENT. 60 s was the first choice and TestBootWindow_CommonCaseFitsInsideTheDeadAppGrace
+//     rejected it at 95 s — the test is the reason this number is 50 and not a round 60.
+//     Extending the grace to make a bigger budget fit was rejected (§8.3): that hides a late
+//     recovery rather than reporting it. A window that genuinely overruns is reported instead —
+//     see recordLateRecovery below.
+//
+// The window ends on WHICHEVER COMES FIRST — settled, or budget exhausted — and the log says which,
+// because "settled and found nothing" and "ran out of time still churning" are different facts about
+// the box and must not read the same (the v0.91.2 lesson).
+var (
+	bootReconcileSample    = 5 * time.Second
+	bootReconcileStableFor = 3
+	bootReconcileBudget    = 50 * time.Second
+)
+
 // bootReconcileFn is the R-52 sweep, a package var purely so the wiring below is testable from
 // package main (the v0.154.0 / v0.91.0 lesson: a seam proven only through injection proves the
 // component and not the caller).
 var bootReconcileFn = func(ctx context.Context, mgr bootrecon.StackProvider, logger *log.Logger) bootrecon.Result {
-	return bootrecon.New(mgr, logger).Run(ctx)
+	r := bootrecon.New(mgr, logger)
+	// R-171: the sweep must not start an app whose data drive is absent. Wired HERE, at the one
+	// place the sweep is constructed, so there is no path that builds an ungated reconciler.
+	if sm, ok := mgr.(*stacks.Manager); ok {
+		r.SetDriveGate(bootDriveGate{mgr: sm, sett: bootDriveSettings})
+	}
+	return r.Run(ctx)
 }

-// runBootReconcile waits out the settle window, then performs exactly one bounded recovery sweep.
-// Called from main() in a goroutine; returns after the single sweep — there is no loop by design.
+// bootDriveSettings / bootStartHolders are what the boot start gate reads. Init-only, set in main()
+// before the reconcile goroutine is launched; both are nil-safe (see MayStart).
+var (
+	bootDriveSettings *settings.Settings
+	bootQuiesceLoop   *quiesce.Loop
+	bootAppStopGuard  *backup.AppStopGuard
+)
+
+// bootDriveGate answers bootrecon.StartGate for the real controller. It enforces §8.2: an app that
+// something else is deliberately holding must NOT be started by the boot sweep.
+//
+// THE THREE HOLDERS, in the order they are checked. The first two only became reachable when R-157
+// mechanism A widened the window — the old T+5 s single sweep never overlapped a quiesce or a
+// running app-data operation, and that is exactly why widening it needed these:
+//
+//  1. QUIESCE — the whole-guest backup loop stops app stacks and restarts exactly the ones it
+//     stopped. Starting one mid-backup would put a running app inside a snapshot that is supposed to
+//     be clean-shutdown-consistent, which is the entire point of quiescing. `SuppressedStacks` is
+//     the set it already publishes for precisely this "an app WE stopped is not a fault" question,
+//     so reusing it means the two cannot drift.
+//  2. THE APP-STOP GUARD — a volume dump / offbox reconstitute / .fab export that is CURRENTLY
+//     holding an app down. Its own Recover already ran to completion before this goroutine started,
+//     so the marker seen here belongs to an operation running NOW, not to a crashed one.
+//  3. THE DRIVE — R-171. Two questions, because they fail in opposite directions and neither alone
+//     is sufficient at boot:
+//     • the settings flags (`Disconnected`/`Decommissioned`) — the SAME signal the API's own
+//     `startGatedByMissingDrive` uses, so the customer's path and the sweep cannot disagree about
+//     whether an app may start. But they are the drive gate's bookkeeping, and inside the boot
+//     window that gate may not have ticked yet, so a genuinely absent drive can still read as
+//     connected.
+//     • `Manager.DriveLive` — the live mountpoint check, the SAME `isMountPoint` seam the userdata
+//     belt already uses. It answers immediately and needs no gate tick.
+//
+// Fail-safe per bootrecon.StartGate's contract: anything that cannot be determined returns false.
+type bootDriveGate struct {
+	mgr  *stacks.Manager
+	sett *settings.Settings
+}
+
+func (g bootDriveGate) MayStart(stackName string) (bool, string) {
+	// 1. quiesce (nil-safe on an unprovisioned guest: SuppressedStacks returns nil)
+	if bootQuiesceLoop.SuppressedStacks()[stackName] {
+		return false, "a whole-guest backup (quiesce) is holding it — the quiesce loop restarts its own stacks"
+	}
+	// 2. an app-data operation in flight
+	for _, held := range bootAppStopGuard.HeldStacks() {
+		if held == stackName {
+			return false, "an app-data operation is holding it — the app-stop guard restarts it when the operation ends"
+		}
+	}
+	// 3. the drive
+	cfg := g.mgr.LoadAppConfigByName(stackName)
+	if cfg == nil {
+		// CANNOT DETERMINE. A deployed app whose app.yaml will not load cannot have its drive
+		// resolved, so the fail-safe direction applies rather than a hopeful start.
+		return false, "app.yaml could not be read"
+	}
+	hdd := cfg.Env["HDD_PATH"]
+	if hdd == "" {
+		return true, "" // SSD-resident: there is no external drive to be absent (mirrors the API gate)
+	}
+	if g.sett != nil {
+		for _, sp := range g.sett.GetStoragePaths() {
+			if sp.Path != hdd {
+				continue
+			}
+			if sp.Decommissioned {
+				return false, "drive " + hdd + " is decommissioned"
+			}
+			if sp.Disconnected {
+				return false, "drive " + hdd + " is flagged disconnected"
+			}
+		}
+	}
+	if !g.mgr.DriveLive(hdd) {
+		return false, "drive " + hdd + " is not a live mountpoint"
+	}
+	return true, ""
+}
+
+// bootFleetSample is a comparable snapshot of one deployed app — name, state and container count,
+// per §8.1. Container count is in it deliberately: a stack can go from 3 containers to 0 without its
+// aggregate state changing, and that IS the boot still moving.
+type bootFleetSample struct {
+	name       string
+	state      string
+	containers int
+}
+
+// sampleBootFleet returns the fleet snapshot, sorted, so two samples compare by equality.
+func sampleBootFleet(mgr bootrecon.StackProvider) []bootFleetSample {
+	stacksNow := mgr.GetStacks()
+	out := make([]bootFleetSample, 0, len(stacksNow))
+	for _, s := range stacksNow {
+		if !s.Deployed {
+			continue
+		}
+		out = append(out, bootFleetSample{name: s.Name, state: string(s.State), containers: len(s.Containers)})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].name < out[j].name })
+	return out
+}
+
+func sameBootFleet(a, b []bootFleetSample) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// runBootReconcile waits out the settle delay, then samples the fleet until it stops changing (or
+// the budget runs out) and performs the bounded sweep ONCE, at the end.
+//
+// Sweeping on every sample was rejected: the sweep's own StartStack changes the fleet, so a
+// sweep-per-sample would never observe a settled fleet and would race docker's restore. Sampling is
+// read-only; exactly one sweep runs, and it re-derives its candidate set from a settled fleet —
+// which is the whole point, since the pre-v0.190.0 bug was a candidate set derived too early.
+//
+// Called from main() in a goroutine.
 func runBootReconcile(ctx context.Context, mgr bootrecon.StackProvider, logger *log.Logger) {
 	select {
 	case <-ctx.Done():
 		return
 	case <-time.After(bootReconcileSettle):
 	}
-	bootReconcileFn(ctx, mgr, logger)
+
+	started := time.Now()
+	prev := sampleBootFleet(mgr)
+	stable := 1
+	settled := false
+
+	for time.Since(started) < bootReconcileBudget {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(bootReconcileSample):
+		}
+		cur := sampleBootFleet(mgr)
+		if sameBootFleet(prev, cur) {
+			stable++
+		} else {
+			// Not settled — the boot is still moving. Log at DEBUG: at 5 s cadence an INFO line per
+			// sample would bury the one line that matters, which is the verdict below.
+			logger.Printf("[DEBUG] [bootrecon] boot window: fleet still changing (%d app(s)) — resampling", len(cur))
+			stable = 1
+		}
+		prev = cur
+		if stable >= bootReconcileStableFor {
+			settled = true
+			break
+		}
+	}
+
+	if settled {
+		logger.Printf("[INFO] [bootrecon] boot window: fleet settled after %.0fs (%d identical samples %s apart) — sweeping",
+			time.Since(started).Seconds(), bootReconcileStableFor, bootReconcileSample)
+	} else {
+		// NOT a failure — a box whose apps are still churning at the budget is exactly the box that
+		// most needs the sweep. But it is a different fact from "settled", and saying so is what makes
+		// a stuck boot visible instead of looking like a quiet one.
+		logger.Printf("[INFO] [bootrecon] boot window: budget %s exhausted while the fleet was still changing — sweeping anyway",
+			bootReconcileBudget)
+	}
+
+	res := bootReconcileFn(ctx, mgr, logger)
+	recordLateRecovery(logger, started, res)
+}
+
+// recordLateRecovery keeps §8.3 honest. The window can finish AFTER deadAppBootGrace, and when it
+// does the dead-app alarm has already fired for an app this sweep then recovered. Extending the
+// grace to hide that was rejected; reporting it is the alternative, so the record is truthful and
+// the operator is not left with a stale alarm and no counter-evidence.
+//
+// Measured from controller start, which is what the grace is measured from.
+func recordLateRecovery(logger *log.Logger, started time.Time, res bootrecon.Result) {
+	if len(res.Recovered) == 0 {
+		return
+	}
+	elapsed := bootReconcileSettle + time.Since(started)
+	if elapsed <= deadAppBootGrace {
+		return
+	}
+	logger.Printf("[WARN] [bootrecon] LATE RECOVERY: %d app(s) recovered %.0fs after start, past the %s dead-app grace — an alert may already have fired for: %v",
+		len(res.Recovered), elapsed.Seconds(), deadAppBootGrace, res.Recovered)
 }

 // scanDeployedAppRunStates returns the fix-3 view of the deployed apps: the DEAD ones (for the