v0.190.0 — the boot settle window, both gates on intent, and R-171
gates / gates (push) Successful in 8s

R-171 (a regression v0.189.0 introduced, CONFIRMED on hardware before any fix
was written). Replacing isBootOrphan's container-count term with recorded intent
made a drive-gate-stopped app read as a boot orphan: the gate stops apps with
`compose down` (zero containers) and never touches desired_state, because it is
not the customer. Observed on 9201 with the drive held unmounted — the sweep
found and started it, burned both attempts, and handed it to the dead-app alarm.
The write hazard did not materialise (the unbound mountpoint is host-root-owned
and the guest is unprivileged) but that protection is accidental and untested.
New consumer-side seam bootrecon.StartGate, fail-safe (cannot determine ⇒ do not
start), wired in main.go. The rule is not new: the API's startGatedByMissingDrive
already refuses this; the sweep bypassed it.

R-157 mechanism A. The sweep looked once at T+5s, deriving candidates from a
fleet docker was still restoring — three of six hard resets. Now a settle-then-
sweep window: sample every 5s, settled after 3 identical samples, sweep ONCE at
the end; ends on settled or a 50s budget, and the log says which. The budget is
50s because settle+budget+one retry must stay under the 90s dead-app grace — a
test rejected 60s at 95s. A window that overruns emits a LATE RECOVERY warn
rather than the grace being widened to hide it.

Widening the window made two more holders reachable, so the one gate covers all
three: an absent drive, a quiesce, and an in-flight app-data operation — reusing
quiesce.SuppressedStacks() and a new read-only AppStopGuard.HeldStacks().

R-170. shouldRecreateOnBoot now reads desired_state with the identical three-way
table; absent keeps the old hasContainers behaviour exactly. Its comment argued
for the container count and was rewritten. presentStable is untouched. The two
gates' agreement is pinned from both sides against one fixture table.

27/27 packages green; 6 red-proofs observed FAIL then restored.
This commit is contained in:
2026-08-02 19:56:20 +02:00
parent 3446609420
commit 582135f861
13 changed files with 1272 additions and 43 deletions
@@ -0,0 +1,147 @@
package web
import (
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)
// R-170 — the drive-backed boot recreate gate reads the customer's recorded intent instead of
// counting containers, so the two boot gates stop disagreeing about the same question.
const rgDrive = "/mnt/felhom-drives/hdd_1"
func rgPresent() map[string]bool { return map[string]bool{rgDrive: true} }
// --- Group D / Scenario E — the three-way table, every row ----------------------------------------
func TestShouldRecreateOnBoot_IntentTable(t *testing.T) {
// RED-PROOF: restore the old `&& hasContainers` ending (i.e. ignore `desired` entirely) and the
// `running/no containers` row fails — that row is the whole point of R-170, and it is the shape a
// power cut mid-compose leaves behind on a drive-backed app.
// Demonstrated in REPORT.md §4.
cases := []struct {
name string
desired string
hasContainers bool
want bool
}{
{"stopped + no containers", stacks.DesiredStateStopped, false, false},
{"stopped + containers present", stacks.DesiredStateStopped, true, false},
{"running + no containers", stacks.DesiredStateRunning, false, true},
{"running + containers present", stacks.DesiredStateRunning, true, true},
{"legacy + no containers", stacks.DesiredStateUnknown, false, false},
{"legacy + containers present", stacks.DesiredStateUnknown, true, true},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := shouldRecreateOnBoot(true, rgDrive, rgPresent(), c.hasContainers, c.desired)
if got != c.want {
t.Fatalf("shouldRecreateOnBoot(desired=%q hasContainers=%v) = %v, want %v",
c.desired, c.hasContainers, got, c.want)
}
})
}
}
func TestShouldRecreateOnBoot_StoppedIsNeverRecreated_WhateverElseIsTrue(t *testing.T) {
// The safety property R-55 added this gate's filter for, restated on the new signal: a customer's
// Stop must survive a guest reboot. It must hold across every other axis.
for _, hasContainers := range []bool{true, false} {
if shouldRecreateOnBoot(true, rgDrive, rgPresent(), hasContainers, stacks.DesiredStateStopped) {
t.Fatalf("a recorded Stop was recreated (hasContainers=%v) — this silently undoes the "+
"customer's decision on every guest reboot, which is the defect R-55 existed to fix",
hasContainers)
}
}
}
func TestShouldRecreateOnBoot_DriveAbsentStillWinsOverIntent(t *testing.T) {
// The `presentStable` term is load-bearing and must NOT have been dropped in sympathy with the
// container count. An app whose drive is absent is never recreated here, whatever its intent —
// the drive gate's Return branch owns it. This is the exact term the BOOT SWEEP was missing
// (R-171), so removing it here would reproduce that hazard in the other gate.
absent := map[string]bool{rgDrive: false}
for _, desired := range []string{stacks.DesiredStateRunning, stacks.DesiredStateStopped, stacks.DesiredStateUnknown} {
if shouldRecreateOnBoot(true, rgDrive, absent, true, desired) {
t.Fatalf("an app whose drive is ABSENT was recreated (desired=%q) — the drive-presence "+
"term is load-bearing and must outrank intent", desired)
}
}
}
func TestShouldRecreateOnBoot_OtherGuardsSurviveTheRewrite(t *testing.T) {
// deployed / stable-parent-prefix / HDD_PATH were terms before R-170 and must still be, at the
// strongest intent available — the rewrite reordered the terms and a reorder is how a guard
// silently disappears.
if shouldRecreateOnBoot(false, rgDrive, rgPresent(), true, stacks.DesiredStateRunning) {
t.Fatal("a NOT-deployed app was recreated")
}
if shouldRecreateOnBoot(true, "", rgPresent(), true, stacks.DesiredStateRunning) {
t.Fatal("an app with no HDD_PATH (SSD-resident) was recreated by the DRIVE gate")
}
sys := "/mnt/sys_drive/felhom-data"
if shouldRecreateOnBoot(true, sys, map[string]bool{sys: true}, true, stacks.DesiredStateRunning) {
t.Fatal("a non-stable-parent (system) path was recreated by the drive gate")
}
}
// --- Scenario F — the two boot gates agree --------------------------------------------------------
func TestShouldRecreateOnBoot_AgreesWithBootrecon(t *testing.T) {
// The sibling of bootrecon's TestBothBootGatesAgreeOnIntent, over the SAME fixture table. The two
// gates cannot be called from one package without an import cycle, so the agreement is pinned
// from both sides against an identical table. If this table changes, change the other.
//
// Both answer: "did the customer want this app running?" On a live drive, this gate's verdict
// must equal that answer exactly.
cases := []struct {
desired string
hasContainers bool
wantWanted bool
}{
{stacks.DesiredStateStopped, false, false},
{stacks.DesiredStateStopped, true, false},
{stacks.DesiredStateRunning, false, true},
{stacks.DesiredStateRunning, true, true},
{stacks.DesiredStateUnknown, false, false},
{stacks.DesiredStateUnknown, true, true},
}
for _, c := range cases {
got := shouldRecreateOnBoot(true, rgDrive, rgPresent(), c.hasContainers, c.desired)
if got != c.wantWanted {
t.Fatalf("gate disagreement: shouldRecreateOnBoot(desired=%q containers=%v) = %v, but "+
"bootrecon.isBootOrphan answers %v for the same facts. Two boot gates answering the "+
"same question differently is how R-157 mechanism B survived a year",
c.desired, c.hasContainers, got, c.wantWanted)
}
}
}
// --- the counter that reports the honoured path ---------------------------------------------------
func TestRecreateDriveBackedApps_RecordedStopCountsAsLeftStopped(t *testing.T) {
// `leftStopped` has always meant "deliberately not touched", and it is reported at INFO as the
// gate working as intended. After R-170 a recorded Stop must land in that bucket — NOT in
// `skipped`, which is the "bind never went live" failure and fires a WARN.
apps := []bootStack{
{name: "stopped-app", deployed: true, hdd: rgDrive, hasContainers: false, desired: stacks.DesiredStateStopped},
{name: "legacy-stopped", deployed: true, hdd: rgDrive, hasContainers: false, desired: stacks.DesiredStateUnknown},
{name: "wanted-running", deployed: true, hdd: rgDrive, hasContainers: false, desired: stacks.DesiredStateRunning},
}
var recreated []string
n, skipped, leftStopped := recreateDriveBackedApps(apps, rgPresent(),
func(bs bootStack) { recreated = append(recreated, bs.name) }, func() {})
if n != 1 || len(recreated) != 1 || recreated[0] != "wanted-running" {
t.Fatalf("recreated=%v (n=%d), want exactly [wanted-running] — the zero-container app the "+
"customer wants RUNNING is the R-170 case", recreated, n)
}
if leftStopped != 2 {
t.Fatalf("leftStopped=%d, want 2 (the recorded Stop and the legacy zero-container app)", leftStopped)
}
if skipped != 0 {
t.Fatalf("skipped=%d, want 0 — the drive is live, so nothing was skipped for a missing bind; "+
"counting a deliberate Stop as skipped would fire a WARN for healthy behaviour", skipped)
}
}