admin
dbcb306fcf
gates / gates (push) Successful in 8s
The box stops inferring the customer's intent from a container count and reads
what they actually asked for.
Part 1 — desired state. AppConfig gains a tri-state `desired_state`
(""/running/stopped), written ONLY by the customer's own action: the API action
switch, DeployStack, UpdateOptionalConfig's redeploy branch, and the .fab
import. Intent is written BEFORE the act and a failed write REFUSES the act.
StartStack/StopStack are deliberately not writers — 14 callers, only 2 are the
customer. bootrecon.isBootOrphan now reads intent instead of len(Containers)>0,
which closes R-157 mechanism B (a power cut or interrupted deploy left an app
with zero containers, read as a deliberate stop, and stranded silently).
ABSENT MEANS UNKNOWN, NEVER "running": every pre-v0.189.0 app.yaml reads absent,
so the legacy fallback is byte-identical to the old rule. A running-only startup
backfill converges the unambiguous cases; `stopped` is never inferred.
Part 2 — backup.AppStopGuard, a persisted marker over every stop→work→start
window (volume dump, offbox reconstitute, .fab export). Its own file, never
quiesce's. Written before the stop, cleared only after a restart that succeeded,
kept when one fails. Recover() completes before the boot reconciler is launched
and returns its outcome, which main.go reports on the existing backup_failed
event once the notifier exists. A defer is not the mechanism — a SIGKILL runs
none (Campaign 8 fault 10).
Also: SaveAppConfig rebuilt AppConfig field-by-field (the R-100 shape) and would
have dropped desired_state on every save across nine call sites. Replaced with
copy-and-overlay. Measured: app.yaml does not round-trip unknown YAML keys.
No hub change, no agent coupling, no user-visible string. 27/27 packages green;
7 red-proofs observed FAIL then restored.
237 lines
11 KiB
Go
237 lines
11 KiB
Go
package bootrecon
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
|
)
|
|
|
|
// R-166 / decision D-b: the boot reconciler reads the CUSTOMER'S RECORDED INTENT instead of
|
|
// inferring it from a container count. These tests are the §8.1 decision table, one row each, plus
|
|
// the two red-proofs that make the safety properties falsifiable.
|
|
//
|
|
// The assertions are EFFECTS — which apps the sweep actually started — not "isBootOrphan returned
|
|
// true". A predicate can be right while the sweep does nothing with it.
|
|
|
|
// withDesired returns a copy of s carrying a recorded desired state.
|
|
func withDesired(s stacks.Stack, desired string) stacks.Stack {
|
|
s.AppConfig = &stacks.AppConfig{Deployed: true, DesiredState: desired}
|
|
return s
|
|
}
|
|
|
|
// vanished is the R-157 shape this whole change exists to see: the app is deployed and wanted
|
|
// running, and its containers are simply GONE — a power cut mid-compose, or an interrupted deploy.
|
|
// Byte-identical on the Docker side to a user stop, which is exactly why the old container-count
|
|
// rule could not tell them apart.
|
|
func vanished(name string) stacks.Stack {
|
|
return stacks.Stack{Name: name, Deployed: true, State: stacks.StateStopped, Containers: nil}
|
|
}
|
|
|
|
// runSweep runs one full reconciliation and returns which apps were started, and how often.
|
|
func runSweep(t *testing.T, list []stacks.Stack) (*fakeStacks, Result) {
|
|
t.Helper()
|
|
f := &fakeStacks{list: list, onStart: comesUp}
|
|
r, _ := newTestReconciler(f)
|
|
res := r.Run(context.Background())
|
|
return f, res
|
|
}
|
|
|
|
// --- Scenario A — the customer's Stop survives everything ---------------------------------------
|
|
|
|
func TestReconcile_DesiredStopped_IsNeverStartedAndNeverACandidate(t *testing.T) {
|
|
// Recorded stopped, and down in every way the box can be down: no containers at all, and (second
|
|
// app) containers present but exited. Neither may be touched, and neither may even be LISTED —
|
|
// a candidate that is never started still tells the operator an app is broken when it is not.
|
|
f, res := runSweep(t, []stacks.Stack{
|
|
withDesired(vanished("nextcloud"), stacks.DesiredStateStopped),
|
|
withDesired(bootOrphan("immich"), stacks.DesiredStateStopped),
|
|
})
|
|
|
|
if len(f.starts) != 0 {
|
|
t.Fatalf("an app the customer deliberately stopped was started: %v", f.starts)
|
|
}
|
|
if len(res.Candidates) != 0 {
|
|
t.Fatalf("desired=stopped app listed as a boot orphan: %v", res.Candidates)
|
|
}
|
|
if res.Attempts != 0 {
|
|
t.Fatalf("attempts=%d, want 0 — the sweep should have had nothing to do", res.Attempts)
|
|
}
|
|
}
|
|
|
|
// --- Scenario B — the power-cut app comes back (THE R-157 CASE) ---------------------------------
|
|
|
|
func TestReconcile_DesiredRunning_ZeroContainers_IsRecovered(t *testing.T) {
|
|
// THE POINT OF THE RELEASE. Before v0.189.0 this app was invisible to the reconciler: zero
|
|
// containers failed the `len(s.Containers) > 0` term, so it was skipped as "the customer stopped
|
|
// it" and stayed down until a human noticed.
|
|
//
|
|
// RED-PROOF: restore that term in isBootOrphan's DesiredStateRunning branch — i.e. make it
|
|
// return len(s.Containers) > 0 && stacks.IsDownState(s.State)
|
|
// and this test fails with `zero starts`. Demonstrated in REPORT.md §5.
|
|
f, res := runSweep(t, []stacks.Stack{withDesired(vanished("immich"), stacks.DesiredStateRunning)})
|
|
|
|
if f.starts["immich"] == 0 {
|
|
t.Fatalf("an app recorded desired=running with zero containers was NOT started — this is the R-157 defect")
|
|
}
|
|
if len(res.Recovered) != 1 || res.Recovered[0] != "immich" {
|
|
t.Fatalf("recovered=%v, want [immich]", res.Recovered)
|
|
}
|
|
if len(res.StillDown) != 0 {
|
|
t.Fatalf("still down after a successful start: %v", res.StillDown)
|
|
}
|
|
}
|
|
|
|
func TestReconcile_DesiredRunning_ContainersDown_IsRecovered(t *testing.T) {
|
|
// The pre-existing F5 shape, unchanged by R-166 — proven still covered so the rewrite cannot
|
|
// have traded one case for the other.
|
|
f, _ := runSweep(t, []stacks.Stack{withDesired(bootOrphan("calibre-web"), stacks.DesiredStateRunning)})
|
|
if f.starts["calibre-web"] == 0 {
|
|
t.Fatal("an app recorded desired=running with exited containers was not started")
|
|
}
|
|
}
|
|
|
|
func TestReconcile_DesiredRunning_AlreadyUp_IsLeftAlone(t *testing.T) {
|
|
up := withDesired(stacks.Stack{
|
|
Name: "vaultwarden", Deployed: true, State: stacks.StateRunning,
|
|
Containers: []stacks.ContainerInfo{{Name: "vw", State: stacks.StateRunning}},
|
|
}, stacks.DesiredStateRunning)
|
|
|
|
f, res := runSweep(t, []stacks.Stack{up})
|
|
if len(f.starts) != 0 {
|
|
t.Fatalf("a running app was restarted: %v", f.starts)
|
|
}
|
|
if len(res.Candidates) != 0 {
|
|
t.Fatalf("a running app was listed as a boot orphan: %v", res.Candidates)
|
|
}
|
|
}
|
|
|
|
// --- Scenario C — a legacy app.yaml behaves EXACTLY as it does today -----------------------------
|
|
|
|
func TestReconcile_LegacyNoDesiredState_BehavesExactlyAsBefore(t *testing.T) {
|
|
// THE MOST DANGEROUS MISTAKE AVAILABLE IN THIS CHANGE. Every app.yaml on every existing box was
|
|
// written before desired_state existed, so `absent` is what the whole fleet reads on upgrade.
|
|
// Treating absent as "running" would start, on the first boot after the upgrade, every app its
|
|
// owner had deliberately stopped — silently, fleet-wide.
|
|
//
|
|
// Both legacy rows of §8.1 asserted together, because the safety property is the PAIR: absent +
|
|
// zero containers must be skipped, and absent + down containers must still be recovered. A
|
|
// change that broke only one of them would look correct from the other.
|
|
//
|
|
// RED-PROOF: make the `default:` branch of isBootOrphan return
|
|
// len(s.Containers) == 0 || stacks.IsDownState(s.State)
|
|
// (i.e. treat absent as running) and this test fails on the "started" assertion.
|
|
// Demonstrated in REPORT.md §5.
|
|
legacyStopped := vanished("nextcloud") // no AppConfig at all — the true legacy shape
|
|
legacyOrphan := bootOrphan("calibre-web") // no AppConfig, containers present and exited
|
|
legacyOrphan.AppConfig = nil
|
|
legacyStopped.AppConfig = nil
|
|
|
|
f, res := runSweep(t, []stacks.Stack{legacyStopped, legacyOrphan})
|
|
|
|
if n := f.starts["nextcloud"]; n != 0 {
|
|
t.Fatalf("a LEGACY app with no recorded intent and zero containers was started %d time(s) — "+
|
|
"this is the upgrade regression that restarts apps customers deliberately stopped", n)
|
|
}
|
|
if f.starts["calibre-web"] == 0 {
|
|
t.Fatal("a LEGACY boot orphan (containers present, exited) was not recovered — the pre-R-166 behaviour regressed")
|
|
}
|
|
if len(res.Candidates) != 1 || res.Candidates[0] != "calibre-web" {
|
|
t.Fatalf("candidates=%v, want exactly [calibre-web]", res.Candidates)
|
|
}
|
|
}
|
|
|
|
func TestReconcile_LegacyAppConfigPresentButFieldAbsent_IsAlsoLegacy(t *testing.T) {
|
|
// An app.yaml that EXISTS but predates the field: AppConfig is non-nil, DesiredState is "".
|
|
// This is the realistic fleet shape (nil AppConfig only happens with no app.yaml at all), and it
|
|
// must take the same legacy path — a nil-vs-empty distinction slipping in here would silently
|
|
// split the fleet in two.
|
|
s := vanished("immich")
|
|
s.AppConfig = &stacks.AppConfig{Deployed: true} // DesiredState is the zero value
|
|
f, _ := runSweep(t, []stacks.Stack{s})
|
|
if len(f.starts) != 0 {
|
|
t.Fatalf("an app.yaml with no desired_state key was treated as running: %v", f.starts)
|
|
}
|
|
}
|
|
|
|
// --- The §8.1 table, every row, in one place ----------------------------------------------------
|
|
|
|
func TestIsBootOrphan_DecisionTable(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
desired string
|
|
containers int
|
|
state stacks.ContainerState
|
|
want bool
|
|
}{
|
|
{"stopped/no containers", stacks.DesiredStateStopped, 0, stacks.StateStopped, false},
|
|
{"stopped/down containers", stacks.DesiredStateStopped, 2, stacks.StateExited, false},
|
|
{"stopped/running", stacks.DesiredStateStopped, 2, stacks.StateRunning, false},
|
|
{"running/no containers", stacks.DesiredStateRunning, 0, stacks.StateStopped, true},
|
|
{"running/down containers", stacks.DesiredStateRunning, 2, stacks.StateExited, true},
|
|
{"running/degraded", stacks.DesiredStateRunning, 2, stacks.StateDegraded, true},
|
|
{"running/up", stacks.DesiredStateRunning, 2, stacks.StateRunning, false},
|
|
{"absent/no containers", stacks.DesiredStateUnknown, 0, stacks.StateStopped, false},
|
|
{"absent/down containers", stacks.DesiredStateUnknown, 2, stacks.StateExited, true},
|
|
{"absent/up", stacks.DesiredStateUnknown, 2, stacks.StateRunning, false},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s := stacks.Stack{
|
|
Name: "app", Deployed: true, State: tc.state,
|
|
Containers: make([]stacks.ContainerInfo, tc.containers),
|
|
AppConfig: &stacks.AppConfig{Deployed: true, DesiredState: tc.desired},
|
|
}
|
|
if got := isBootOrphan(s); got != tc.want {
|
|
t.Fatalf("isBootOrphan(desired=%q containers=%d state=%s) = %v, want %v",
|
|
tc.desired, tc.containers, tc.state, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestIsBootOrphan_ExistingGuardsSurviveTheRewrite(t *testing.T) {
|
|
// Protected and Deploying were guards before R-166 and must still be, at the strongest desired
|
|
// state available — the rewrite reordered the terms, and a reorder is exactly how a guard gets
|
|
// dropped without anyone noticing.
|
|
base := func() stacks.Stack {
|
|
return withDesired(vanished("traefik"), stacks.DesiredStateRunning)
|
|
}
|
|
protected := base()
|
|
protected.Protected = true
|
|
if isBootOrphan(protected) {
|
|
t.Fatal("a PROTECTED stack became a boot orphan — the base-stack self-heal owns those")
|
|
}
|
|
deploying := base()
|
|
deploying.Deploying = true
|
|
if isBootOrphan(deploying) {
|
|
t.Fatal("a DEPLOYING stack became a boot orphan — mid-deploy is not a fault")
|
|
}
|
|
notDeployed := base()
|
|
notDeployed.Deployed = false
|
|
if isBootOrphan(notDeployed) {
|
|
t.Fatal("a stack that is not deployed became a boot orphan")
|
|
}
|
|
}
|
|
|
|
// --- Scenario G — the two recoveries do not fight ------------------------------------------------
|
|
|
|
func TestReconcile_AppAlreadyRestartedByTheMarker_IsNotAlsoAnOrphan(t *testing.T) {
|
|
// §8.4's REPORTING requirement. The app-stop marker's Recover runs to completion before this
|
|
// sweep is launched, so by the time the reconciler looks, the app it restarted is UP. It must
|
|
// therefore not appear as a candidate at all — an app the marker already explained must not also
|
|
// be reported as an unexplained boot orphan, or one fault reads as two.
|
|
restoredByMarker := withDesired(stacks.Stack{
|
|
Name: "immich", Deployed: true, State: stacks.StateRunning,
|
|
Containers: []stacks.ContainerInfo{{Name: "immich-server", State: stacks.StateRunning}},
|
|
}, stacks.DesiredStateRunning)
|
|
|
|
f, res := runSweep(t, []stacks.Stack{restoredByMarker})
|
|
if len(res.Candidates) != 0 {
|
|
t.Fatalf("an app the marker had already restarted was ALSO reported as a boot orphan: %v", res.Candidates)
|
|
}
|
|
if n := f.starts["immich"]; n != 0 {
|
|
t.Fatalf("the app was started a second time (%d) — one fault, one start", n)
|
|
}
|
|
}
|