package bootrecon import ( "context" "testing" "gitea.dooplex.hu/admin/felhom-controller/internal/stacks" ) // R-166 / decision D-b: the boot reconciler reads the CUSTOMER'S RECORDED INTENT instead of // inferring it from a container count. These tests are the §8.1 decision table, one row each, plus // the two red-proofs that make the safety properties falsifiable. // // The assertions are EFFECTS — which apps the sweep actually started — not "isBootOrphan returned // true". A predicate can be right while the sweep does nothing with it. // withDesired returns a copy of s carrying a recorded desired state. func withDesired(s stacks.Stack, desired string) stacks.Stack { s.AppConfig = &stacks.AppConfig{Deployed: true, DesiredState: desired} return s } // vanished is the R-157 shape this whole change exists to see: the app is deployed and wanted // running, and its containers are simply GONE — a power cut mid-compose, or an interrupted deploy. // Byte-identical on the Docker side to a user stop, which is exactly why the old container-count // rule could not tell them apart. func vanished(name string) stacks.Stack { return stacks.Stack{Name: name, Deployed: true, State: stacks.StateStopped, Containers: nil} } // runSweep runs one full reconciliation and returns which apps were started, and how often. func runSweep(t *testing.T, list []stacks.Stack) (*fakeStacks, Result) { t.Helper() f := &fakeStacks{list: list, onStart: comesUp} r, _ := newTestReconciler(f) res := r.Run(context.Background()) return f, res } // --- Scenario A — the customer's Stop survives everything --------------------------------------- func TestReconcile_DesiredStopped_IsNeverStartedAndNeverACandidate(t *testing.T) { // Recorded stopped, and down in every way the box can be down: no containers at all, and (second // app) containers present but exited. Neither may be touched, and neither may even be LISTED — // a candidate that is never started still tells the operator an app is broken when it is not. f, res := runSweep(t, []stacks.Stack{ withDesired(vanished("nextcloud"), stacks.DesiredStateStopped), withDesired(bootOrphan("immich"), stacks.DesiredStateStopped), }) if len(f.starts) != 0 { t.Fatalf("an app the customer deliberately stopped was started: %v", f.starts) } if len(res.Candidates) != 0 { t.Fatalf("desired=stopped app listed as a boot orphan: %v", res.Candidates) } if res.Attempts != 0 { t.Fatalf("attempts=%d, want 0 — the sweep should have had nothing to do", res.Attempts) } } // --- Scenario B — the power-cut app comes back (THE R-157 CASE) --------------------------------- func TestReconcile_DesiredRunning_ZeroContainers_IsRecovered(t *testing.T) { // THE POINT OF THE RELEASE. Before v0.189.0 this app was invisible to the reconciler: zero // containers failed the `len(s.Containers) > 0` term, so it was skipped as "the customer stopped // it" and stayed down until a human noticed. // // RED-PROOF: restore that term in isBootOrphan's DesiredStateRunning branch — i.e. make it // return len(s.Containers) > 0 && stacks.IsDownState(s.State) // and this test fails with `zero starts`. Demonstrated in REPORT.md §5. f, res := runSweep(t, []stacks.Stack{withDesired(vanished("immich"), stacks.DesiredStateRunning)}) if f.starts["immich"] == 0 { t.Fatalf("an app recorded desired=running with zero containers was NOT started — this is the R-157 defect") } if len(res.Recovered) != 1 || res.Recovered[0] != "immich" { t.Fatalf("recovered=%v, want [immich]", res.Recovered) } if len(res.StillDown) != 0 { t.Fatalf("still down after a successful start: %v", res.StillDown) } } func TestReconcile_DesiredRunning_ContainersDown_IsRecovered(t *testing.T) { // The pre-existing F5 shape, unchanged by R-166 — proven still covered so the rewrite cannot // have traded one case for the other. f, _ := runSweep(t, []stacks.Stack{withDesired(bootOrphan("calibre-web"), stacks.DesiredStateRunning)}) if f.starts["calibre-web"] == 0 { t.Fatal("an app recorded desired=running with exited containers was not started") } } func TestReconcile_DesiredRunning_AlreadyUp_IsLeftAlone(t *testing.T) { up := withDesired(stacks.Stack{ Name: "vaultwarden", Deployed: true, State: stacks.StateRunning, Containers: []stacks.ContainerInfo{{Name: "vw", State: stacks.StateRunning}}, }, stacks.DesiredStateRunning) f, res := runSweep(t, []stacks.Stack{up}) if len(f.starts) != 0 { t.Fatalf("a running app was restarted: %v", f.starts) } if len(res.Candidates) != 0 { t.Fatalf("a running app was listed as a boot orphan: %v", res.Candidates) } } // --- Scenario C — a legacy app.yaml behaves EXACTLY as it does today ----------------------------- func TestReconcile_LegacyNoDesiredState_BehavesExactlyAsBefore(t *testing.T) { // THE MOST DANGEROUS MISTAKE AVAILABLE IN THIS CHANGE. Every app.yaml on every existing box was // written before desired_state existed, so `absent` is what the whole fleet reads on upgrade. // Treating absent as "running" would start, on the first boot after the upgrade, every app its // owner had deliberately stopped — silently, fleet-wide. // // Both legacy rows of §8.1 asserted together, because the safety property is the PAIR: absent + // zero containers must be skipped, and absent + down containers must still be recovered. A // change that broke only one of them would look correct from the other. // // RED-PROOF: make the `default:` branch of isBootOrphan return // len(s.Containers) == 0 || stacks.IsDownState(s.State) // (i.e. treat absent as running) and this test fails on the "started" assertion. // Demonstrated in REPORT.md §5. legacyStopped := vanished("nextcloud") // no AppConfig at all — the true legacy shape legacyOrphan := bootOrphan("calibre-web") // no AppConfig, containers present and exited legacyOrphan.AppConfig = nil legacyStopped.AppConfig = nil f, res := runSweep(t, []stacks.Stack{legacyStopped, legacyOrphan}) if n := f.starts["nextcloud"]; n != 0 { t.Fatalf("a LEGACY app with no recorded intent and zero containers was started %d time(s) — "+ "this is the upgrade regression that restarts apps customers deliberately stopped", n) } if f.starts["calibre-web"] == 0 { t.Fatal("a LEGACY boot orphan (containers present, exited) was not recovered — the pre-R-166 behaviour regressed") } if len(res.Candidates) != 1 || res.Candidates[0] != "calibre-web" { t.Fatalf("candidates=%v, want exactly [calibre-web]", res.Candidates) } } func TestReconcile_LegacyAppConfigPresentButFieldAbsent_IsAlsoLegacy(t *testing.T) { // An app.yaml that EXISTS but predates the field: AppConfig is non-nil, DesiredState is "". // This is the realistic fleet shape (nil AppConfig only happens with no app.yaml at all), and it // must take the same legacy path — a nil-vs-empty distinction slipping in here would silently // split the fleet in two. s := vanished("immich") s.AppConfig = &stacks.AppConfig{Deployed: true} // DesiredState is the zero value f, _ := runSweep(t, []stacks.Stack{s}) if len(f.starts) != 0 { t.Fatalf("an app.yaml with no desired_state key was treated as running: %v", f.starts) } } // --- The §8.1 table, every row, in one place ---------------------------------------------------- func TestIsBootOrphan_DecisionTable(t *testing.T) { cases := []struct { name string desired string containers int state stacks.ContainerState want bool }{ {"stopped/no containers", stacks.DesiredStateStopped, 0, stacks.StateStopped, false}, {"stopped/down containers", stacks.DesiredStateStopped, 2, stacks.StateExited, false}, {"stopped/running", stacks.DesiredStateStopped, 2, stacks.StateRunning, false}, {"running/no containers", stacks.DesiredStateRunning, 0, stacks.StateStopped, true}, {"running/down containers", stacks.DesiredStateRunning, 2, stacks.StateExited, true}, {"running/degraded", stacks.DesiredStateRunning, 2, stacks.StateDegraded, true}, {"running/up", stacks.DesiredStateRunning, 2, stacks.StateRunning, false}, {"absent/no containers", stacks.DesiredStateUnknown, 0, stacks.StateStopped, false}, {"absent/down containers", stacks.DesiredStateUnknown, 2, stacks.StateExited, true}, {"absent/up", stacks.DesiredStateUnknown, 2, stacks.StateRunning, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { s := stacks.Stack{ Name: "app", Deployed: true, State: tc.state, Containers: make([]stacks.ContainerInfo, tc.containers), AppConfig: &stacks.AppConfig{Deployed: true, DesiredState: tc.desired}, } if got := isBootOrphan(s); got != tc.want { t.Fatalf("isBootOrphan(desired=%q containers=%d state=%s) = %v, want %v", tc.desired, tc.containers, tc.state, got, tc.want) } }) } } func TestIsBootOrphan_ExistingGuardsSurviveTheRewrite(t *testing.T) { // Protected and Deploying were guards before R-166 and must still be, at the strongest desired // state available — the rewrite reordered the terms, and a reorder is exactly how a guard gets // dropped without anyone noticing. base := func() stacks.Stack { return withDesired(vanished("traefik"), stacks.DesiredStateRunning) } protected := base() protected.Protected = true if isBootOrphan(protected) { t.Fatal("a PROTECTED stack became a boot orphan — the base-stack self-heal owns those") } deploying := base() deploying.Deploying = true if isBootOrphan(deploying) { t.Fatal("a DEPLOYING stack became a boot orphan — mid-deploy is not a fault") } notDeployed := base() notDeployed.Deployed = false if isBootOrphan(notDeployed) { t.Fatal("a stack that is not deployed became a boot orphan") } } // --- Scenario G — the two recoveries do not fight ------------------------------------------------ func TestReconcile_AppAlreadyRestartedByTheMarker_IsNotAlsoAnOrphan(t *testing.T) { // §8.4's REPORTING requirement. The app-stop marker's Recover runs to completion before this // sweep is launched, so by the time the reconciler looks, the app it restarted is UP. It must // therefore not appear as a candidate at all — an app the marker already explained must not also // be reported as an unexplained boot orphan, or one fault reads as two. restoredByMarker := withDesired(stacks.Stack{ Name: "immich", Deployed: true, State: stacks.StateRunning, Containers: []stacks.ContainerInfo{{Name: "immich-server", State: stacks.StateRunning}}, }, stacks.DesiredStateRunning) f, res := runSweep(t, []stacks.Stack{restoredByMarker}) if len(res.Candidates) != 0 { t.Fatalf("an app the marker had already restarted was ALSO reported as a boot orphan: %v", res.Candidates) } if n := f.starts["immich"]; n != 0 { t.Fatalf("the app was started a second time (%d) — one fault, one start", n) } }