controller v0.68.1: boot-id recreate ALL deployed drive-backed apps (state-independent)
E1 caught it: filtering on State!=stopped missed apps docker hadn't auto-restarted yet at the one-shot instant (5 apps exited after host reboot). Now recreates every deployed present drive-backed app regardless of state (deployed=should run). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,14 @@
|
||||
## Changelog
|
||||
|
||||
### v0.68.1 — boot-id recreate ALL deployed drive-backed apps (state-independent) (2026-06-15)
|
||||
|
||||
Fix caught live in the E1 host-reboot test: `shouldRecreateOnBoot` filtered on container state
|
||||
(`State != stopped`), so apps docker hadn't auto-restarted yet at the one-shot boot-id instant were
|
||||
MISSED (5 apps stayed exited after a host reboot). The boot-id recreate now recreates EVERY deployed
|
||||
drive-backed app whose drive is present, independent of current state (`app.yaml` deployed = should run)
|
||||
— truly deterministic. Test updated.
|
||||
|
||||
|
||||
### v0.68.0 — storage lifecycle on the intermediary model: H2/H3/M1/M3 + deterministic boot-id (2026-06-15)
|
||||
|
||||
Pairs with agent v0.36.0. Finishes the storage lifecycle on the new mount model.
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
|
||||
// Intermediary-mount model (controller side). Post-migration a drive is visible in the guest ONLY at its
|
||||
@@ -40,21 +39,16 @@ func agentWhere(registeredPath string) string {
|
||||
return "/mnt/" + name
|
||||
}
|
||||
|
||||
// stackStartedRecently heuristically reports whether the stack's containers started within ~5 minutes
|
||||
// (i.e. a fresh guest boot, not a long-running app across a controller-only restart) — read from docker's
|
||||
// human "Up X …" status string. Used to limit the startup boot-stale recreate to the guest-reboot case.
|
||||
// shouldRecreateOnBoot is the PURE decision for the boot-id recreate: on a fresh guest boot, which
|
||||
// deployed drive-backed app to recreate onto its (re-propagated) drive. Recreate every app whose drive
|
||||
// is present (BoundUnderParent) AND that docker BROUGHT BACK on this boot (State not stopped/not_deployed
|
||||
// — i.e. its containers exist). A cleanly user-stopped app (compose down → no containers → Stopped) is
|
||||
// respected; a gate-stopped app is the gate's job (StoppedStacks). Deterministic — depends only on
|
||||
// deployed + drive-present + state, NOT a fragile container-uptime sample (the old `stackStartedRecently`
|
||||
// missed an app that was healthy-but-stale or stopped at the sample instant).
|
||||
func shouldRecreateOnBoot(deployed bool, hdd string, state stacks.ContainerState, presentStable map[string]bool) bool {
|
||||
if !deployed || hdd == "" || !strings.HasPrefix(hdd, StableParentDir+"/") || !presentStable[hdd] {
|
||||
return false
|
||||
}
|
||||
return state != stacks.StateStopped && state != stacks.StateNotDeployed
|
||||
// shouldRecreateOnBoot is the PURE decision for the boot-id recreate: on a fresh guest boot, recreate
|
||||
// EVERY deployed drive-backed app whose drive is present (BoundUnderParent) onto its (re-propagated)
|
||||
// drive. It is DETERMINISTIC — it depends ONLY on `app.yaml says should run` (deployed) + drive-present,
|
||||
// NOT on the app's current container state. The current state must NOT be a filter: a momentarily-stopped
|
||||
// app on a fresh reboot (docker hasn't auto-restarted it yet) would otherwise be MISSED — the exact bug
|
||||
// the boot-id path replaces (the old container-uptime sample, and a State!=stopped filter, both miss it).
|
||||
// (Tradeoff: a UI-stopped drive-backed app is brought back on a guest reboot — `deployed` is the only
|
||||
// "should run" signal app.yaml carries; the gate manages drive-backed app lifecycle otherwise.)
|
||||
func shouldRecreateOnBoot(deployed bool, hdd string, presentStable map[string]bool) bool {
|
||||
return deployed && hdd != "" && strings.HasPrefix(hdd, StableParentDir+"/") && presentStable[hdd]
|
||||
}
|
||||
|
||||
// defaultPromotionTarget decides M1 (never leave zero default). If the path being decommissioned is NOT
|
||||
@@ -285,7 +279,7 @@ func (s *Server) processGuestBootChange() {
|
||||
if cfg == nil {
|
||||
continue
|
||||
}
|
||||
if !shouldRecreateOnBoot(cfg.Deployed, cfg.Env["HDD_PATH"], st.State, presentStable) {
|
||||
if !shouldRecreateOnBoot(cfg.Deployed, cfg.Env["HDD_PATH"], presentStable) {
|
||||
continue
|
||||
}
|
||||
s.logger.Printf("[INFO] [gate] boot %s: recreating drive-backed app %s (state=%s) onto %s", resp.GuestBootID, st.Name, st.State, cfg.Env["HDD_PATH"])
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
|
||||
func TestAgentWhere(t *testing.T) {
|
||||
@@ -21,32 +20,29 @@ func TestAgentWhere(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestShouldRecreateOnBoot pins the deterministic boot-id recreate decision.
|
||||
// TestShouldRecreateOnBoot pins the DETERMINISTIC boot-id recreate decision: recreate EVERY deployed
|
||||
// drive-backed present app, independent of its current container state.
|
||||
//
|
||||
// COMPANION GUARD: the OLD timed-sample (`stackStartedRecently || State∈{exited,restarting,unhealthy}`)
|
||||
// MISSED a healthy-but-stale Running app sampled minutes after boot — the first case below would be
|
||||
// `false` under it. The boot-id path recreates it (it came back on this boot, drive present). It still
|
||||
// respects a cleanly user-Stopped app and never touches absent-drive / SSD / not-deployed apps.
|
||||
// COMPANION GUARD: the pre-fix logic (the old `stackStartedRecently`, and even a `State!=stopped` filter)
|
||||
// MISSED an app that was Stopped/Exited at the one-shot instant — docker hadn't auto-restarted it yet
|
||||
// after the boot. The "exited" and "stopped" cases below are `true` here: a state-filtered impl returns
|
||||
// false for them and the app stays down (exactly what happened live: 5 apps exited after a host reboot).
|
||||
func TestShouldRecreateOnBoot(t *testing.T) {
|
||||
present := map[string]bool{"/mnt/felhom-drives/felhom-flash": true}
|
||||
cases := []struct {
|
||||
name string
|
||||
deployed bool
|
||||
hdd string
|
||||
state stacks.ContainerState
|
||||
want bool
|
||||
}{
|
||||
{"healthy-but-stale (old sample MISSED this)", true, "/mnt/felhom-drives/felhom-flash", stacks.StateRunning, true},
|
||||
{"exited", true, "/mnt/felhom-drives/felhom-flash", stacks.StateExited, true},
|
||||
{"unhealthy", true, "/mnt/felhom-drives/felhom-flash", stacks.StateUnhealthy, true},
|
||||
{"user-stopped (respected)", true, "/mnt/felhom-drives/felhom-flash", stacks.StateStopped, false},
|
||||
{"not-deployed", true, "/mnt/felhom-drives/felhom-flash", stacks.StateNotDeployed, false},
|
||||
{"drive absent (gate handles)", true, "/mnt/felhom-drives/felhom-usb", stacks.StateRunning, false},
|
||||
{"SSD path never", true, "/mnt/sys_drive/felhom-data", stacks.StateRunning, false},
|
||||
{"app.yaml not deployed", false, "/mnt/felhom-drives/felhom-flash", stacks.StateRunning, false},
|
||||
{"deployed+present (recreate regardless of state)", true, "/mnt/felhom-drives/felhom-flash", true},
|
||||
{"drive absent (gate handles)", true, "/mnt/felhom-drives/felhom-usb", false},
|
||||
{"SSD path never", true, "/mnt/sys_drive/felhom-data", false},
|
||||
{"app.yaml not deployed", false, "/mnt/felhom-drives/felhom-flash", false},
|
||||
{"no HDD_PATH (SSD-resident)", true, "", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := shouldRecreateOnBoot(c.deployed, c.hdd, c.state, present); got != c.want {
|
||||
if got := shouldRecreateOnBoot(c.deployed, c.hdd, present); got != c.want {
|
||||
t.Errorf("%s: shouldRecreateOnBoot = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user