From 91a6dcfa7538f2e4ed7aca1176f61e30cb8d6204 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Mon, 15 Jun 2026 17:40:19 +0200 Subject: [PATCH] controller v0.67.3: startup recreate of boot-stale drive-backed apps Completes guest-reboot convergence. driveGateLoop runs recreateBootStaleApps once at startup: deployed drive-backed apps whose drive is present (BoundUnderParent) and whose containers started recently (fresh guest boot, not a controller-only restart) are recreated (down+up) onto the re-propagated drive. Paired with agent v0.35.0's drive re-propagation. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 12 ++++ controller/internal/web/intermediary.go | 77 ++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 789b143..9b49836 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ ## Changelog +### v0.67.3 — gate: startup recreate of boot-stale drive-backed apps (2026-06-15) + +Completes guest-reboot convergence (caught in the live migration). On a guest reboot docker auto-starts +the app containers (restart:unless-stopped) potentially BEFORE the agent re-propagates the drive under +the parent, so they bind the empty fail-closed stable dir and (leaf-bind pinning) never pick up the +later propagation. `driveGateLoop` now runs a one-time `recreateBootStaleApps` at startup (the controller +restarts with the guest): for each deployed drive-backed app whose drive is NOW present +(BoundUnderParent) and whose containers started recently (a fresh boot, not a controller-only restart — +`stackStartedRecently`), it recreates the app (down+up) onto the populated path. Apps whose drive is +still absent are left to the normal stop→return→restart gate. Paired with agent v0.35.0 (the drive +re-propagation). + ### v0.67.2 — gate: key "present" on BoundUnderParent (reboot convergence) (2026-06-15) The drive-absent gate now treats a stable path as usable only when the agent reports it BOUND UNDER THE diff --git a/controller/internal/web/intermediary.go b/controller/internal/web/intermediary.go index 95d5b30..c93ec84 100644 --- a/controller/internal/web/intermediary.go +++ b/controller/internal/web/intermediary.go @@ -39,6 +39,28 @@ func agentWhere(registeredPath string) string { return "/mnt/" + name } +// stackStartedRecently heuristically reports whether the stack's containers started within ~5 minutes +// (i.e. a fresh guest boot, not a long-running app across a controller-only restart) — read from docker's +// human "Up X …" status string. Used to limit the startup boot-stale recreate to the guest-reboot case. +func (s *Server) stackStartedRecently(name string, _ time.Duration) bool { + st, ok := s.stackMgr.GetStack(name) + if !ok { + return false + } + for _, c := range st.Containers { + status := strings.ToLower(c.Status) + if strings.Contains(status, "second") || strings.Contains(status, "about a minute") { + return true + } + for _, m := range []string{"up 1 minute", "up 2 minute", "up 3 minute", "up 4 minute"} { + if strings.Contains(status, m) { + return true + } + } + } + return false +} + // appsOnStoragePath returns the deployed stack names whose HDD_PATH equals the given (stable) storage // path — the apps that depend on that drive. func (s *Server) appsOnStoragePath(storagePath string) []string { @@ -176,8 +198,12 @@ func (s *Server) ReconcileDriveGates() { } // driveGateLoop runs ReconcileDriveGates on a timer (the periodic absent/return detector — the slice-8C -// watchdog was retired). Started as a goroutine at server startup. +// watchdog was retired). Started as a goroutine at server startup. The FIRST action is a one-time +// startup recreate (see recreateBootStaleApps) to converge a guest reboot deterministically, then the +// periodic gate. func (s *Server) driveGateLoop() { + s.recreateBootStaleApps() + s.ReconcileDriveGates() t := time.NewTicker(30 * time.Second) defer t.Stop() for range t.C { @@ -185,6 +211,55 @@ func (s *Server) driveGateLoop() { } } +// recreateBootStaleApps converges a GUEST REBOOT deterministically. On a guest reboot docker auto-starts +// the app containers (restart:unless-stopped) potentially BEFORE the agent has re-propagated the drive +// under the parent — so those containers bind the empty fail-closed stable dir (and the non-recursive +// parent bind + leaf-bind pinning means they never pick up the later propagation in their own ns). This +// runs ONCE at controller startup (the controller itself restarts with the guest): for every deployed +// drive-backed app whose drive is NOW present (BoundUnderParent) AND whose containers started recently +// (a fresh guest boot, not a long-running app across a controller-only restart), it recreates the app +// (Stop=down + Start=up) so it binds the populated path. Apps whose drive is still absent are left to the +// normal gate (stop→return→restart). Best-effort. +func (s *Server) recreateBootStaleApps() { + if s.settings == nil || s.stackMgr == nil { + return + } + agent, err := s.agentClient() + if err != nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + resp, derr := agent.Disks(ctx) + cancel() + if derr != nil { + return + } + presentStable := map[string]bool{} + for _, d := range resp.Disks { + if d.GuestPath != "" && d.BoundUnderParent { + presentStable[d.GuestPath] = true + } + } + for _, st := range s.stackMgr.GetStacks() { + cfg := s.stackMgr.LoadAppConfigByName(st.Name) + if cfg == nil { + continue + } + hdd := cfg.Env["HDD_PATH"] + if hdd == "" || !strings.HasPrefix(hdd, StableParentDir+"/") || !presentStable[hdd] { + continue + } + if !s.stackStartedRecently(st.Name, 5*time.Minute) { + continue // long-running across a controller-only restart — don't bounce it + } + s.logger.Printf("[INFO] [gate] startup: recreating drive-backed app %s onto its (re-propagated) drive %s", st.Name, hdd) + _ = s.stackMgr.StopStack(st.Name) + if serr := s.stackMgr.StartStack(st.Name); serr != nil { + s.logger.Printf("[WARN] [gate] startup recreate %s: %v", st.Name, serr) + } + } +} + // ---- H1 endpoints (the UI's settings.js calls these; previously 404/unrouted) ----------------- // handleStorageDisconnect EJECTS a drive without restart: stop its apps (gate-stopped), agent-detach the