controller v0.67.3: startup recreate of boot-stale drive-backed apps

Completes guest-reboot convergence. driveGateLoop runs recreateBootStaleApps once
at startup: deployed drive-backed apps whose drive is present (BoundUnderParent)
and whose containers started recently (fresh guest boot, not a controller-only
restart) are recreated (down+up) onto the re-propagated drive. Paired with agent
v0.35.0's drive re-propagation.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-15 17:40:19 +02:00
parent 3adfa41a09
commit 91a6dcfa75
2 changed files with 88 additions and 1 deletions
+76 -1
View File
@@ -39,6 +39,28 @@ func agentWhere(registeredPath string) string {
return "/mnt/" + name
}
// stackStartedRecently heuristically reports whether the stack's containers started within ~5 minutes
// (i.e. a fresh guest boot, not a long-running app across a controller-only restart) — read from docker's
// human "Up X …" status string. Used to limit the startup boot-stale recreate to the guest-reboot case.
func (s *Server) stackStartedRecently(name string, _ time.Duration) bool {
st, ok := s.stackMgr.GetStack(name)
if !ok {
return false
}
for _, c := range st.Containers {
status := strings.ToLower(c.Status)
if strings.Contains(status, "second") || strings.Contains(status, "about a minute") {
return true
}
for _, m := range []string{"up 1 minute", "up 2 minute", "up 3 minute", "up 4 minute"} {
if strings.Contains(status, m) {
return true
}
}
}
return false
}
// appsOnStoragePath returns the deployed stack names whose HDD_PATH equals the given (stable) storage
// path — the apps that depend on that drive.
func (s *Server) appsOnStoragePath(storagePath string) []string {
@@ -176,8 +198,12 @@ func (s *Server) ReconcileDriveGates() {
}
// driveGateLoop runs ReconcileDriveGates on a timer (the periodic absent/return detector — the slice-8C
// watchdog was retired). Started as a goroutine at server startup.
// watchdog was retired). Started as a goroutine at server startup. The FIRST action is a one-time
// startup recreate (see recreateBootStaleApps) to converge a guest reboot deterministically, then the
// periodic gate.
func (s *Server) driveGateLoop() {
s.recreateBootStaleApps()
s.ReconcileDriveGates()
t := time.NewTicker(30 * time.Second)
defer t.Stop()
for range t.C {
@@ -185,6 +211,55 @@ func (s *Server) driveGateLoop() {
}
}
// recreateBootStaleApps converges a GUEST REBOOT deterministically. On a guest reboot docker auto-starts
// the app containers (restart:unless-stopped) potentially BEFORE the agent has re-propagated the drive
// under the parent — so those containers bind the empty fail-closed stable dir (and the non-recursive
// parent bind + leaf-bind pinning means they never pick up the later propagation in their own ns). This
// runs ONCE at controller startup (the controller itself restarts with the guest): for every deployed
// drive-backed app whose drive is NOW present (BoundUnderParent) AND whose containers started recently
// (a fresh guest boot, not a long-running app across a controller-only restart), it recreates the app
// (Stop=down + Start=up) so it binds the populated path. Apps whose drive is still absent are left to the
// normal gate (stop→return→restart). Best-effort.
func (s *Server) recreateBootStaleApps() {
if s.settings == nil || s.stackMgr == nil {
return
}
agent, err := s.agentClient()
if err != nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
resp, derr := agent.Disks(ctx)
cancel()
if derr != nil {
return
}
presentStable := map[string]bool{}
for _, d := range resp.Disks {
if d.GuestPath != "" && d.BoundUnderParent {
presentStable[d.GuestPath] = true
}
}
for _, st := range s.stackMgr.GetStacks() {
cfg := s.stackMgr.LoadAppConfigByName(st.Name)
if cfg == nil {
continue
}
hdd := cfg.Env["HDD_PATH"]
if hdd == "" || !strings.HasPrefix(hdd, StableParentDir+"/") || !presentStable[hdd] {
continue
}
if !s.stackStartedRecently(st.Name, 5*time.Minute) {
continue // long-running across a controller-only restart — don't bounce it
}
s.logger.Printf("[INFO] [gate] startup: recreating drive-backed app %s onto its (re-propagated) drive %s", st.Name, hdd)
_ = s.stackMgr.StopStack(st.Name)
if serr := s.stackMgr.StartStack(st.Name); serr != nil {
s.logger.Printf("[WARN] [gate] startup recreate %s: %v", st.Name, serr)
}
}
}
// ---- H1 endpoints (the UI's settings.js calls these; previously 404/unrouted) -----------------
// handleStorageDisconnect EJECTS a drive without restart: stop its apps (gate-stopped), agent-detach the