controller v0.67.3: startup recreate of boot-stale drive-backed apps
Completes guest-reboot convergence. driveGateLoop runs recreateBootStaleApps once at startup: deployed drive-backed apps whose drive is present (BoundUnderParent) and whose containers started recently (fresh guest boot, not a controller-only restart) are recreated (down+up) onto the re-propagated drive. Paired with agent v0.35.0's drive re-propagation. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -39,6 +39,28 @@ func agentWhere(registeredPath string) string {
|
||||
return "/mnt/" + name
|
||||
}
|
||||
|
||||
// stackStartedRecently heuristically reports whether the stack's containers started within ~5 minutes
|
||||
// (i.e. a fresh guest boot, not a long-running app across a controller-only restart) — read from docker's
|
||||
// human "Up X …" status string. Used to limit the startup boot-stale recreate to the guest-reboot case.
|
||||
func (s *Server) stackStartedRecently(name string, _ time.Duration) bool {
|
||||
st, ok := s.stackMgr.GetStack(name)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
for _, c := range st.Containers {
|
||||
status := strings.ToLower(c.Status)
|
||||
if strings.Contains(status, "second") || strings.Contains(status, "about a minute") {
|
||||
return true
|
||||
}
|
||||
for _, m := range []string{"up 1 minute", "up 2 minute", "up 3 minute", "up 4 minute"} {
|
||||
if strings.Contains(status, m) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// appsOnStoragePath returns the deployed stack names whose HDD_PATH equals the given (stable) storage
|
||||
// path — the apps that depend on that drive.
|
||||
func (s *Server) appsOnStoragePath(storagePath string) []string {
|
||||
@@ -176,8 +198,12 @@ func (s *Server) ReconcileDriveGates() {
|
||||
}
|
||||
|
||||
// driveGateLoop runs ReconcileDriveGates on a timer (the periodic absent/return detector — the slice-8C
|
||||
// watchdog was retired). Started as a goroutine at server startup.
|
||||
// watchdog was retired). Started as a goroutine at server startup. The FIRST action is a one-time
|
||||
// startup recreate (see recreateBootStaleApps) to converge a guest reboot deterministically, then the
|
||||
// periodic gate.
|
||||
func (s *Server) driveGateLoop() {
|
||||
s.recreateBootStaleApps()
|
||||
s.ReconcileDriveGates()
|
||||
t := time.NewTicker(30 * time.Second)
|
||||
defer t.Stop()
|
||||
for range t.C {
|
||||
@@ -185,6 +211,55 @@ func (s *Server) driveGateLoop() {
|
||||
}
|
||||
}
|
||||
|
||||
// recreateBootStaleApps converges a GUEST REBOOT deterministically. On a guest reboot docker auto-starts
|
||||
// the app containers (restart:unless-stopped) potentially BEFORE the agent has re-propagated the drive
|
||||
// under the parent — so those containers bind the empty fail-closed stable dir (and the non-recursive
|
||||
// parent bind + leaf-bind pinning means they never pick up the later propagation in their own ns). This
|
||||
// runs ONCE at controller startup (the controller itself restarts with the guest): for every deployed
|
||||
// drive-backed app whose drive is NOW present (BoundUnderParent) AND whose containers started recently
|
||||
// (a fresh guest boot, not a long-running app across a controller-only restart), it recreates the app
|
||||
// (Stop=down + Start=up) so it binds the populated path. Apps whose drive is still absent are left to the
|
||||
// normal gate (stop→return→restart). Best-effort.
|
||||
func (s *Server) recreateBootStaleApps() {
|
||||
if s.settings == nil || s.stackMgr == nil {
|
||||
return
|
||||
}
|
||||
agent, err := s.agentClient()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
resp, derr := agent.Disks(ctx)
|
||||
cancel()
|
||||
if derr != nil {
|
||||
return
|
||||
}
|
||||
presentStable := map[string]bool{}
|
||||
for _, d := range resp.Disks {
|
||||
if d.GuestPath != "" && d.BoundUnderParent {
|
||||
presentStable[d.GuestPath] = true
|
||||
}
|
||||
}
|
||||
for _, st := range s.stackMgr.GetStacks() {
|
||||
cfg := s.stackMgr.LoadAppConfigByName(st.Name)
|
||||
if cfg == nil {
|
||||
continue
|
||||
}
|
||||
hdd := cfg.Env["HDD_PATH"]
|
||||
if hdd == "" || !strings.HasPrefix(hdd, StableParentDir+"/") || !presentStable[hdd] {
|
||||
continue
|
||||
}
|
||||
if !s.stackStartedRecently(st.Name, 5*time.Minute) {
|
||||
continue // long-running across a controller-only restart — don't bounce it
|
||||
}
|
||||
s.logger.Printf("[INFO] [gate] startup: recreating drive-backed app %s onto its (re-propagated) drive %s", st.Name, hdd)
|
||||
_ = s.stackMgr.StopStack(st.Name)
|
||||
if serr := s.stackMgr.StartStack(st.Name); serr != nil {
|
||||
s.logger.Printf("[WARN] [gate] startup recreate %s: %v", st.Name, serr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- H1 endpoints (the UI's settings.js calls these; previously 404/unrouted) -----------------
|
||||
|
||||
// handleStorageDisconnect EJECTS a drive without restart: stop its apps (gate-stopped), agent-detach the
|
||||
|
||||
Reference in New Issue
Block a user