controller v0.68.0: storage lifecycle on intermediary model (H2/H3/M1/M3 + boot-id)

H2 decommission UI button (migrate / anyway); H3 one-click re-enroll of a decommissioned drive; M1 default reassignment (auto-promote + block-if-none); M3 migrate re-asserts 2775 setgid on userdata dirs; deterministic guest-reboot recreate via agent boot_id (replaces the timed sample). Fixes the {path}/{where} H1 JS bug. Non-hollow tests + companions. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-15 19:26:02 +02:00
parent bc41acf4da
commit 42f69dadda
10 changed files with 293 additions and 46 deletions
@@ -43,23 +43,51 @@ func agentWhere(registeredPath string) string {
 // stackStartedRecently heuristically reports whether the stack's containers started within ~5 minutes
 // (i.e. a fresh guest boot, not a long-running app across a controller-only restart) — read from docker's
 // human "Up X …" status string. Used to limit the startup boot-stale recreate to the guest-reboot case.
-func (s *Server) stackStartedRecently(name string, _ time.Duration) bool {
-	st, ok := s.stackMgr.GetStack(name)
-	if !ok {
+// shouldRecreateOnBoot is the PURE decision for the boot-id recreate: on a fresh guest boot, which
+// deployed drive-backed app to recreate onto its (re-propagated) drive. Recreate every app whose drive
+// is present (BoundUnderParent) AND that docker BROUGHT BACK on this boot (State not stopped/not_deployed
+// — i.e. its containers exist). A cleanly user-stopped app (compose down → no containers → Stopped) is
+// respected; a gate-stopped app is the gate's job (StoppedStacks). Deterministic — depends only on
+// deployed + drive-present + state, NOT a fragile container-uptime sample (the old `stackStartedRecently`
+// missed an app that was healthy-but-stale or stopped at the sample instant).
+func shouldRecreateOnBoot(deployed bool, hdd string, state stacks.ContainerState, presentStable map[string]bool) bool {
+	if !deployed || hdd == "" || !strings.HasPrefix(hdd, StableParentDir+"/") || !presentStable[hdd] {
 		return false
 	}
-	for _, c := range st.Containers {
-		status := strings.ToLower(c.Status)
-		if strings.Contains(status, "second") || strings.Contains(status, "about a minute") {
-			return true
+	return state != stacks.StateStopped && state != stacks.StateNotDeployed
+}
+
+// defaultPromotionTarget decides M1 (never leave zero default). If the path being decommissioned is NOT
+// the current default → ("", false): nothing to do. If it IS the default → pick a promotion target from
+// the OTHER schedulable, non-decommissioned paths (preferring `migratedTo` — the migrate-then-decommission
+// target — when valid); ("", true) = MUST BLOCK (no other usable drive exists). Pure → unit-testable.
+func defaultPromotionTarget(paths []settings.StoragePath, decommissioning, migratedTo string) (target string, mustBlock bool) {
+	isDefault := false
+	for _, sp := range paths {
+		if sp.Path == decommissioning && sp.IsDefault {
+			isDefault = true
 		}
-		for _, m := range []string{"up 1 minute", "up 2 minute", "up 3 minute", "up 4 minute"} {
-			if strings.Contains(status, m) {
+	}
+	if !isDefault {
+		return "", false
+	}
+	usable := func(p string) bool {
+		for _, sp := range paths {
+			if sp.Path == p && sp.Path != decommissioning && sp.Schedulable && !sp.Decommissioned {
 				return true
 			}
 		}
+		return false
 	}
-	return false
+	if migratedTo != "" && usable(migratedTo) {
+		return migratedTo, false
+	}
+	for _, sp := range paths {
+		if sp.Path != decommissioning && sp.Schedulable && !sp.Decommissioned {
+			return sp.Path, false
+		}
+	}
+	return "", true // no other usable drive → block the decommission
 }

 // appsOnStoragePath returns the deployed stack names whose HDD_PATH equals the given (stable) storage
@@ -204,14 +232,14 @@ func (s *Server) ReconcileDriveGates() {
 // periodic gate.
 func (s *Server) driveGateLoop() {
 	// Wait for the stack scan to complete (GetStacks empty at NewServer time) and the agent to come up,
-	// so the one-time boot-stale recreate sees the real deployed apps + drive state. Bounded poll.
+	// so the boot-id recreate sees the real deployed apps + drive state. Bounded poll.
 	for i := 0; i < 30; i++ {
 		if s.stackMgr != nil && len(s.stackMgr.GetStacks()) > 0 {
 			break
 		}
 		time.Sleep(time.Second)
 	}
-	s.recreateBootStaleApps()
+	s.processGuestBootChange()
 	s.ReconcileDriveGates()
 	t := time.NewTicker(30 * time.Second)
 	defer t.Stop()
@@ -220,16 +248,16 @@ func (s *Server) driveGateLoop() {
 	}
 }

-// recreateBootStaleApps converges a GUEST REBOOT deterministically. On a guest reboot docker auto-starts
-// the app containers (restart:unless-stopped) potentially BEFORE the agent has re-propagated the drive
-// under the parent — so those containers bind the empty fail-closed stable dir (and the non-recursive
-// parent bind + leaf-bind pinning means they never pick up the later propagation in their own ns). This
-// runs ONCE at controller startup (the controller itself restarts with the guest): for every deployed
-// drive-backed app whose drive is NOW present (BoundUnderParent) AND whose containers started recently
-// (a fresh guest boot, not a long-running app across a controller-only restart), it recreates the app
-// (Stop=down + Start=up) so it binds the populated path. Apps whose drive is still absent are left to the
-// normal gate (stop→return→restart). Best-effort.
-func (s *Server) recreateBootStaleApps() {
+// processGuestBootChange converges a GUEST REBOOT DETERMINISTICALLY. On a guest reboot docker auto-starts
+// the app containers (restart:unless-stopped) potentially BEFORE the agent re-propagates the drive under
+// the parent — so they bind the empty fail-closed stable dir and (non-recursive parent bind + leaf-bind
+// pinning) never pick up the later propagation. The agent reports a `guest_boot_id` that changes on every
+// guest boot but is stable across a controller-only restart; the controller persists the last-seen value.
+// When it changes (the controller restarts WITH the guest), this recreates every deployed drive-backed
+// app whose drive is present + that docker brought back (`shouldRecreateOnBoot`) onto the populated path,
+// then persists the new boot-id. Apps on a still-absent drive are handled by the normal gate
+// (stop→return→restart). Replaces the old fragile container-uptime sample. Best-effort.
+func (s *Server) processGuestBootChange() {
 	if s.settings == nil || s.stackMgr == nil {
 		return
 	}
@@ -243,6 +271,9 @@ func (s *Server) recreateBootStaleApps() {
 	if derr != nil {
 		return
 	}
+	if resp.GuestBootID == "" || resp.GuestBootID == s.settings.GetLastGuestBootID() {
+		return // no boot-id, or unchanged (controller-only restart) → no recreate
+	}
 	presentStable := map[string]bool{}
 	for _, d := range resp.Disks {
 		if d.GuestPath != "" && d.BoundUnderParent {
@@ -254,25 +285,18 @@ func (s *Server) recreateBootStaleApps() {
 		if cfg == nil {
 			continue
 		}
-		hdd := cfg.Env["HDD_PATH"]
-		if hdd == "" || !strings.HasPrefix(hdd, StableParentDir+"/") || !presentStable[hdd] {
+		if !shouldRecreateOnBoot(cfg.Deployed, cfg.Env["HDD_PATH"], st.State, presentStable) {
 			continue
 		}
-		// Recreate when the app is boot-stale or not cleanly running: recently-started (it likely came up
-		// on the empty bind before the drive was re-propagated) OR currently exited/restarting/unhealthy
-		// (came up wrong and bailed). SKIP a healthy long-running app (no bounce on a controller-only
-		// restart) and a cleanly user-Stopped app (respect the user's intent).
-		needs := s.stackStartedRecently(st.Name, 5*time.Minute) ||
-			st.State == stacks.StateExited || st.State == stacks.StateRestarting || st.State == stacks.StateUnhealthy
-		if !needs {
-			continue
-		}
-		s.logger.Printf("[INFO] [gate] startup: recreating drive-backed app %s (state=%s) onto its drive %s", st.Name, st.State, hdd)
+		s.logger.Printf("[INFO] [gate] boot %s: recreating drive-backed app %s (state=%s) onto %s", resp.GuestBootID, st.Name, st.State, cfg.Env["HDD_PATH"])
 		_ = s.stackMgr.StopStack(st.Name)
 		if serr := s.stackMgr.StartStack(st.Name); serr != nil {
-			s.logger.Printf("[WARN] [gate] startup recreate %s: %v", st.Name, serr)
+			s.logger.Printf("[WARN] [gate] boot recreate %s: %v", st.Name, serr)
 		}
 	}
+	if serr := s.settings.SetLastGuestBootID(resp.GuestBootID); serr != nil {
+		s.logger.Printf("[WARN] [gate] persist boot-id: %v", serr)
+	}
 }

 // ---- H1 endpoints (the UI's settings.js calls these; previously 404/unrouted) -----------------
@@ -302,17 +326,30 @@ func (s *Server) handleStorageDisconnect(w http.ResponseWriter, r *http.Request)

 // handleStorageReconnect re-attaches a returned drive without restart: agent-attach the felhom-data bind
 // under the parent (live), restart the gate-stopped apps, clear the disconnected mark.
+// handleStorageReconnect re-enrolls a DISCONNECTED **or** DECOMMISSIONED drive in one click (H3):
+// (for a decommissioned drive) clears the soft marker + restores schedulable, then re-attaches the
+// felhom-data under the parent (agent, raw path, no reboot) and restarts the gate-stopped apps. The data
+// is intact — decommission is non-destructive — so the apps resolve once the bind is live.
 func (s *Server) handleStorageReconnect(w http.ResponseWriter, r *http.Request) {
 	where, ok := s.gateWhere(w, r)
 	if !ok {
 		return
 	}
+	decommissioned := s.settings.IsDecommissioned(where)
 	var stopped []string
 	for _, sp := range s.settings.GetStoragePaths() {
 		if sp.Path == where {
 			stopped = sp.StoppedStacks
 		}
 	}
+	if decommissioned {
+		if _, cerr := s.reEnrollClearMarker(where); cerr != nil {
+			writeDiskJSON(w, http.StatusInternalServerError, false, cerr.Error(), nil)
+			return
+		}
+		// decommission-anyway stopped the apps WITHOUT persisting StoppedStacks — re-discover them.
+		stopped = s.appsOnStoragePath(where)
+	}
 	agent, err := s.agentClient()
 	if err != nil {
 		writeDiskJSON(w, http.StatusServiceUnavailable, false, err.Error(), nil)
@@ -328,7 +365,7 @@ func (s *Server) handleStorageReconnect(w http.ResponseWriter, r *http.Request)
 		return
 	}
 	go s.SyncFileBrowserMounts()
-	writeDiskJSON(w, http.StatusOK, true, "", map[string]any{"where": where, "restarted": stopped})
+	writeDiskJSON(w, http.StatusOK, true, "", map[string]any{"where": where, "restarted": stopped, "reenrolled": decommissioned})
 }

 // handleStorageRestartApps restarts the gate-stopped apps on a path (without changing connection state) —