package web import ( "context" "encoding/json" "net/http" "os" "path" "strings" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/agentapi" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // bootBindWait / bootBindPoll bound the readiness gate in processGuestBootChange. On a guest reboot // the agent re-binds each drive under the stable parent ~18s after boot (well after docker has tried // and permanently failed to auto-start the drive-backed apps); the wait must comfortably cover that // with margin, the poll be cheap. const ( bootBindWait = 120 * time.Second bootBindPoll = 2 * time.Second ) // driveBindLive reports whether `root` (a /mnt/felhom-drives/ stable path) is a REAL live // mountpoint in this process's own mount namespace — i.e. the agent's per-drive bind has actually // propagated into the controller's `/mnt` (rslave) view, which is exactly the moment docker can // (re)create an app whose volume binds under it. It reads /proc/self/mountinfo (pure Go, no syscall; // the mount point is always field index 4). Before the agent re-binds the drive post-boot, the stable // path is the empty fail-closed placeholder (a plain subdir of the parent, NOT its own mount) → false. // This is the load-bearing signal the old code lacked: it sampled the agent's host-side // BoundUnderParent ONCE during fast controller startup, racing the ~18s rebind. func driveBindLive(root string) bool { data, err := os.ReadFile("/proc/self/mountinfo") if err != nil { return false } for _, line := range strings.Split(string(data), "\n") { f := strings.Fields(line) if len(f) >= 5 && f[4] == root { return true } } return false } // pollLiveBinds waits (bounded by `wait`, polling every `poll`) for each path's drive bind to go LIVE, // returning the final liveness map. It breaks early once every path is live. bindLive/sleep/now are // injected so the readiness gate is deterministically unit-testable. A path that never goes live // within the window stays false (its apps are left to the normal drive gate). func pollLiveBinds(paths []string, bindLive func(string) bool, sleep func(time.Duration), now func() time.Time, wait, poll time.Duration) map[string]bool { live := make(map[string]bool, len(paths)) deadline := now().Add(wait) for { all := true for _, p := range paths { if live[p] { continue } if bindLive(p) { live[p] = true } else { all = false } } if all || !now().Before(deadline) { return live } sleep(poll) } } // Intermediary-mount model (controller side). Post-migration a drive is visible in the guest ONLY at its // STABLE path /mnt/felhom-drives/ (the host swaps the backing drive underneath it; see the agent's // internal/localapi/intermediary.go + SPIKE-intermediary-mount). So: // - the REGISTERED storage path + every app's HDD_PATH + FileBrowser source = the STABLE path; // - the AGENT still operates on the RAW /mnt/ host PVE mount (assign/attach/eject/decommission), // so controller→agent `where` is mapped back to raw via agentWhere(). // The drive-absent GATE stops + blocks apps when their drive vanishes and auto-restarts them when it // returns (host-side, no guest reboot). // StableParentDir is the permanent in-guest parent the agent binds drives under (mirrors // localapi.StableParentDir). const StableParentDir = "/mnt/felhom-drives" // stablePathForName maps a drive name to its registered stable in-guest path. func stablePathForName(name string) string { return StableParentDir + "/" + name } // agentWhere maps a registered storage path — stable /mnt/felhom-drives/ OR a legacy raw /mnt/ // — to the RAW /mnt/ host mount the agent operates on. Idempotent for an already-raw path // (path.Base drops the directory either way). func agentWhere(registeredPath string) string { name := path.Base(strings.TrimRight(registeredPath, "/")) if name == "" || name == "." || name == "/" { return registeredPath } return "/mnt/" + name } // shouldRecreateOnBoot is the PURE decision for the boot-id recreate: on a fresh guest boot, recreate // a deployed drive-backed app whose drive is present (BoundUnderParent) onto its (re-propagated) drive. // // It is DETERMINISTIC and it still does NOT filter on the app's current container STATE. That remains // load-bearing: a momentarily-stopped app on a fresh reboot (docker hasn't auto-restarted it yet, or // its create-time bind failed with RestartCount=0) must be recreated, and both the old // container-uptime sample and a `State != stopped` filter MISS exactly that case. Do not reintroduce // a state filter here. // // R-55: it DOES filter on whether the app still HAS containers, which is a different question and the // one that tells the truth about intent. This is R-52's `existing-Exited vs absent` distinction // (bootrecon.isBootOrphan), translated to this gate: // // - containers EXIST but are down → the guest went down under the app; docker's own records survive // the reboot, so this is a boot orphan → recreate. // - ZERO containers → a UI Stop is `compose down`, which REMOVES the containers. // Nothing else in the controller leaves a deployed app at zero containers. → the customer stopped // this on purpose → LEAVE IT ALONE. // // `deployed` cannot answer this: it is a deploy-lifecycle flag and stays true across a Stop. Before // R-55 the gate had no other signal and therefore silently undid a customer's Stop on every guest // reboot — including when apps were stopped deliberately to free resources for others, which is // precisely when resurrecting them is most harmful. // // The evidence is read from the snapshot taken BEFORE any recreate runs, because `recreate` itself // calls StopStack (`compose down`) and so destroys it. // // NOTE on the drive-absent gate: apps it stopped are also at zero containers, so they are skipped // here too. That is correct — they are recorded in StoragePath.StoppedStacks and restarted by // ReconcileDriveGates' `Return` branch, which runs on the same loop tick. Their recovery is that // path's job, not this one's. func shouldRecreateOnBoot(deployed bool, hdd string, presentStable map[string]bool, hasContainers bool) bool { return deployed && hdd != "" && strings.HasPrefix(hdd, StableParentDir+"/") && presentStable[hdd] && hasContainers } // defaultPromotionTarget decides M1 (never leave zero default). If the path being decommissioned is NOT // the current default → ("", false): nothing to do. If it IS the default → pick a promotion target from // the OTHER schedulable, non-decommissioned paths (preferring `migratedTo` — the migrate-then-decommission // target — when valid); ("", true) = MUST BLOCK (no other usable drive exists). Pure → unit-testable. func defaultPromotionTarget(paths []settings.StoragePath, decommissioning, migratedTo string) (target string, mustBlock bool) { isDefault := false for _, sp := range paths { if sp.Path == decommissioning && sp.IsDefault { isDefault = true } } if !isDefault { return "", false } usable := func(p string) bool { for _, sp := range paths { if sp.Path == p && sp.Path != decommissioning && sp.Schedulable && !sp.Decommissioned { return true } } return false } if migratedTo != "" && usable(migratedTo) { return migratedTo, false } for _, sp := range paths { if sp.Path != decommissioning && sp.Schedulable && !sp.Decommissioned { return sp.Path, false } } return "", true // no other usable drive → block the decommission } // appsOnStoragePath returns the deployed stack names whose HDD_PATH equals the given (stable) storage // path — the apps that depend on that drive. func (s *Server) appsOnStoragePath(storagePath string) []string { var names []string for _, st := range s.stackMgr.GetStacks() { if cfg := s.stackMgr.LoadAppConfigByName(st.Name); cfg != nil && cfg.Env["HDD_PATH"] == storagePath { names = append(names, st.Name) } } return names } // stopAppsOnPath stops every deployed app on the given storage path and returns their names (the // gate-stopped set — distinct from a user stop, which never enters this set). Best-effort per app. func (s *Server) stopAppsOnPath(storagePath string) []string { var stopped []string for _, name := range s.appsOnStoragePath(storagePath) { if err := s.stackMgr.StopStack(name); err != nil { s.logger.Printf("[WARN] [gate] stop %s on absent %s: %v", name, storagePath, err) continue } stopped = append(stopped, name) } return stopped } // restartStacks starts each named stack (the gate-stopped set on drive return). Best-effort per app. func (s *Server) restartStacks(names []string) { for _, name := range names { if err := s.stackMgr.StartStack(name); err != nil { s.logger.Printf("[WARN] [gate] restart %s: %v", name, err) } } } // gateAction is the reconcile's decision for one registered storage path. type gateAction struct { Path string Stop bool // drive ABSENT + not yet marked → stop apps, mark disconnected Return bool // drive RETURNED (present + currently disconnected) → re-attach, restart, clear Raw string // raw /mnt/ for the re-attach } // planDriveGates is the PURE decision core: given the registry + the agent's disk list, decide per path // whether to gate (stop) or un-gate (return). A drive is "present" iff a disk with a matching GuestPath // (stable) or MountPath (legacy raw) is State=="attached". Decommissioned paths are skipped (handled by // the decommission flow, not the transient gate). No side effects → unit-testable. func planDriveGates(paths []settings.StoragePath, disks []agentapi.DiskInfo) []gateAction { present := map[string]bool{} rawByPath := map[string]string{} for _, d := range disks { // A STABLE path is usable only when the drive's felhom-data is actually BOUND UNDER THE PARENT // (BoundUnderParent) — NOT merely when the raw drive is host-mounted (State==attached). This is // what makes the gate converge a reboot correctly: at boot the raw drive mounts early but the // agent binds it under the parent slightly later; until then the apps' stable-path binds are empty, // so the gate keeps them stopped and restarts (recreates) them once the bind is live. if d.GuestPath != "" { present[d.GuestPath] = present[d.GuestPath] || d.BoundUnderParent rawByPath[d.GuestPath] = d.MountPath } if d.MountPath != "" { // legacy raw path registered directly — host mount is the usable signal present[d.MountPath] = present[d.MountPath] || d.State == "attached" rawByPath[d.MountPath] = d.MountPath } } var actions []gateAction for _, sp := range paths { if sp.Decommissioned { continue } // NAS network storage is NOT a drive — the agent never lists it in /disks, so the drive-absent gate // would falsely see it "absent" (present[sp.Path]==false) and STOP its apps. A NAS blip is // recoverable (the agent's per-share liveness → a warning badge), never the drive stop-cascade. // Skip network paths here entirely (Scenario C: unreachable ≠ missing). if sp.IsNetwork() { continue } // ONLY gate EXTERNAL drives — those registered under the stable parent /mnt/felhom-drives/. // Internal SSD / system paths (e.g. /mnt/sys_drive/felhom-data) are always-present locals the agent // never reports as drives; gating them on "absence" would falsely stop/block their apps. (Legacy // raw /mnt/ external paths are present via the agent's MountPath during the transition and get // repointed under the parent by the migration.) if !strings.HasPrefix(sp.Path, StableParentDir+"/") { continue } switch { case !present[sp.Path] && !sp.Disconnected: actions = append(actions, gateAction{Path: sp.Path, Stop: true}) case present[sp.Path] && sp.Disconnected: actions = append(actions, gateAction{Path: sp.Path, Return: true, Raw: rawByPath[sp.Path]}) } } return actions } // ReconcileDriveGates enforces the drive-absent gate: an ABSENT registered drive gets its apps STOPPED + // recorded (disconnected); a RETURNED drive gets re-bound under the parent and its gate-stopped apps // restarted. Best-effort + idempotent — safe to call on a timer and on demand. func (s *Server) ReconcileDriveGates() { if s.settings == nil || s.stackMgr == nil { return } agent, err := s.agentClient() if err != nil { return } ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() resp, err := agent.Disks(ctx) if err != nil { return } for _, a := range planDriveGates(s.settings.GetStoragePaths(), resp.Disks) { switch { case a.Stop: stopped := s.stopAppsOnPath(a.Path) if err := s.settings.SetDisconnected(a.Path, true, stopped); err != nil { s.logger.Printf("[WARN] [gate] mark disconnected %s: %v", a.Path, err) } s.logger.Printf("[WARN] [gate] drive ABSENT %s — stopped+blocked %d app(s): %v", a.Path, len(stopped), stopped) go s.SyncFileBrowserMounts() case a.Return: if a.Raw != "" { if err := agent.GuestAttach(ctx, a.Raw); err != nil { s.logger.Printf("[WARN] [gate] re-attach %s (raw %s): %v", a.Path, a.Raw, err) } } var stopped []string for _, sp := range s.settings.GetStoragePaths() { if sp.Path == a.Path { stopped = sp.StoppedStacks } } s.restartStacks(stopped) if err := s.settings.ClearDisconnected(a.Path); err != nil { s.logger.Printf("[WARN] [gate] clear disconnected %s: %v", a.Path, err) } s.logger.Printf("[INFO] [gate] drive RETURNED %s — re-attached + restarted gate-stopped apps", a.Path) go s.SyncFileBrowserMounts() } } } // driveGateLoop runs ReconcileDriveGates on a timer (the periodic absent/return detector — the slice-8C // watchdog was retired). Started as a goroutine at server startup. The FIRST action is a one-time // startup recreate (see recreateBootStaleApps) to converge a guest reboot deterministically, then the // periodic gate. func (s *Server) driveGateLoop() { // Wait for the stack scan to complete (GetStacks empty at NewServer time) and the agent to come up, // so the boot-id recreate sees the real deployed apps + drive state. Bounded poll. for i := 0; i < 30; i++ { if s.stackMgr != nil && len(s.stackMgr.GetStacks()) > 0 { break } time.Sleep(time.Second) } s.processGuestBootChange() s.ReconcileDriveGates() t := time.NewTicker(30 * time.Second) defer t.Stop() for range t.C { // Re-run the boot-change converger every tick (not just once at startup): right after a GUEST // reboot the agent's local API may briefly be unreachable / its per-guest token stale, so the // single startup attempt can bail before reading the boot-id. It is idempotent (boot-id gated: // a no-op once the current boot has been converged), so retrying until the agent is reachable // is safe and is what makes guest-reboot recovery robust. s.processGuestBootChange() s.ReconcileDriveGates() } } // processGuestBootChange converges a GUEST REBOOT DETERMINISTICALLY. On a guest reboot docker auto-starts // the app containers (restart:unless-stopped) BEFORE the agent re-propagates the drive under the parent // (~18s later) — so the create-time bind of an app's volume fails (`mkdir …/userdata: permission denied` // on the empty fail-closed placeholder) and, being a create-time failure (RestartCount=0), is NEVER // retried → the app is stuck Exited forever even after the bind lands. The agent reports a `guest_boot_id` // that changes on every guest boot but is stable across a controller-only restart; the controller // persists the last-seen value. When it changes (the controller restarts WITH the guest), this: // 1. gathers the deployed drive-backed apps' stable drive paths, // 2. GATES on the REAL live in-guest bind — `pollLiveBinds`/`driveBindLive` wait (bounded) until each // drive's stable path is an actual mountpoint in the controller's own /mnt (rslave) view, which is // exactly when docker can recreate the app. (The old code sampled the agent's BoundUnderParent // ONCE during fast startup, raced the ~18s rebind, recreated nothing, and persisted the boot-id — // burning its one-shot. That is the bug this fixes.) // 3. recreates every deployed drive-backed app whose bind is now live (`shouldRecreateOnBoot` is // state-independent, so a stuck-Exited create-time-failure app is included), then persists the // new boot-id. // // Apps on a drive that never goes live within the window are left to the normal gate (stop→return→ // restart). Single-flight (runs once, before the periodic gate, in driveGateLoop). Best-effort. func (s *Server) processGuestBootChange() { if s.settings == nil || s.stackMgr == nil { return } agent, err := s.agentClient() if err != nil { return } ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) resp, derr := agent.Disks(ctx) cancel() if derr != nil { return } if resp.GuestBootID == "" || resp.GuestBootID == s.settings.GetLastGuestBootID() { return // no boot-id, or unchanged (controller-only restart) → no recreate } // Distinct stable drive paths that deployed drive-backed apps depend on (HDD_PATH is the drive root). needed := map[string]bool{} for _, st := range s.stackMgr.GetStacks() { cfg := s.stackMgr.LoadAppConfigByName(st.Name) if cfg == nil { continue } hdd := cfg.Env["HDD_PATH"] if cfg.Deployed && hdd != "" && strings.HasPrefix(hdd, StableParentDir+"/") { needed[hdd] = true } } if len(needed) == 0 { if serr := s.settings.SetLastGuestBootID(resp.GuestBootID); serr != nil { s.logger.Printf("[WARN] [gate] persist boot-id: %v", serr) } return } paths := make([]string, 0, len(needed)) for p := range needed { paths = append(paths, p) } // READINESS GATE (the fix): wait for the REAL live in-guest bind, not the once-sampled agent view. s.logger.Printf("[INFO] [gate] boot %s: waiting (≤%s) for live drive bind(s) %v before recreating drive-backed apps", resp.GuestBootID, bootBindWait, paths) presentStable := pollLiveBinds(paths, driveBindLive, time.Sleep, time.Now, bootBindWait, bootBindPoll) var bootStacks []bootStack for _, st := range s.stackMgr.GetStacks() { cfg := s.stackMgr.LoadAppConfigByName(st.Name) if cfg == nil { continue } // hasContainers is sampled HERE, before any recreate — recreate's StopStack destroys it (R-55). bootStacks = append(bootStacks, bootStack{ name: st.Name, deployed: cfg.Deployed, hdd: cfg.Env["HDD_PATH"], state: string(st.State), hasContainers: len(st.Containers) > 0, }) } recreate := func(bs bootStack) { s.logger.Printf("[INFO] [gate] boot %s: live bind confirmed — recreating drive-backed app %s (state=%s) onto %s", resp.GuestBootID, bs.name, bs.state, bs.hdd) _ = s.stackMgr.StopStack(bs.name) if serr := s.stackMgr.StartStack(bs.name); serr != nil { s.logger.Printf("[WARN] [gate] boot recreate %s: %v", bs.name, serr) } } syncFB := func() { s.logger.Printf("[INFO] [gate] boot %s: re-syncing FileBrowser mounts against the live binds", resp.GuestBootID) go s.SyncFileBrowserMounts() } _, skipped, leftStopped := recreateDriveBackedApps(bootStacks, presentStable, recreate, syncFB) if skipped > 0 { s.logger.Printf("[WARN] [gate] boot %s: %d drive-backed app(s) had no live bind within %s — leaving to the drive gate", resp.GuestBootID, skipped, bootBindWait) } if leftStopped > 0 { // INFO, not WARN: this is the gate working as intended (R-55). Make the honoured path // observable — a silent correct path is how an inert seam hides. s.logger.Printf("[INFO] [gate] boot %s: %d drive-backed app(s) left stopped — zero containers means the customer stopped them on purpose", resp.GuestBootID, leftStopped) } if serr := s.settings.SetLastGuestBootID(resp.GuestBootID); serr != nil { s.logger.Printf("[WARN] [gate] persist boot-id: %v", serr) } } // bootStack is one deployed stack's boot-recreate inputs (decoupled from stacks.Manager for testing). type bootStack struct { name string deployed bool hdd string state string // hasContainers is len(Stack.Containers) > 0, from `docker ps -a` — so Exited containers COUNT. // R-55's running-at-shutdown signal: a UI Stop is `compose down` and leaves zero. MUST be sampled // before any recreate runs, since recreate's StopStack erases it. hasContainers bool } // recreateDriveBackedApps recreates every deployed drive-backed app whose drive bind is live, then // triggers the FileBrowser sync. FileBrowser binds the drives' userdata but is base-infra (no HDD_PATH), // so it is NOT in the recreate set — it must be converged HERE, AFTER the recreate (which itself only // ran once pollLiveBinds confirmed the live binds), so FileBrowser's mounts reflect the now-live drives // instead of going stale (the gap a host/guest reboot left before this fix). syncFB is always called so // FileBrowser reflects the current bind state even if no app needed recreating. Pure (ops injected). // R-55: `leftStopped` counts drive-backed apps deliberately NOT touched because they have zero // containers (a customer Stop). It is reported separately from `skipped` — conflating the two would // make an honoured Stop look like the "bind never went live" failure and fire a WARN for healthy, // intended behaviour. func recreateDriveBackedApps(stacks []bootStack, presentStable map[string]bool, recreate func(bootStack), syncFB func()) (recreated, skipped, leftStopped int) { for _, bs := range stacks { if !shouldRecreateOnBoot(bs.deployed, bs.hdd, presentStable, bs.hasContainers) { if bs.deployed && strings.HasPrefix(bs.hdd, StableParentDir+"/") { switch { case presentStable[bs.hdd] && !bs.hasContainers: leftStopped++ // drive IS live; the app is at zero containers → stopped on purpose default: skipped++ // a deployed drive-backed app whose bind never went live → gate's job } } continue } recreate(bs) recreated++ } syncFB() return } // ---- H1 endpoints (the UI's settings.js calls these; previously 404/unrouted) ----------------- // handleStorageDisconnect EJECTS a drive without restart: stop its apps (gate-stopped), agent-detach the // felhom-data bind from under the parent (live, fail-closed), and mark it disconnected. The drive is then // safely removable. POST {where} where = the registered (stable) path. func (s *Server) handleStorageDisconnect(w http.ResponseWriter, r *http.Request) { where, ok := s.gateWhere(w, r) if !ok { return } stopped := s.stopAppsOnPath(where) agent, err := s.agentClient() if err == nil { if _, derr := agent.EjectDisk(r.Context(), agentWhere(where)); derr != nil { s.logger.Printf("[WARN] [web] disconnect: agent detach %s failed: %v", where, derr) } } if err := s.settings.SetDisconnected(where, true, stopped); err != nil { writeDiskJSON(w, http.StatusInternalServerError, false, err.Error(), nil) return } go s.SyncFileBrowserMounts() writeDiskJSON(w, http.StatusOK, true, "", map[string]any{"where": where, "stopped": stopped}) } // handleStorageReconnect re-attaches a returned drive without restart: agent-attach the felhom-data bind // under the parent (live), restart the gate-stopped apps, clear the disconnected mark. // handleStorageReconnect re-enrolls a DISCONNECTED **or** DECOMMISSIONED drive in one click (H3): // (for a decommissioned drive) clears the soft marker + restores schedulable, then re-attaches the // felhom-data under the parent (agent, raw path, no reboot) and restarts the gate-stopped apps. The data // is intact — decommission is non-destructive — so the apps resolve once the bind is live. func (s *Server) handleStorageReconnect(w http.ResponseWriter, r *http.Request) { where, ok := s.gateWhere(w, r) if !ok { return } decommissioned := s.settings.IsDecommissioned(where) var stopped []string for _, sp := range s.settings.GetStoragePaths() { if sp.Path == where { stopped = sp.StoppedStacks } } if decommissioned { if _, cerr := s.reEnrollClearMarker(where); cerr != nil { writeDiskJSON(w, http.StatusInternalServerError, false, cerr.Error(), nil) return } // decommission-anyway stopped the apps WITHOUT persisting StoppedStacks — re-discover them. stopped = s.appsOnStoragePath(where) } agent, err := s.agentClient() if err != nil { writeDiskJSON(w, http.StatusServiceUnavailable, false, err.Error(), nil) return } if aerr := agent.GuestAttach(r.Context(), agentWhere(where)); aerr != nil { writeDiskJSON(w, http.StatusBadGateway, false, "újracsatolás sikertelen: "+aerr.Error(), nil) return } s.restartStacks(stopped) if err := s.settings.ClearDisconnected(where); err != nil { writeDiskJSON(w, http.StatusInternalServerError, false, err.Error(), nil) return } go s.SyncFileBrowserMounts() writeDiskJSON(w, http.StatusOK, true, "", map[string]any{"where": where, "restarted": stopped, "reenrolled": decommissioned}) } // handleStorageRestartApps restarts the gate-stopped apps on a path (without changing connection state) — // the manual "restart the apps that were stopped" action. func (s *Server) handleStorageRestartApps(w http.ResponseWriter, r *http.Request) { where, ok := s.gateWhere(w, r) if !ok { return } var stopped []string for _, sp := range s.settings.GetStoragePaths() { if sp.Path == where { stopped = sp.StoppedStacks } } s.restartStacks(stopped) writeDiskJSON(w, http.StatusOK, true, "", map[string]any{"where": where, "restarted": stopped}) } // gateWhere decodes + validates the {where} body shared by the H1 endpoints. func (s *Server) gateWhere(w http.ResponseWriter, r *http.Request) (string, bool) { var req struct { Where string `json:"where"` } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeDiskJSON(w, http.StatusBadRequest, false, "érvénytelen kérés", nil) return "", false } where := path.Clean(strings.TrimSpace(req.Where)) if where == "" || where == "." || !strings.HasPrefix(where, "/mnt/") { writeDiskJSON(w, http.StatusBadRequest, false, "érvénytelen csatlakoztatási pont", nil) return "", false } return where, true }