felhom-controller/controller/internal/web/intermediary.go

package web

import (
	"context"
	"encoding/json"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
	"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)

// bootBindWait / bootBindPoll bound the readiness gate in processGuestBootChange. On a guest reboot
// the agent re-binds each drive under the stable parent ~18s after boot (well after docker has tried
// and permanently failed to auto-start the drive-backed apps); the wait must comfortably cover that
// with margin, the poll be cheap.
const (
	bootBindWait = 120 * time.Second
	bootBindPoll = 2 * time.Second
)

// driveBindLive reports whether `root` (a /mnt/felhom-drives/<name> stable path) is a REAL live
// mountpoint in this process's own mount namespace — i.e. the agent's per-drive bind has actually
// propagated into the controller's `/mnt` (rslave) view, which is exactly the moment docker can
// (re)create an app whose volume binds under it. It reads /proc/self/mountinfo (pure Go, no syscall;
// the mount point is always field index 4). Before the agent re-binds the drive post-boot, the stable
// path is the empty fail-closed placeholder (a plain subdir of the parent, NOT its own mount) → false.
// This is the load-bearing signal the old code lacked: it sampled the agent's host-side
// BoundUnderParent ONCE during fast controller startup, racing the ~18s rebind.
func driveBindLive(root string) bool {
	data, err := os.ReadFile("/proc/self/mountinfo")
	if err != nil {
		return false
	}
	for _, line := range strings.Split(string(data), "\n") {
		f := strings.Fields(line)
		if len(f) >= 5 && f[4] == root {
			return true
		}
	}
	return false
}

// pollLiveBinds waits (bounded by `wait`, polling every `poll`) for each path's drive bind to go LIVE,
// returning the final liveness map. It breaks early once every path is live. bindLive/sleep/now are
// injected so the readiness gate is deterministically unit-testable. A path that never goes live
// within the window stays false (its apps are left to the normal drive gate).
func pollLiveBinds(paths []string, bindLive func(string) bool, sleep func(time.Duration), now func() time.Time, wait, poll time.Duration) map[string]bool {
	live := make(map[string]bool, len(paths))
	deadline := now().Add(wait)
	for {
		all := true
		for _, p := range paths {
			if live[p] {
				continue
			}
			if bindLive(p) {
				live[p] = true
			} else {
				all = false
			}
		}
		if all || !now().Before(deadline) {
			return live
		}
		sleep(poll)
	}
}

// Intermediary-mount model (controller side). Post-migration a drive is visible in the guest ONLY at its
// STABLE path /mnt/felhom-drives/<name> (the host swaps the backing drive underneath it; see the agent's
// internal/localapi/intermediary.go + SPIKE-intermediary-mount). So:
//   - the REGISTERED storage path + every app's HDD_PATH + FileBrowser source = the STABLE path;
//   - the AGENT still operates on the RAW /mnt/<name> host PVE mount (assign/attach/eject/decommission),
//     so controller→agent `where` is mapped back to raw via agentWhere().
// The drive-absent GATE stops + blocks apps when their drive vanishes and auto-restarts them when it
// returns (host-side, no guest reboot).

// StableParentDir is the permanent in-guest parent the agent binds drives under (mirrors
// localapi.StableParentDir).
const StableParentDir = "/mnt/felhom-drives"

// stablePathForName maps a drive name to its registered stable in-guest path.
func stablePathForName(name string) string { return StableParentDir + "/" + name }

// agentWhere maps a registered storage path — stable /mnt/felhom-drives/<name> OR a legacy raw /mnt/<name>
// — to the RAW /mnt/<name> host mount the agent operates on. Idempotent for an already-raw path
// (path.Base drops the directory either way).
func agentWhere(registeredPath string) string {
	name := path.Base(strings.TrimRight(registeredPath, "/"))
	if name == "" || name == "." || name == "/" {
		return registeredPath
	}
	return "/mnt/" + name
}

// shouldRecreateOnBoot is the PURE decision for the boot-id recreate: on a fresh guest boot, recreate
// a deployed drive-backed app whose drive is present (BoundUnderParent) onto its (re-propagated) drive.
//
// It is DETERMINISTIC and it still does NOT filter on the app's current container STATE. That remains
// load-bearing: a momentarily-stopped app on a fresh reboot (docker hasn't auto-restarted it yet, or
// its create-time bind failed with RestartCount=0) must be recreated, and both the old
// container-uptime sample and a `State != stopped` filter MISS exactly that case. Do not reintroduce
// a state filter here.
//
// ── R-170: intent replaced the container count (v0.190.0) ────────────────────────────────────────
//
// The question this gate has to answer is "did the customer want this app running?", and until
// v0.190.0 it answered by counting containers, exactly as bootrecon.isBootOrphan did:
//
//	ZERO containers → a UI Stop is `compose down`, which REMOVES them → the customer stopped it.
//
// R-55 added that term for a real defect and it was the right fix FOR THE SIGNAL AVAILABLE THEN:
// before it, the gate silently undid a customer's Stop on every guest reboot — including apps
// stopped deliberately to free resources for others, which is precisely when resurrecting them is
// most harmful. `deployed` could not answer it: it is a deploy-lifecycle flag and stays true across
// a Stop.
//
// But the count has at least three causes and cannot separate them — a deliberate Stop, a power cut
// mid-compose, an interrupted backup — so R-166 replaced it in `bootrecon` with the customer's
// RECORDED intent (`desired_state` in app.yaml). This gate was left on the old signal for one
// release, which left the two boot gates disagreeing about the same question. They now agree:
//
//	stopped  → NEVER recreate. The customer said so; no observation overrides it.
//	running  → recreate, whatever the container count. This is the case the count could not see.
//	absent   → fall back to `hasContainers`, i.e. EXACTLY the pre-v0.190.0 behaviour.
//
// The absent branch is not a leftover. Every app.yaml written before v0.189.0 lacks the field, so
// absent is what an upgraded box reads for every app nobody has pressed a button on since; treating
// it as `running` would recreate — and therefore start — apps their owners had deliberately stopped,
// fleet-wide, on the first reboot after the upgrade.
//
// The evidence is read from the snapshot taken BEFORE any recreate runs, because `recreate` itself
// calls StopStack (`compose down`) and so destroys it (R-55).
//
// NOTE on the drive-absent gate: `presentStable[hdd]` is still required and is still load-bearing.
// An app whose drive is absent is never recreated here no matter what its intent says — it is
// recorded in StoragePath.StoppedStacks and restarted by ReconcileDriveGates' `Return` branch, which
// runs on the same loop tick. That term is what keeps this gate safe; it is the term the BOOT SWEEP
// was missing until R-171 (audits/DIAG-bootrecon-drive-absent-2026-08-02.md), and it must not be
// dropped in sympathy with the count.
func shouldRecreateOnBoot(deployed bool, hdd string, presentStable map[string]bool, hasContainers bool, desired string) bool {
	if !deployed || hdd == "" || !strings.HasPrefix(hdd, StableParentDir+"/") || !presentStable[hdd] {
		return false
	}
	switch desired {
	case stacks.DesiredStateStopped:
		return false
	case stacks.DesiredStateRunning:
		return true
	default:
		return hasContainers // absent/legacy — byte-identical to the pre-v0.190.0 rule
	}
}

// defaultPromotionTarget decides M1 (never leave zero default). If the path being decommissioned is NOT
// the current default → ("", false): nothing to do. If it IS the default → pick a promotion target from
// the OTHER schedulable, non-decommissioned paths (preferring `migratedTo` — the migrate-then-decommission
// target — when valid); ("", true) = MUST BLOCK (no other usable drive exists). Pure → unit-testable.
func defaultPromotionTarget(paths []settings.StoragePath, decommissioning, migratedTo string) (target string, mustBlock bool) {
	isDefault := false
	for _, sp := range paths {
		if sp.Path == decommissioning && sp.IsDefault {
			isDefault = true
		}
	}
	if !isDefault {
		return "", false
	}
	usable := func(p string) bool {
		for _, sp := range paths {
			if sp.Path == p && sp.Path != decommissioning && sp.Schedulable && !sp.Decommissioned {
				return true
			}
		}
		return false
	}
	if migratedTo != "" && usable(migratedTo) {
		return migratedTo, false
	}
	for _, sp := range paths {
		if sp.Path != decommissioning && sp.Schedulable && !sp.Decommissioned {
			return sp.Path, false
		}
	}
	return "", true // no other usable drive → block the decommission
}

// appsOnStoragePath returns the deployed stack names whose HDD_PATH equals the given (stable) storage
// path — the apps that depend on that drive.
func (s *Server) appsOnStoragePath(storagePath string) []string {
	var names []string
	for _, st := range s.stackMgr.GetStacks() {
		if cfg := s.stackMgr.LoadAppConfigByName(st.Name); cfg != nil && cfg.Env["HDD_PATH"] == storagePath {
			names = append(names, st.Name)
		}
	}
	return names
}

// stopAppsOnPath stops every deployed app on the given storage path and returns their names (the
// gate-stopped set — distinct from a user stop, which never enters this set). Best-effort per app.
func (s *Server) stopAppsOnPath(storagePath string) []string {
	var stopped []string
	for _, name := range s.appsOnStoragePath(storagePath) {
		if err := s.stackMgr.StopStack(name); err != nil {
			s.logger.Printf("[WARN] [gate] stop %s on absent %s: %v", name, storagePath, err)
			continue
		}
		stopped = append(stopped, name)
	}
	return stopped
}

// restartStacks starts each named stack (the gate-stopped set on drive return). Best-effort per app.
func (s *Server) restartStacks(names []string) {
	for _, name := range names {
		if err := s.stackMgr.StartStack(name); err != nil {
			s.logger.Printf("[WARN] [gate] restart %s: %v", name, err)
		}
	}
}

// gateAction is the reconcile's decision for one registered storage path.
type gateAction struct {
	Path   string
	Stop   bool   // drive ABSENT + not yet marked → stop apps, mark disconnected
	Return bool   // drive RETURNED (present + currently disconnected) → re-attach, restart, clear
	Raw    string // raw /mnt/<name> for the re-attach
}

// planDriveGates is the PURE decision core: given the registry + the agent's disk list, decide per path
// whether to gate (stop) or un-gate (return). A drive is "present" iff a disk with a matching GuestPath
// (stable) or MountPath (legacy raw) is State=="attached". Decommissioned paths are skipped (handled by
// the decommission flow, not the transient gate). No side effects → unit-testable.
func planDriveGates(paths []settings.StoragePath, disks []agentapi.DiskInfo) []gateAction {
	present := map[string]bool{}
	rawByPath := map[string]string{}
	for _, d := range disks {
		// A STABLE path is usable only when the drive's felhom-data is actually BOUND UNDER THE PARENT
		// (BoundUnderParent) — NOT merely when the raw drive is host-mounted (State==attached). This is
		// what makes the gate converge a reboot correctly: at boot the raw drive mounts early but the
		// agent binds it under the parent slightly later; until then the apps' stable-path binds are empty,
		// so the gate keeps them stopped and restarts (recreates) them once the bind is live.
		if d.GuestPath != "" {
			present[d.GuestPath] = present[d.GuestPath] || d.BoundUnderParent
			rawByPath[d.GuestPath] = d.MountPath
		}
		if d.MountPath != "" { // legacy raw path registered directly — host mount is the usable signal
			present[d.MountPath] = present[d.MountPath] || d.State == "attached"
			rawByPath[d.MountPath] = d.MountPath
		}
	}
	var actions []gateAction
	for _, sp := range paths {
		if sp.Decommissioned {
			continue
		}
		// NAS network storage is NOT a drive — the agent never lists it in /disks, so the drive-absent gate
		// would falsely see it "absent" (present[sp.Path]==false) and STOP its apps. A NAS blip is
		// recoverable (the agent's per-share liveness → a warning badge), never the drive stop-cascade.
		// Skip network paths here entirely (Scenario C: unreachable ≠ missing).
		if sp.IsNetwork() {
			continue
		}
		// ONLY gate EXTERNAL drives — those registered under the stable parent /mnt/felhom-drives/<name>.
		// Internal SSD / system paths (e.g. /mnt/sys_drive/felhom-data) are always-present locals the agent
		// never reports as drives; gating them on "absence" would falsely stop/block their apps. (Legacy
		// raw /mnt/<name> external paths are present via the agent's MountPath during the transition and get
		// repointed under the parent by the migration.)
		if !strings.HasPrefix(sp.Path, StableParentDir+"/") {
			continue
		}
		switch {
		case !present[sp.Path] && !sp.Disconnected:
			actions = append(actions, gateAction{Path: sp.Path, Stop: true})
		case present[sp.Path] && sp.Disconnected:
			actions = append(actions, gateAction{Path: sp.Path, Return: true, Raw: rawByPath[sp.Path]})
		}
	}
	return actions
}

// ReconcileDriveGates enforces the drive-absent gate: an ABSENT registered drive gets its apps STOPPED +
// recorded (disconnected); a RETURNED drive gets re-bound under the parent and its gate-stopped apps
// restarted. Best-effort + idempotent — safe to call on a timer and on demand.
func (s *Server) ReconcileDriveGates() {
	if s.settings == nil || s.stackMgr == nil {
		return
	}
	agent, err := s.agentClient()
	if err != nil {
		return
	}
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
	defer cancel()
	resp, err := agent.Disks(ctx)
	if err != nil {
		return
	}
	for _, a := range planDriveGates(s.settings.GetStoragePaths(), resp.Disks) {
		switch {
		case a.Stop:
			stopped := s.stopAppsOnPath(a.Path)
			if err := s.settings.SetDisconnected(a.Path, true, stopped); err != nil {
				s.logger.Printf("[WARN] [gate] mark disconnected %s: %v", a.Path, err)
			}
			s.logger.Printf("[WARN] [gate] drive ABSENT %s — stopped+blocked %d app(s): %v", a.Path, len(stopped), stopped)
			// E-2b: THE SEAM THAT WAS NEVER WIRED. NotifyStorageDisconnected existed, was registered in
			// allowedEventTypes + DefaultEnabledEvents + the hub's Hungarian customerMessages — and was
			// called from nowhere, so a drive going absent produced a log line and silence on every
			// channel. A drive that is ONLY a backup target has no apps to stop, so it was silent twice
			// over. Fifth instance of this class in the project; found by E-2's Phase 0.
			//
			// E-2 Part 5: when the absent drive is the BACKUP TARGET, that is the more specific and more
			// urgent fact, so it gets its own event rather than being folded into the generic one.
			s.notifyDriveAbsent(a.Path, stopped, driveTargetByPath(resp.Disks))
			go s.SyncFileBrowserMounts()
		case a.Return:
			if a.Raw != "" {
				if err := agent.GuestAttach(ctx, a.Raw); err != nil {
					s.logger.Printf("[WARN] [gate] re-attach %s (raw %s): %v", a.Path, a.Raw, err)
				}
			}
			var stopped []string
			for _, sp := range s.settings.GetStoragePaths() {
				if sp.Path == a.Path {
					stopped = sp.StoppedStacks
				}
			}
			s.restartStacks(stopped)
			if err := s.settings.ClearDisconnected(a.Path); err != nil {
				s.logger.Printf("[WARN] [gate] clear disconnected %s: %v", a.Path, err)
			}
			s.logger.Printf("[INFO] [gate] drive RETURNED %s — re-attached + restarted gate-stopped apps", a.Path)
			// E-2b: the recovery half. An operator told a drive vanished must be told it came back —
			// otherwise the alarm is a dead end and the next one is trusted less.
			s.notifyDriveReturned(a.Path, driveTargetByPath(resp.Disks))
			go s.SyncFileBrowserMounts()
		}
	}
}

// driveGateLoop runs ReconcileDriveGates on a timer (the periodic absent/return detector — the slice-8C
// watchdog was retired). Started as a goroutine at server startup. The FIRST action is a one-time
// startup recreate (see recreateBootStaleApps) to converge a guest reboot deterministically, then the
// periodic gate.
func (s *Server) driveGateLoop() {
	// Wait for the stack scan to complete (GetStacks empty at NewServer time) and the agent to come up,
	// so the boot-id recreate sees the real deployed apps + drive state. Bounded poll.
	for i := 0; i < 30; i++ {
		if s.stackMgr != nil && len(s.stackMgr.GetStacks()) > 0 {
			break
		}
		time.Sleep(time.Second)
	}
	s.processGuestBootChange()
	s.ReconcileDriveGates()
	t := time.NewTicker(30 * time.Second)
	defer t.Stop()
	for range t.C {
		// Re-run the boot-change converger every tick (not just once at startup): right after a GUEST
		// reboot the agent's local API may briefly be unreachable / its per-guest token stale, so the
		// single startup attempt can bail before reading the boot-id. It is idempotent (boot-id gated:
		// a no-op once the current boot has been converged), so retrying until the agent is reachable
		// is safe and is what makes guest-reboot recovery robust.
		s.processGuestBootChange()
		s.ReconcileDriveGates()
	}
}

// processGuestBootChange converges a GUEST REBOOT DETERMINISTICALLY. On a guest reboot docker auto-starts
// the app containers (restart:unless-stopped) BEFORE the agent re-propagates the drive under the parent
// (~18s later) — so the create-time bind of an app's volume fails (`mkdir …/userdata: permission denied`
// on the empty fail-closed placeholder) and, being a create-time failure (RestartCount=0), is NEVER
// retried → the app is stuck Exited forever even after the bind lands. The agent reports a `guest_boot_id`
// that changes on every guest boot but is stable across a controller-only restart; the controller
// persists the last-seen value. When it changes (the controller restarts WITH the guest), this:
//  1. gathers the deployed drive-backed apps' stable drive paths,
//  2. GATES on the REAL live in-guest bind — `pollLiveBinds`/`driveBindLive` wait (bounded) until each
//     drive's stable path is an actual mountpoint in the controller's own /mnt (rslave) view, which is
//     exactly when docker can recreate the app. (The old code sampled the agent's BoundUnderParent
//     ONCE during fast startup, raced the ~18s rebind, recreated nothing, and persisted the boot-id —
//     burning its one-shot. That is the bug this fixes.)
//  3. recreates every deployed drive-backed app whose bind is now live (`shouldRecreateOnBoot` is
//     state-independent, so a stuck-Exited create-time-failure app is included), then persists the
//     new boot-id.
//
// Apps on a drive that never goes live within the window are left to the normal gate (stop→return→
// restart). Single-flight (runs once, before the periodic gate, in driveGateLoop). Best-effort.
func (s *Server) processGuestBootChange() {
	if s.settings == nil || s.stackMgr == nil {
		return
	}
	agent, err := s.agentClient()
	if err != nil {
		return
	}
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
	resp, derr := agent.Disks(ctx)
	cancel()
	if derr != nil {
		return
	}
	if resp.GuestBootID == "" || resp.GuestBootID == s.settings.GetLastGuestBootID() {
		return // no boot-id, or unchanged (controller-only restart) → no recreate
	}

	// Distinct stable drive paths that deployed drive-backed apps depend on (HDD_PATH is the drive root).
	needed := map[string]bool{}
	for _, st := range s.stackMgr.GetStacks() {
		cfg := s.stackMgr.LoadAppConfigByName(st.Name)
		if cfg == nil {
			continue
		}
		hdd := cfg.Env["HDD_PATH"]
		if cfg.Deployed && hdd != "" && strings.HasPrefix(hdd, StableParentDir+"/") {
			needed[hdd] = true
		}
	}
	if len(needed) == 0 {
		if serr := s.settings.SetLastGuestBootID(resp.GuestBootID); serr != nil {
			s.logger.Printf("[WARN] [gate] persist boot-id: %v", serr)
		}
		return
	}
	paths := make([]string, 0, len(needed))
	for p := range needed {
		paths = append(paths, p)
	}

	// READINESS GATE (the fix): wait for the REAL live in-guest bind, not the once-sampled agent view.
	s.logger.Printf("[INFO] [gate] boot %s: waiting (≤%s) for live drive bind(s) %v before recreating drive-backed apps", resp.GuestBootID, bootBindWait, paths)
	presentStable := pollLiveBinds(paths, driveBindLive, time.Sleep, time.Now, bootBindWait, bootBindPoll)

	var bootStacks []bootStack
	for _, st := range s.stackMgr.GetStacks() {
		cfg := s.stackMgr.LoadAppConfigByName(st.Name)
		if cfg == nil {
			continue
		}
		// hasContainers is sampled HERE, before any recreate — recreate's StopStack destroys it (R-55).
		bootStacks = append(bootStacks, bootStack{
			name: st.Name, deployed: cfg.Deployed, hdd: cfg.Env["HDD_PATH"], state: string(st.State),
			hasContainers: len(st.Containers) > 0,
			// R-170: read intent from the app.yaml just loaded, not from st.AppConfig — cfg is the
			// fresh on-disk read this loop already performs, so the two cannot disagree.
			desired: cfg.DesiredState,
		})
	}
	recreate := func(bs bootStack) {
		s.logger.Printf("[INFO] [gate] boot %s: live bind confirmed — recreating drive-backed app %s (state=%s) onto %s", resp.GuestBootID, bs.name, bs.state, bs.hdd)
		_ = s.stackMgr.StopStack(bs.name)
		if serr := s.stackMgr.StartStack(bs.name); serr != nil {
			s.logger.Printf("[WARN] [gate] boot recreate %s: %v", bs.name, serr)
		}
	}
	syncFB := func() {
		s.logger.Printf("[INFO] [gate] boot %s: re-syncing FileBrowser mounts against the live binds", resp.GuestBootID)
		go s.SyncFileBrowserMounts()
	}
	_, skipped, leftStopped := recreateDriveBackedApps(bootStacks, presentStable, recreate, syncFB)
	if skipped > 0 {
		s.logger.Printf("[WARN] [gate] boot %s: %d drive-backed app(s) had no live bind within %s — leaving to the drive gate", resp.GuestBootID, skipped, bootBindWait)
	}
	if leftStopped > 0 {
		// INFO, not WARN: this is the gate working as intended (R-55). Make the honoured path
		// observable — a silent correct path is how an inert seam hides.
		s.logger.Printf("[INFO] [gate] boot %s: %d drive-backed app(s) left stopped on purpose (recorded Stop, or a legacy app.yaml at zero containers)", resp.GuestBootID, leftStopped)
	}
	if serr := s.settings.SetLastGuestBootID(resp.GuestBootID); serr != nil {
		s.logger.Printf("[WARN] [gate] persist boot-id: %v", serr)
	}
}

// bootStack is one deployed stack's boot-recreate inputs (decoupled from stacks.Manager for testing).
type bootStack struct {
	name     string
	deployed bool
	hdd      string
	state    string
	// hasContainers is len(Stack.Containers) > 0, from `docker ps -a` — so Exited containers COUNT.
	// R-55's running-at-shutdown signal: a UI Stop is `compose down` and leaves zero. MUST be sampled
	// before any recreate runs, since recreate's StopStack erases it. Since R-170 it is only consulted
	// for apps with NO recorded intent (legacy app.yaml), but it is still sampled for all of them —
	// the legacy branch needs it and the snapshot has exactly one chance to take it.
	hasContainers bool
	// desired is the customer's recorded intent (stacks.DesiredState*), "" when the app.yaml predates
	// v0.189.0. R-170: this is what the gate decides on now, with hasContainers as the legacy fallback.
	desired string
}

// recreateDriveBackedApps recreates every deployed drive-backed app whose drive bind is live, then
// triggers the FileBrowser sync. FileBrowser binds the drives' userdata but is base-infra (no HDD_PATH),
// so it is NOT in the recreate set — it must be converged HERE, AFTER the recreate (which itself only
// ran once pollLiveBinds confirmed the live binds), so FileBrowser's mounts reflect the now-live drives
// instead of going stale (the gap a host/guest reboot left before this fix). syncFB is always called so
// FileBrowser reflects the current bind state even if no app needed recreating. Pure (ops injected).
// R-55: `leftStopped` counts drive-backed apps deliberately NOT touched because they have zero
// containers (a customer Stop). It is reported separately from `skipped` — conflating the two would
// make an honoured Stop look like the "bind never went live" failure and fire a WARN for healthy,
// intended behaviour.
func recreateDriveBackedApps(stacks []bootStack, presentStable map[string]bool, recreate func(bootStack), syncFB func()) (recreated, skipped, leftStopped int) {
	for _, bs := range stacks {
		if !shouldRecreateOnBoot(bs.deployed, bs.hdd, presentStable, bs.hasContainers, bs.desired) {
			if bs.deployed && strings.HasPrefix(bs.hdd, StableParentDir+"/") {
				switch {
				case presentStable[bs.hdd]:
					// The drive IS live and we still declined: the customer's recorded Stop, or a legacy
					// app.yaml at zero containers. Both are "left stopped on purpose", which is what this
					// counter has always meant — only the signal behind it changed (R-170).
					leftStopped++
				default:
					skipped++ // a deployed drive-backed app whose bind never went live → gate's job
				}
			}
			continue
		}
		recreate(bs)
		recreated++
	}
	syncFB()
	return
}

// ---- H1 endpoints (the UI's settings.js calls these; previously 404/unrouted) -----------------

// handleStorageDisconnect EJECTS a drive without restart: stop its apps (gate-stopped), agent-detach the
// felhom-data bind from under the parent (live, fail-closed), and mark it disconnected. The drive is then
// safely removable. POST {where} where = the registered (stable) path.
func (s *Server) handleStorageDisconnect(w http.ResponseWriter, r *http.Request) {
	where, ok := s.gateWhere(w, r)
	if !ok {
		return
	}
	stopped := s.stopAppsOnPath(where)
	agent, err := s.agentClient()
	if err == nil {
		if _, derr := agent.EjectDisk(r.Context(), agentWhere(where)); derr != nil {
			s.logger.Printf("[WARN] [web] disconnect: agent detach %s failed: %v", where, derr)
		}
	}
	if err := s.settings.SetDisconnected(where, true, stopped); err != nil {
		writeDiskJSON(w, http.StatusInternalServerError, false, err.Error(), nil)
		return
	}
	go s.SyncFileBrowserMounts()
	writeDiskJSON(w, http.StatusOK, true, "", map[string]any{"where": where, "stopped": stopped})
}

// handleStorageReconnect re-attaches a returned drive without restart: agent-attach the felhom-data bind
// under the parent (live), restart the gate-stopped apps, clear the disconnected mark.
// handleStorageReconnect re-enrolls a DISCONNECTED **or** DECOMMISSIONED drive in one click (H3):
// (for a decommissioned drive) clears the soft marker + restores schedulable, then re-attaches the
// felhom-data under the parent (agent, raw path, no reboot) and restarts the gate-stopped apps. The data
// is intact — decommission is non-destructive — so the apps resolve once the bind is live.
func (s *Server) handleStorageReconnect(w http.ResponseWriter, r *http.Request) {
	where, ok := s.gateWhere(w, r)
	if !ok {
		return
	}
	decommissioned := s.settings.IsDecommissioned(where)
	var stopped []string
	for _, sp := range s.settings.GetStoragePaths() {
		if sp.Path == where {
			stopped = sp.StoppedStacks
		}
	}
	if decommissioned {
		if _, cerr := s.reEnrollClearMarker(where); cerr != nil {
			writeDiskJSON(w, http.StatusInternalServerError, false, cerr.Error(), nil)
			return
		}
		// decommission-anyway stopped the apps WITHOUT persisting StoppedStacks — re-discover them.
		stopped = s.appsOnStoragePath(where)
	}
	agent, err := s.agentClient()
	if err != nil {
		writeDiskJSON(w, http.StatusServiceUnavailable, false, err.Error(), nil)
		return
	}
	if aerr := agent.GuestAttach(r.Context(), agentWhere(where)); aerr != nil {
		writeDiskJSON(w, http.StatusBadGateway, false, "újracsatolás sikertelen: "+aerr.Error(), nil)
		return
	}
	s.restartStacks(stopped)
	if err := s.settings.ClearDisconnected(where); err != nil {
		writeDiskJSON(w, http.StatusInternalServerError, false, err.Error(), nil)
		return
	}
	go s.SyncFileBrowserMounts()
	writeDiskJSON(w, http.StatusOK, true, "", map[string]any{"where": where, "restarted": stopped, "reenrolled": decommissioned})
}

// handleStorageRestartApps restarts the gate-stopped apps on a path (without changing connection state) —
// the manual "restart the apps that were stopped" action.
func (s *Server) handleStorageRestartApps(w http.ResponseWriter, r *http.Request) {
	where, ok := s.gateWhere(w, r)
	if !ok {
		return
	}
	var stopped []string
	for _, sp := range s.settings.GetStoragePaths() {
		if sp.Path == where {
			stopped = sp.StoppedStacks
		}
	}
	s.restartStacks(stopped)
	writeDiskJSON(w, http.StatusOK, true, "", map[string]any{"where": where, "restarted": stopped})
}

// gateWhere decodes + validates the {where} body shared by the H1 endpoints.
func (s *Server) gateWhere(w http.ResponseWriter, r *http.Request) (string, bool) {
	var req struct {
		Where string `json:"where"`
	}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		writeDiskJSON(w, http.StatusBadRequest, false, "érvénytelen kérés", nil)
		return "", false
	}
	where := path.Clean(strings.TrimSpace(req.Where))
	if where == "" || where == "." || !strings.HasPrefix(where, "/mnt/") {
		writeDiskJSON(w, http.StatusBadRequest, false, "érvénytelen csatlakoztatási pont", nil)
		return "", false
	}
	return where, true
}

// driveTargetByPath maps host mount path → the agent's backup-target flag. The agent is the authority
// (E-2): our own StoragePath.BackupTarget is customer intent, and on the two hand-migrated boxes that
// intent was never recorded while the drive really is the target. An older agent omits the field, so
// every entry is false and we degrade to the generic disconnect alarm — never a wrong one.
func driveTargetByPath(disks []agentapi.DiskInfo) map[string]bool {
	out := make(map[string]bool, 2*len(disks))
	for _, d := range disks {
		// BOTH keyings, mirroring planDriveGates (which registers present[] under GuestPath AND
		// MountPath). The gate's a.Path is the REGISTERED StoragePath, and for an external drive that
		// is the GUEST path /mnt/felhom-drives/<name> — not the agent's host /mnt/<name>. Keying this
		// map on MountPath alone made the backup-target branch unreachable: every absent drive,
		// including the target, fell through to the generic storage_disconnected. Caught before
		// deploy by tracing a.Path back to its source rather than assuming it matched.
		if d.GuestPath != "" {
			out[d.GuestPath] = d.BackupTarget
		}
		if d.MountPath != "" {
			out[d.MountPath] = d.BackupTarget
		}
	}
	return out
}

// storageLabelFor returns the customer-facing label for a registered path, falling back to the path
// itself. An alarm that names a device node the customer has never seen is not actionable.
func (s *Server) storageLabelFor(path string) string {
	for _, sp := range s.settings.GetStoragePaths() {
		if sp.Path == path && strings.TrimSpace(sp.Label) != "" {
			return sp.Label
		}
	}
	return path
}

// notifyDriveAbsent raises the right alarm for a drive that vanished: the backup-target-specific one
// when it holds the whole-guest backup, the generic one otherwise. Never both — two emails for one
// event trains people to ignore the channel.
func (s *Server) notifyDriveAbsent(path string, stopped []string, isTarget map[string]bool) {
	if s.notifier == nil {
		return
	}
	label := s.storageLabelFor(path)
	if isTarget[path] {
		s.logger.Printf("[ERROR] [gate] the ABSENT drive %s is the WHOLE-GUEST BACKUP TARGET — the system backup cannot run until it returns", path)
		s.notifier.NotifyBackupTargetAbsent(label, path)
		return
	}
	s.notifier.NotifyStorageDisconnected(label, stopped)
}

// notifyDriveReturned is the recovery counterpart, and it must mirror notifyDriveAbsent's choice or
// the pairing breaks: a target that alarmed as backup_target_absent has to recover as
// backup_target_restored, not as a generic reconnect the operator cannot match to the original.
func (s *Server) notifyDriveReturned(path string, isTarget map[string]bool) {
	if s.notifier == nil {
		return
	}
	label := s.storageLabelFor(path)
	if isTarget[path] {
		s.notifier.NotifyBackupTargetRestored(label, path)
		return
	}
	s.notifier.NotifyStorageReconnected(label)
}