felhom-agent/internal/localapi/controllersupervisor.go

package localapi

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)

// R-523 — the in-guest controller supervisor.
//
// THE OUTAGE THIS EXISTS TO KILL (BIGNIGHT F9, 2026-09-14). `docker kill felhom-controller` left the
// container `Exited (137)`. Nothing restarted it: Docker never restarts a container whose stop it
// records as deliberate — measured 2026-09-15 on Docker 29.8.0 for BOTH `unless-stopped` and `always`
// (evidence-p1fixes-2026-09-15/A1) — and the golden's `felhom-controller-bootstrap.service` is a
// oneshot (`RemainAfterExit=yes`) that ran once at boot and watches nothing. The household's
// dashboard answered 502 for 33 minutes until the box was power-cycled.
//
// This is doc 03 §4's sentence made real: "Healing a crashed controller is non-destructive by
// construction … redeploy = restart … inside the existing guest — never a guest destroy." The act
// is exactly the swap's own restart (`systemctl restart felhom-controller-bootstrap.service`, which
// does `docker rm -f` + `docker run` from the baked image and the guest's persistent volume), over
// the same GuestExecutor and the same two sudoers grants (`docker inspect -f *`, the unit restart).
// No new privilege.
//
// THE GUARDS, each because doing the act at the wrong moment is worse than not doing it:
//   - not during a swap (the swap stops the controller ON PURPOSE and owns its own rollback);
//   - not when the operator parked it (`<guests>/<vmid>/controller-parked` on the HOST);
//   - not on a guest that is not running, is locked (backup/restore/snapshot/migrate), or has a
//     vzdump in flight — a stopping or restoring guest is someone else's transaction;
//   - not on ONE observation: the container must be seen not-running on two consecutive sweeps, so
//     the bootstrap's own rm-f/run window (boot, path-unit hot-plug) is never raced;
//   - no thrash: 3 restarts inside 15 minutes → stop restarting, raise `controller_crashloop`, try
//     again after 30 minutes.
//
// THE EVENTS. The agent has no event channel of its own; its heartbeat IS the channel (the
// capability/leaf precedent). The per-guest record rides the host report as `controller_supervisor`,
// and the hub's ControllerSupervisorChecker mints `controller_restarted_by_agent` (info) when a
// guest's `last_restart_at` moves and `controller_crashloop` (error, operator-only) when
// `crashloop_since` moves. Timestamps, not counters, so an agent restart (which zeroes the in-memory
// record) can never read as a new restart.

const (
	// controllerSupervisorInterval is the sweep cadence. Two not-running observations are required,
	// so a killed controller is restarted 30–60 s after it died.
	controllerSupervisorInterval = 30 * time.Second
	// controllerSupervisorConfirm is how many consecutive not-running observations license a restart.
	controllerSupervisorConfirm = 2
	// Backoff: controllerCrashloopMax restarts inside controllerCrashloopWindow → give up for
	// controllerCrashloopPause.
	controllerCrashloopMax    = 3
	controllerCrashloopWindow = 15 * time.Minute
	controllerCrashloopPause  = 30 * time.Minute
	// controllerSupervisorHeartbeatEvery: a liveness line every 20 sweeps (10 minutes) — a silent
	// watchdog is indistinguishable from a dead one (standing rule 3).
	controllerSupervisorHeartbeatEvery = 20

	// ControllerParkedMarker is the host-side file that parks a guest's controller. The operator
	// creates it with `touch /var/lib/felhom-agent/guests/<vmid>/controller-parked` and removes it to
	// unpark. Host-side on purpose: it needs no in-guest exec grant, it survives a guest rebuild of
	// the controller container, and a customer inside the guest cannot park the supervisor.
	ControllerParkedMarker = "controller-parked"

	defaultGuestsStateDir = "/var/lib/felhom-agent/guests"

	// R-539 (operator ruling 3 of 2026-09-16) — the SLOW crash loop. The 3-in-15-minutes brake above
	// cannot see a controller that dies every 20 minutes: no two restarts share its window, so it is
	// restarted for ever and the only trace is an info event that mails nobody (measured 2026-09-16,
	// R-531). A second counter over 24 hours raises a WARNING at the fifth restart. It does NOT stop
	// restarting — the fast brake stays the only brake, unchanged. Every restart the supervisor
	// performs counts, including one that follows a deliberate operator `docker kill` (measured
	// 2026-09-15: the supervisor cannot tell a kill from a crash, and a controller that is killed five
	// times a day is worth a line to the operator either way).
	controllerSlowCrashloopWindow = 24 * time.Hour
	controllerSlowCrashloopMax    = 5
	// controllerSlowCounterFile holds the 24-hour restart times and the last raise, per guest, beside
	// the parked marker.
	controllerSlowCounterFile = "controller-restarts-24h.json"

	// R-1344 (operator brief 2026-10-29, Part 2) — a controller that RUNS but does not answer. Docker's own HEALTHCHECK
	// (`curl -f localhost:8080/api/health`, every 30 s, timeout 5 s) reads „unhealthy" only after 3 failed probes (~90 s);
	// controllerUnresponsiveConfirm more sweeps (3 minutes) make it an episode: logged, carried by the stanza, mailed by
	// the hub (controller_unresponsive). ALARM ONLY — it is NOT restarted for this until the measurement (R-1344) shows a
	// busy night (backups, updates) never reads unhealthy that long. Not counted during a swap, a parked guest, a locked
	// guest or a vzdump.
	controllerUnresponsiveConfirm = 6
)

// controllerSupState is one guest's supervisor record. In-memory on purpose (the guest-power
// precedent): an agent restart forgets a crash-loop pause, which costs at most one more restart
// attempt, whereas persisting it could carry a stale "give up" across the restart that fixed it.
type controllerSupState struct {
	notRunningSeen int
	restarts       []time.Time // restart times inside the crash-loop window (pruned)
	restartsTotal  int
	lastRestartAt  time.Time
	lastReason     string
	crashloopSince time.Time // zero = not in a crash-loop pause
	parked         bool

	// R-539 — the slow counter. PERSISTED, unlike everything above, and the precedent's reason does not
	// apply to it: persisting the fast record could carry a stale "give up" across the restart that
	// fixed it, but this record never gives anything up — it only warns. Losing it on an agent restart,
	// on the other hand, would hide exactly the box it exists for (one whose agent restarts too).
	restarts24h        []time.Time
	slowCrashloopSince time.Time // the last raise; kept after it ages out, the hub keys on it MOVING

	// opBrakeUntil (2026-10-29): an operator restart refused by the brake refuses every press until then (30 min).
	opBrakeUntil time.Time

	// R-1344: consecutive running-but-unhealthy sweeps, and the current episode (since kept after it ends).
	unhealthySeen     int
	unresponsive      bool
	unresponsiveSince time.Time
}

type controllerSupervisor struct {
	mu     sync.Mutex
	guests map[int]*controllerSupState
	sweeps int
}

// WatchControllers runs the controller supervisor sweep until ctx is done. No-op when the guest list
// (staleLock) or the guest executor is not wired.
func (s *Server) WatchControllers(ctx context.Context) {
	if s.staleLock == nil || s.guestExec == nil {
		s.logger.Info("controller-supervisor: not wired (no guest list or no guest executor) — disabled")
		return
	}
	s.logger.Info("controller-supervisor: started", "interval", controllerSupervisorInterval.String(),
		"confirm_sweeps", controllerSupervisorConfirm, "crashloop_max", controllerCrashloopMax,
		"crashloop_window", controllerCrashloopWindow.String(),
		"slow_crashloop_max", controllerSlowCrashloopMax, "slow_crashloop_window", controllerSlowCrashloopWindow.String(),
		"guests_dir", s.guestsStateDir())
	t := time.NewTicker(controllerSupervisorInterval)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
			s.ControllerSupervisorTick(ctx)
		}
	}
}

func (s *Server) guestsStateDir() string {
	if s.guestsDir != "" {
		return s.guestsDir
	}
	return defaultGuestsStateDir
}

// provisionedGuest reports whether the agent provisioned a controller into this guest: the
// `<guests>/<vmid>/bootstrap` directory exists. The directory itself, not bootstrap.json inside it —
// the directory is owned by the mapped guest root (0700), so the non-root agent can see the entry but
// not stat the file within.
func (s *Server) provisionedGuest(vmid int) bool {
	fi, err := os.Stat(filepath.Join(s.guestsStateDir(), strconv.Itoa(vmid), "bootstrap"))
	return err == nil && fi.IsDir()
}

func (s *Server) controllerParked(vmid int) bool {
	_, err := os.Stat(filepath.Join(s.guestsStateDir(), strconv.Itoa(vmid), ControllerParkedMarker))
	return err == nil
}

func (s *Server) supState(vmid int) *controllerSupState {
	if s.ctrlSup.guests == nil {
		s.ctrlSup.guests = map[int]*controllerSupState{}
	}
	st := s.ctrlSup.guests[vmid]
	if st == nil {
		st = &controllerSupState{}
		s.loadSlowCounter(vmid, st)
		s.ctrlSup.guests[vmid] = st
	}
	return st
}

// slowCounterRecord is the on-disk shape of the R-539 counter.
type slowCounterRecord struct {
	Restarts           []time.Time `json:"restarts"`
	SlowCrashloopSince time.Time   `json:"slow_crashloop_since,omitempty"`
}

func (s *Server) slowCounterPath(vmid int) string {
	return filepath.Join(s.guestsStateDir(), strconv.Itoa(vmid), controllerSlowCounterFile)
}

// loadSlowCounter restores the persisted counter into a fresh state. Absent = a clean start; unreadable
// or corrupt = a clean start with a WARN (a warning counter must never block supervision).
func (s *Server) loadSlowCounter(vmid int, st *controllerSupState) {
	b, err := os.ReadFile(s.slowCounterPath(vmid))
	if err != nil {
		if !os.IsNotExist(err) {
			s.logger.Warn("controller-supervisor: slow counter unreadable — starting it from zero", "vmid", vmid, "err", err)
		}
		return
	}
	var rec slowCounterRecord
	if err := json.Unmarshal(b, &rec); err != nil {
		s.logger.Warn("controller-supervisor: slow counter corrupt — starting it from zero", "vmid", vmid, "err", err)
		return
	}
	st.restarts24h = pruneBefore(rec.Restarts, s.clock().Add(-controllerSlowCrashloopWindow))
	st.slowCrashloopSince = rec.SlowCrashloopSince
	if len(st.restarts24h) > 0 || !st.slowCrashloopSince.IsZero() {
		s.logger.Info("controller-supervisor: slow counter restored from disk", "vmid", vmid,
			"restarts_24h", len(st.restarts24h), "slow_crashloop_since", st.slowCrashloopSince.Format(time.RFC3339))
	}
}

// saveSlowCounter writes the counter atomically (tmp + rename, 0600). A failure is logged and the
// in-memory counter carries on — the next restart retries the write.
func (s *Server) saveSlowCounter(vmid int, rec slowCounterRecord) {
	path := s.slowCounterPath(vmid)
	b, err := json.Marshal(rec)
	if err == nil {
		tmp := path + ".tmp"
		if err = os.WriteFile(tmp, b, 0o600); err == nil {
			err = os.Rename(tmp, path)
		}
	}
	if err != nil {
		s.logger.Warn("controller-supervisor: could not persist the slow counter (kept in memory)", "vmid", vmid, "path", path, "err", err)
	}
}

// ControllerSupervisorTick performs one sweep. Exported so a test (and a live check) can drive one
// cycle without waiting on the ticker.
func (s *Server) ControllerSupervisorTick(ctx context.Context) {
	if s.staleLock == nil || s.guestExec == nil {
		return
	}
	guests, err := s.staleLock.Guests(ctx)
	if err != nil {
		// Ownership unproven ⇒ touch nothing (the guest-power rule).
		s.logger.Warn("controller-supervisor: guest list unavailable — skipping sweep (ownership unproven)", "err", err)
		return
	}
	var evaluated, down int
	for _, g := range guests {
		if ctx.Err() != nil {
			return
		}
		if !s.provisionedGuest(g.VMID) {
			continue
		}
		evaluated++
		if !s.superviseOneController(ctx, g.VMID, g.Status) {
			down++
		}
	}
	s.ctrlSup.mu.Lock()
	s.ctrlSup.sweeps++
	sweeps := s.ctrlSup.sweeps
	s.ctrlSup.mu.Unlock()
	if sweeps%controllerSupervisorHeartbeatEvery == 0 {
		s.logger.Info("controller-supervisor: alive", "sweeps_since_boot", sweeps,
			"guests_evaluated", evaluated, "controllers_not_running", down)
	}
}

// controllerRunning asks the guest's Docker for the controller's state. Returns (running, known).
// known=false means the question could not be answered (pct exec failed for a reason other than a
// missing container) — the caller does nothing on unknown. An ABSENT container is a known "not
// running": `docker rm` of the controller is the same outage as a kill.
func (s *Server) controllerRunning(ctx context.Context, vmid int) (running, known bool, status string) {
	out, err := s.guestExec.GuestExec(ctx, vmid, "docker", "inspect", "-f", "{{.State.Status}}", controllerContainer)
	if err != nil {
		msg := strings.ToLower(err.Error() + " " + out)
		if strings.Contains(msg, "no such object") || strings.Contains(msg, "no such container") {
			return false, true, "absent"
		}
		return false, false, ""
	}
	status = strings.TrimSpace(out)
	// "restarting" is Docker's own restart loop at work — not ours to fight on this sweep.
	return status == "running" || status == "restarting", true, status
}

// superviseOneController evaluates one provisioned guest and restarts its controller when every guard
// allows. Returns false when the controller was observed not running.
func (s *Server) superviseOneController(ctx context.Context, vmid int, guestStatus string) bool {
	now := s.clock()
	if guestStatus != "running" {
		s.resetNotRunning(vmid)
		return true // the guest-power watchdog owns a stopped guest; its controller is not "down"
	}

	running, known, status := s.controllerRunning(ctx, vmid)
	if !known {
		s.logger.Debug("controller-supervisor: controller state unknown (guest exec failed) — no action", "vmid", vmid)
		s.resetNotRunning(vmid)
		return true
	}
	parked := s.controllerParked(vmid)
	s.ctrlSup.mu.Lock()
	st := s.supState(vmid)
	st.parked = parked
	if running {
		st.notRunningSeen = 0
		s.ctrlSup.mu.Unlock()
		s.observeControllerHealth(ctx, vmid, parked)
		return true
	}
	st.notRunningSeen++
	seen := st.notRunningSeen
	s.ctrlSup.mu.Unlock()

	if parked {
		s.logger.Info("controller-supervisor: controller is not running and the guest is PARKED — leaving it",
			"vmid", vmid, "status", status, "marker", filepath.Join(s.guestsStateDir(), strconv.Itoa(vmid), ControllerParkedMarker))
		return false
	}
	s.swapMu.Lock()
	swapping := s.swapInFlight[vmid]
	s.swapMu.Unlock()
	if swapping {
		s.logger.Info("controller-supervisor: controller is not running during a controller SWAP — the swap owns it",
			"vmid", vmid, "status", status)
		s.resetNotRunning(vmid)
		return false
	}
	if seen < controllerSupervisorConfirm {
		s.logger.Info("controller-supervisor: controller observed not running — confirming on the next sweep",
			"vmid", vmid, "status", status, "seen", seen, "of", controllerSupervisorConfirm)
		return false
	}
	lock, _, err := s.staleLock.Lock(ctx, vmid)
	if err != nil {
		s.logger.Warn("controller-supervisor: could not read the guest lock — no action (fail-safe)", "vmid", vmid, "err", err)
		return false
	}
	if lock != "" {
		s.logger.Info("controller-supervisor: guest is LOCKED — another operation owns it, no action", "vmid", vmid, "lock", lock)
		return false
	}
	if busy, berr := s.staleLock.BackupRunning(ctx, vmid); berr != nil || busy {
		s.logger.Info("controller-supervisor: a vzdump may be in flight for the guest — no action",
			"vmid", vmid, "backup_running", busy, "err", berr)
		return false
	}

	// Backoff.
	s.ctrlSup.mu.Lock()
	st = s.supState(vmid)
	if !st.crashloopSince.IsZero() {
		if now.Sub(st.crashloopSince) < controllerCrashloopPause {
			s.ctrlSup.mu.Unlock()
			s.logger.Warn("controller-supervisor: crash-loop pause in force — not restarting",
				"vmid", vmid, "since", st.crashloopSince.Format(time.RFC3339), "resume_after", controllerCrashloopPause.String())
			return false
		}
		// Pause over: resume with a clean window. crashloopSince stays as the record of the last
		// crash-loop (the hub keys on it moving, not on it clearing).
		st.restarts = nil
		st.crashloopSince = time.Time{}
	}
	st.restarts = pruneBefore(st.restarts, now.Add(-controllerCrashloopWindow))
	if len(st.restarts) >= controllerCrashloopMax {
		st.crashloopSince = now
		n := len(st.restarts)
		s.ctrlSup.mu.Unlock()
		s.logger.Error("controller-supervisor: CRASH-LOOP — the controller would not stay up; stopping restarts and raising controller_crashloop",
			"vmid", vmid, "restarts_in_window", n, "window", controllerCrashloopWindow.String(), "pause", controllerCrashloopPause.String())
		return false
	}
	s.ctrlSup.mu.Unlock()

	reason := "controller container " + status + " on " + strconv.Itoa(controllerSupervisorConfirm) + " consecutive sweeps"
	s.logger.Warn("controller-supervisor: controller is NOT running — restarting the bootstrap unit",
		"vmid", vmid, "status", status, "unit", bootstrapUnit)
	if _, err := s.guestExec.GuestExec(ctx, vmid, "systemctl", "restart", bootstrapUnit); err != nil {
		s.logger.Error("controller-supervisor: bootstrap restart failed", "vmid", vmid, "err", err)
		reason += "; restart FAILED: " + err.Error()
	}
	s.ctrlSup.mu.Lock()
	st = s.supState(vmid)
	st.restarts = append(st.restarts, now)
	st.restartsTotal++
	st.lastRestartAt = now
	st.lastReason = reason
	st.notRunningSeen = 0
	// R-539: the slow counter. Raise at most once per 24 hours — the hub mails on the raise MOVING.
	st.restarts24h = append(pruneBefore(st.restarts24h, now.Add(-controllerSlowCrashloopWindow)), now)
	n24 := len(st.restarts24h)
	raised := false
	if n24 >= controllerSlowCrashloopMax && (st.slowCrashloopSince.IsZero() || now.Sub(st.slowCrashloopSince) >= controllerSlowCrashloopWindow) {
		st.slowCrashloopSince = now
		raised = true
	}
	rec := slowCounterRecord{Restarts: append([]time.Time(nil), st.restarts24h...), SlowCrashloopSince: st.slowCrashloopSince}
	s.ctrlSup.mu.Unlock()
	s.saveSlowCounter(vmid, rec)
	s.logger.Warn("controller-supervisor: RESTARTED the controller", "vmid", vmid, "reason", reason, "restarts_24h", n24)
	if raised {
		s.logger.Warn("controller-supervisor: SLOW CRASH-LOOP — the controller keeps dying; still restarting it, raising controller_slow_crashloop",
			"vmid", vmid, "restarts_24h", n24, "window", controllerSlowCrashloopWindow.String(), "threshold", controllerSlowCrashloopMax)
	}
	return false
}

// observeControllerHealth is R-1344's count for a RUNNING controller: Docker's health status, one read per sweep over
// the swap's `docker inspect -f` grant. Anything but „unhealthy" — healthy, starting, no health check, a read that
// fails — resets the count; so do a swap, a parked guest, a locked guest and a vzdump (each is someone else's moment).
func (s *Server) observeControllerHealth(ctx context.Context, vmid int, parked bool) {
	s.swapMu.Lock()
	swapping := s.swapInFlight[vmid]
	s.swapMu.Unlock()
	unhealthy := false
	if !parked && !swapping {
		if out, err := s.guestExec.GuestExec(ctx, vmid, "docker", "inspect", "-f", opHealthTemplate, controllerContainer); err == nil {
			f := strings.SplitN(strings.TrimSpace(out), "|", 2)
			unhealthy = len(f) == 2 && f[0] == "true" && f[1] == "unhealthy"
		}
	}
	if unhealthy {
		if lock, _, err := s.staleLock.Lock(ctx, vmid); err != nil || lock != "" {
			unhealthy = false
		} else if busy, berr := s.staleLock.BackupRunning(ctx, vmid); berr != nil || busy {
			unhealthy = false
		}
	}
	now := s.clock()
	s.ctrlSup.mu.Lock()
	st := s.supState(vmid)
	if !unhealthy {
		was := st.unresponsive
		st.unhealthySeen, st.unresponsive = 0, false
		s.ctrlSup.mu.Unlock()
		if was {
			s.logger.Info("controller-supervisor: the controller answers its health check again", "vmid", vmid)
		}
		return
	}
	st.unhealthySeen++
	seen := st.unhealthySeen
	start := seen >= controllerUnresponsiveConfirm && !st.unresponsive
	if start {
		st.unresponsive, st.unresponsiveSince = true, now
	}
	s.ctrlSup.mu.Unlock()
	if start {
		s.logger.Error("controller-supervisor: the controller RUNS but does not answer its health check — raising controller_unresponsive; NOT restarting it (R-1344: alarm only until measured)",
			"vmid", vmid, "unhealthy_sweeps", seen, "interval", controllerSupervisorInterval.String())
	}
}

func (s *Server) resetNotRunning(vmid int) {
	s.ctrlSup.mu.Lock()
	defer s.ctrlSup.mu.Unlock()
	if st := s.ctrlSup.guests[vmid]; st != nil {
		st.notRunningSeen = 0
	}
}

func (s *Server) clock() time.Time {
	if s.now != nil {
		return s.now()
	}
	return time.Now().UTC()
}

func pruneBefore(ts []time.Time, cutoff time.Time) []time.Time {
	out := ts[:0]
	for _, t := range ts {
		if !t.Before(cutoff) {
			out = append(out, t)
		}
	}
	return out
}

// ControllerSupervisorStatus is the host-report stanza source (hub.ControllerSupervisorReporter).
// Nil when the supervisor is not wired, so the stanza is omitted.
func (s *Server) ControllerSupervisorStatus(_ context.Context) *hub.ControllerSupervisorStatus {
	if s.staleLock == nil || s.guestExec == nil {
		return nil
	}
	s.ctrlSup.mu.Lock()
	defer s.ctrlSup.mu.Unlock()
	out := &hub.ControllerSupervisorStatus{Guests: []hub.ControllerSupervisorGuest{}}
	for vmid, st := range s.ctrlSup.guests {
		g := hub.ControllerSupervisorGuest{
			VMID:          vmid,
			RestartsTotal: st.restartsTotal,
			LastReason:    st.lastReason,
			Parked:        st.parked,
			Crashloop:     !st.crashloopSince.IsZero(),
		}
		if !st.lastRestartAt.IsZero() {
			g.LastRestartAt = st.lastRestartAt.UTC().Format(time.RFC3339)
		}
		if !st.crashloopSince.IsZero() {
			g.CrashloopSince = st.crashloopSince.UTC().Format(time.RFC3339)
		}
		now := s.clock()
		g.Restarts24h = len(pruneBefore(append([]time.Time(nil), st.restarts24h...), now.Add(-controllerSlowCrashloopWindow)))
		if !st.slowCrashloopSince.IsZero() {
			g.SlowCrashloopSince = st.slowCrashloopSince.UTC().Format(time.RFC3339)
			g.SlowCrashloop = now.Sub(st.slowCrashloopSince) < controllerSlowCrashloopWindow
		}
		g.Unresponsive = st.unresponsive
		if !st.unresponsiveSince.IsZero() {
			g.UnresponsiveSince = st.unresponsiveSince.UTC().Format(time.RFC3339)
		}
		out.Guests = append(out.Guests, g)
	}
	sort.Slice(out.Guests, func(i, j int) bool { return out.Guests[i].VMID < out.Guests[j].VMID })
	return out
}