R-1344 decided from the night's measurement: a controller that runs but does not answer is restarted at 10 unhealthy sweeps (alarm at 6) — busy file + the brake refuse; stanza carries unresponsive_restart_after_s / unresponsive_restart_at

internal/localapi/controllersupervisor.go

@@ -3,6 +3,7 @@ package localapi
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
 	"sort"
@@ -87,10 +88,16 @@ const (
 	// R-1344 (operator brief 2026-10-29, Part 2) — a controller that RUNS but does not answer. Docker's own HEALTHCHECK
 	// (`curl -f localhost:8080/api/health`, every 30 s, timeout 5 s) reads „unhealthy" only after 3 failed probes (~90 s);
 	// controllerUnresponsiveConfirm more sweeps (3 minutes) make it an episode: logged, carried by the stanza, mailed by
-	// the hub (controller_unresponsive). ALARM ONLY — it is NOT restarted for this until the measurement (R-1344) shows a
-	// busy night (backups, updates) never reads unhealthy that long. Not counted during a swap, a parked guest, a locked
-	// guest or a vzdump.
+	// the hub (controller_unresponsive). Not counted during a swap, a parked guest, a locked guest or a vzdump.
 	controllerUnresponsiveConfirm = 6
+	// controllerUnresponsiveRestart (decided 2026-10-30 from the measurement): at this many consecutive unhealthy sweeps
+	// (5 minutes after Docker's „unhealthy", ~6.5 minutes after the first failed probe; 2 minutes after the alarm) the
+	// agent restarts the controller — the not-running restart's guards and brake, plus the controller's busy file. The
+	// numbers: Tester 1 and demo-hp, 2026-10-29 11:12 → 2026-10-30 07:00 UTC, 4 733 probes through the night chain (a
+	// PBS and a local vzdump, the DB dumps, tier-2, the off-site push, a restore test, two controller updates): no failed
+	// probe, slowest 303 ms against the 5 s timeout. A busy night never came near even ONE failed probe; ten sweeps is
+	// the margin. Pinned by TestSupervisor_UnresponsiveIsAlarmedThenRestarted.
+	controllerUnresponsiveRestart = 10
 )
 
 // controllerSupState is one guest's supervisor record. In-memory on purpose (the guest-power
@@ -119,6 +126,10 @@ type controllerSupState struct {
 	unhealthySeen     int
 	unresponsive      bool
 	unresponsiveSince time.Time
+	// unresponsiveRestartAt: the last restart for not answering (kept; the hub keys on it MOVING). unrespRefusal: the
+	// last reason the restart was refused, so a refusal is logged once per change, not every sweep.
+	unresponsiveRestartAt time.Time
+	unrespRefusal         string
 }
 
 type controllerSupervisor struct {
@@ -354,9 +365,27 @@ func (s *Server) superviseOneController(ctx context.Context, vmid int, guestStat
 		return false
 	}
 
-	// Backoff.
+	if !s.supervisorBrakeAllows(vmid, now) {
+		return false
+	}
+
+	reason := "controller container " + status + " on " + strconv.Itoa(controllerSupervisorConfirm) + " consecutive sweeps"
+	s.logger.Warn("controller-supervisor: controller is NOT running — restarting the bootstrap unit",
+		"vmid", vmid, "status", status, "unit", bootstrapUnit)
+	if _, err := s.guestExec.GuestExec(ctx, vmid, "systemctl", "restart", bootstrapUnit); err != nil {
+		s.logger.Error("controller-supervisor: bootstrap restart failed", "vmid", vmid, "err", err)
+		reason += "; restart FAILED: " + err.Error()
+	}
+	s.recordSupervisorRestart(vmid, now, reason, nil)
+	return false
+}
+
+// supervisorBrakeAllows is the brake every supervisor restart passes (the not-running one and R-1344's): a crash-loop
+// pause in force refuses; controllerCrashloopMax restarts inside controllerCrashloopWindow start one (logged, carried by
+// the stanza, mailed by the hub as controller_crashloop) and refuse.
+func (s *Server) supervisorBrakeAllows(vmid int, now time.Time) bool {
 	s.ctrlSup.mu.Lock()
-	st = s.supState(vmid)
+	st := s.supState(vmid)
 	if !st.crashloopSince.IsZero() {
 		if now.Sub(st.crashloopSince) < controllerCrashloopPause {
 			s.ctrlSup.mu.Unlock()
@@ -379,22 +408,23 @@ func (s *Server) superviseOneController(ctx context.Context, vmid int, guestStat
 		return false
 	}
 	s.ctrlSup.mu.Unlock()
+	return true
+}
 
-	reason := "controller container " + status + " on " + strconv.Itoa(controllerSupervisorConfirm) + " consecutive sweeps"
-	s.logger.Warn("controller-supervisor: controller is NOT running — restarting the bootstrap unit",
-		"vmid", vmid, "status", status, "unit", bootstrapUnit)
-	if _, err := s.guestExec.GuestExec(ctx, vmid, "systemctl", "restart", bootstrapUnit); err != nil {
-		s.logger.Error("controller-supervisor: bootstrap restart failed", "vmid", vmid, "err", err)
-		reason += "; restart FAILED: " + err.Error()
-	}
+// recordSupervisorRestart puts a restart the supervisor performed on the record: the brake's window, the totals, the
+// reason, and R-539's slow counter (persisted; raised at most once per 24 hours — the hub mails on the raise MOVING).
+// mark, when set, runs under the same lock (R-1344 stamps its own record there).
+func (s *Server) recordSupervisorRestart(vmid int, now time.Time, reason string, mark func(st *controllerSupState)) {
 	s.ctrlSup.mu.Lock()
-	st = s.supState(vmid)
+	st := s.supState(vmid)
 	st.restarts = append(st.restarts, now)
 	st.restartsTotal++
 	st.lastRestartAt = now
 	st.lastReason = reason
 	st.notRunningSeen = 0
-	// R-539: the slow counter. Raise at most once per 24 hours — the hub mails on the raise MOVING.
+	if mark != nil {
+		mark(st)
+	}
 	st.restarts24h = append(pruneBefore(st.restarts24h, now.Add(-controllerSlowCrashloopWindow)), now)
 	n24 := len(st.restarts24h)
 	raised := false
@@ -410,7 +440,6 @@ func (s *Server) superviseOneController(ctx context.Context, vmid int, guestStat
 		s.logger.Warn("controller-supervisor: SLOW CRASH-LOOP — the controller keeps dying; still restarting it, raising controller_slow_crashloop",
 			"vmid", vmid, "restarts_24h", n24, "window", controllerSlowCrashloopWindow.String(), "threshold", controllerSlowCrashloopMax)
 	}
-	return false
 }
 
 // observeControllerHealth is R-1344's count for a RUNNING controller: Docker's health status, one read per sweep over
@@ -452,11 +481,60 @@ func (s *Server) observeControllerHealth(ctx context.Context, vmid int, parked b
 	if start {
 		st.unresponsive, st.unresponsiveSince = true, now
 	}
+	since := st.unresponsiveSince
 	s.ctrlSup.mu.Unlock()
 	if start {
-		s.logger.Error("controller-supervisor: the controller RUNS but does not answer its health check — raising controller_unresponsive; NOT restarting it (R-1344: alarm only until measured)",
-			"vmid", vmid, "unhealthy_sweeps", seen, "interval", controllerSupervisorInterval.String())
+		s.logger.Error("controller-supervisor: the controller RUNS but does not answer its health check — raising controller_unresponsive; restarting it if it stays so",
+			"vmid", vmid, "unhealthy_sweeps", seen, "interval", controllerSupervisorInterval.String(),
+			"restart_after_sweeps", controllerUnresponsiveRestart)
 	}
+	if seen >= controllerUnresponsiveRestart {
+		s.restartUnresponsive(ctx, vmid, seen, since, now)
+	}
+}
+
+// restartUnresponsive is R-1344's restart (decided 2026-10-30): the controller's busy file (fresh = a backup, a
+// restore, an update or an export is running; unreadable = cannot be ruled out) refuses, then the brake every
+// supervisor restart passes. The swap, the parked marker, the lock and the vzdump never reach here — they reset the
+// count. A refusal keeps the alarm and is logged once per reason; the next sweep asks again.
+func (s *Server) restartUnresponsive(ctx context.Context, vmid, seen int, since, now time.Time) {
+	refuse := func(why string) {
+		s.ctrlSup.mu.Lock()
+		st := s.supState(vmid)
+		changed := st.unrespRefusal != why
+		st.unrespRefusal = why
+		s.ctrlSup.mu.Unlock()
+		if changed {
+			s.logger.Warn("controller-supervisor: the controller does not answer, but the restart is refused — the alarm stays",
+				"vmid", vmid, "why", why, "unhealthy_sweeps", seen)
+		}
+	}
+	busy, what, err := s.controllerBusy(ctx, vmid)
+	if err != nil {
+		refuse("the controller's busy file cannot be read: " + err.Error())
+		return
+	}
+	if busy {
+		refuse("the controller is busy: " + what)
+		return
+	}
+	if !s.supervisorBrakeAllows(vmid, now) {
+		refuse("the restart brake")
+		return
+	}
+	reason := fmt.Sprintf("the controller ran but did not answer its health check for %d sweeps (since %s)",
+		seen, since.UTC().Format(time.RFC3339))
+	s.logger.Warn("controller-supervisor: the controller does not answer — restarting the bootstrap unit",
+		"vmid", vmid, "unhealthy_sweeps", seen, "since", since.UTC().Format(time.RFC3339), "unit", bootstrapUnit)
+	if _, err := s.guestExec.GuestExec(ctx, vmid, "systemctl", "restart", bootstrapUnit); err != nil {
+		s.logger.Error("controller-supervisor: bootstrap restart failed", "vmid", vmid, "err", err)
+		reason += "; restart FAILED: " + err.Error()
+	}
+	s.recordSupervisorRestart(vmid, now, reason, func(st *controllerSupState) {
+		st.unresponsiveRestartAt = now
+		st.unhealthySeen = 0
+		st.unrespRefusal = ""
+	})
 }
 
 func (s *Server) resetNotRunning(vmid int) {
@@ -517,6 +595,10 @@ func (s *Server) ControllerSupervisorStatus(_ context.Context) *hub.ControllerSu
 		if !st.unresponsiveSince.IsZero() {
 			g.UnresponsiveSince = st.unresponsiveSince.UTC().Format(time.RFC3339)
 		}
+		g.UnresponsiveRestartAfterS = (controllerUnresponsiveRestart - controllerUnresponsiveConfirm) * int(controllerSupervisorInterval/time.Second)
+		if !st.unresponsiveRestartAt.IsZero() {
+			g.UnresponsiveRestartAt = st.unresponsiveRestartAt.UTC().Format(time.RFC3339)
+		}
 		out.Guests = append(out.Guests, g)
 	}
 	sort.Slice(out.Guests, func(i, j int) bool { return out.Guests[i].VMID < out.Guests[j].VMID })