hub: controller_unresponsive_restart (R-1344 restart, agent >= 0.171.0) — one mail per restart, not also controller_restarted_by_agent; the unresponsive alarm text says what the agent will do

hub/internal/monitor/controller_supervisor.go

@@ -18,7 +18,10 @@ import (
 //   - `controller_crashloop` (error, operator-only) when a guest's crashloop_since MOVES;
 //   - `controller_slow_crashloop` (warning, operator-only) when a guest's slow_crashloop_since MOVES
 //     (R-539, operator ruling 3 of 2026-09-16, agent v0.132.0): five restarts inside 24 hours, the
-//     loop the 15-minute brake cannot see because each restart falls outside its window.
+//     loop the 15-minute brake cannot see because each restart falls outside its window;
+//   - `controller_unresponsive` (warning, operator-only) when unresponsive_since MOVES (R-1344), and
+//     `controller_unresponsive_restart` (warning, operator-only) when unresponsive_restart_at MOVES (agent ≥ 0.171.0) —
+//     that restart is NOT also mailed as controller_restarted_by_agent.
 //
 // TIMESTAMPS, NOT COUNTERS. The agent's record is in-memory, so an agent restart zeroes
 // restarts_total; keying on a counter would read that as nothing (fine) but a later 1 would not
@@ -39,10 +42,11 @@ type ControllerSupervisorChecker struct {
 }
 
 type supSeen struct {
-	lastRestartAt      string
-	crashloopSince     string
-	slowCrashloopSince string
-	unresponsiveSince  string
+	lastRestartAt         string
+	crashloopSince        string
+	slowCrashloopSince    string
+	unresponsiveSince     string
+	unresponsiveRestartAt string
 }
 
 // ControllerSupervisorGuest mirrors the agent's hub.ControllerSupervisorGuest wire shape.
@@ -62,6 +66,10 @@ type ControllerSupervisorGuest struct {
 	// sweeps. ALARM ONLY — no restart until the measurement licenses one. Absent on older agents → never an event.
 	Unresponsive      bool   `json:"unresponsive"`
 	UnresponsiveSince string `json:"unresponsive_since"`
+	// R-1344 restart (decided 2026-10-30, agent ≥ 0.171.0): how long after the alarm the agent restarts a controller that
+	// still does not answer (0 = an older agent: alarm only), and the last such restart (equals last_restart_at for it).
+	UnresponsiveRestartAfterS int    `json:"unresponsive_restart_after_s"`
+	UnresponsiveRestartAt     string `json:"unresponsive_restart_at"`
 }
 
 // ParseControllerSupervisor extracts the stanza from a host report body. A missing or malformed
@@ -104,7 +112,7 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests
 		key := fmt.Sprintf("%s/%d", hostID, g.VMID)
 		prev, known := c.seen[key]
 		c.seen[key] = supSeen{lastRestartAt: g.LastRestartAt, crashloopSince: g.CrashloopSince, slowCrashloopSince: g.SlowCrashloopSince,
-			unresponsiveSince: g.UnresponsiveSince}
+			unresponsiveSince: g.UnresponsiveSince, unresponsiveRestartAt: g.UnresponsiveRestartAt}
 		if !known {
 			if g.Crashloop && g.CrashloopSince != "" {
 				c.emit(customerID, hostID, g, "controller_crashloop")
@@ -117,7 +125,12 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests
 			}
 			continue
 		}
-		if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt {
+		// A restart for not answering moves last_restart_at too: it is ONE act, mailed once, as the warning.
+		unrespRestart := g.UnresponsiveRestartAt != "" && g.UnresponsiveRestartAt != prev.unresponsiveRestartAt
+		if unrespRestart {
+			c.emit(customerID, hostID, g, "controller_unresponsive_restart")
+		}
+		if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt && !(unrespRestart && g.LastRestartAt == g.UnresponsiveRestartAt) {
 			c.emit(customerID, hostID, g, "controller_restarted_by_agent")
 		}
 		if g.CrashloopSince != "" && g.CrashloopSince != prev.crashloopSince {
@@ -149,8 +162,17 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll
 			hostID, g.VMID, g.Restarts24h, g.LastReason)
 	case "controller_unresponsive":
 		severity = "warning"
-		message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). The agent does NOT restart it yet (R-1344: the measurement must first show a busy night never looks like this) — use „Restart controller\" on the host page if it stays so.",
-			hostID, g.VMID, g.UnresponsiveSince)
+		if g.UnresponsiveRestartAfterS > 0 {
+			message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). The agent restarts it if it still does not answer %s after this alarm — unless a backup, a restore or an update is running, or the restart brake holds; then use „Restart controller\" on the host page.",
+				hostID, g.VMID, g.UnresponsiveSince, humanSeconds(g.UnresponsiveRestartAfterS))
+		} else {
+			message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). This agent does NOT restart it itself (an agent older than 0.171.0) — use „Restart controller\" on the host page if it stays so.",
+				hostID, g.VMID, g.UnresponsiveSince)
+		}
+	case "controller_unresponsive_restart":
+		severity = "warning"
+		message = fmt.Sprintf("Host %s guest %d: the controller ran but did not answer its health check (since %s), so the agent restarted it at %s. Look at the box if this repeats. Reason: %s",
+			hostID, g.VMID, g.UnresponsiveSince, g.UnresponsiveRestartAt, g.LastReason)
 	default:
 		return
 	}
@@ -159,7 +181,7 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll
 		"last_restart_at": g.LastRestartAt, "last_reason": g.LastReason,
 		"crashloop_since": g.CrashloopSince, "parked": g.Parked,
 		"restarts_24h": g.Restarts24h, "slow_crashloop_since": g.SlowCrashloopSince,
-		"unresponsive_since": g.UnresponsiveSince,
+		"unresponsive_since": g.UnresponsiveSince, "unresponsive_restart_at": g.UnresponsiveRestartAt,
 	})
 	c.logger.Printf("[INFO] Controller supervisor: %s %s/%d (%s)", eventType, hostID, g.VMID, g.LastReason)
 	if _, err := c.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil {
@@ -170,3 +192,14 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll
 		c.onEvent(customerID, eventType, severity, message, string(details), "hub")
 	}
 }
+
+// humanSeconds says a whole number of minutes as „2 minutes", anything else as „90 seconds".
+func humanSeconds(n int) string {
+	switch {
+	case n == 60:
+		return "1 minute"
+	case n%60 == 0:
+		return fmt.Sprintf("%d minutes", n/60)
+	}
+	return fmt.Sprintf("%d seconds", n)
+}