hub: controller_unresponsive_restart (R-1344 restart, agent >= 0.171.0) — one mail per restart, not also controller_restarted_by_agent; the unresponsive alarm text says what the agent will do

hub/CHANGELOG.md

@@ -1,3 +1,14 @@
+## Unreleased — R-1344: the agent's restart of a controller that does not answer
+
+- **`controller_unresponsive_restart`** (warning, operator-only; in `allowedEventTypes` and `operatorOnlyEvents`, pinned
+  by `TestControllerSupervisorEventsAreAllowlistedAndOperatorOnly`): mailed when the stanza's `unresponsive_restart_at`
+  MOVES (agent ≥ 0.171.0). That restart also moves `last_restart_at`; it is ONE act, so it is NOT also mailed as
+  `controller_restarted_by_agent`.
+- **The `controller_unresponsive` text follows the agent:** with `unresponsive_restart_after_s` it says the agent
+  restarts the controller „2 minutes after this alarm" unless a backup, a restore, an update or the brake holds; without
+  it (an older agent) it says this agent does not restart it. Tests `TestControllerSupervisorChecker_UnresponsiveRestart`,
+  `…UnresponsiveTextFollowsTheAgent` — red first; 3 mutants killed.
+
 ## v0.177.0 (2026-10-29) — „Reboot guest" from the hub (D65 = A); the slow-request line and the vouch save's timing (R-1346)
 
 - **D65 = A (operator ruling 2026-10-29 15:50; `04` §6.1).** The AGENT's closed list (`hostOperatorActionNames`) is

hub/internal/api/controller_supervisor_event_test.go

@@ -10,7 +10,7 @@ import (
 // allowlisted (the project's single register of legitimate types) AND operator-only (a missing
 // customerMessages entry is not a block — the v0.78.0 defect).
 func TestControllerSupervisorEventsAreAllowlistedAndOperatorOnly(t *testing.T) {
-	for _, et := range []string{"controller_restarted_by_agent", "controller_crashloop"} {
+	for _, et := range []string{"controller_restarted_by_agent", "controller_crashloop", "controller_unresponsive", "controller_unresponsive_restart"} {
 		if !allowedEventTypes[et] {
 			t.Fatalf("%s must be in allowedEventTypes (R-77: an unregistered type is an inert seam)", et)
 		}

hub/internal/api/handler.go

@@ -2419,7 +2419,9 @@ var allowedEventTypes = map[string]bool{
 	// R-539 (hub v0.117.0, agent v0.132.0) — hub-generated from slow_crashloop_since; operator-only.
 	"controller_slow_crashloop": true,
 	// R-1344 (2026-10-29): hub-generated from the stanza's unresponsive_since; operator-only (notify.operatorOnlyEvents).
-	"controller_unresponsive": true,
+	// 2026-10-30: controller_unresponsive_restart from unresponsive_restart_at (agent ≥ 0.171.0); operator-only.
+	"controller_unresponsive":         true,
+	"controller_unresponsive_restart": true,
 	// R-550 (hub v0.117.0, controller v0.246.0) — pushed by the controller at startup when it finds a
 	// restore that was in flight when the box stopped. For the household: they can run it again.
 	"restore_interrupted": true,

hub/internal/monitor/controller_supervisor.go

@@ -18,7 +18,10 @@ import (
 //   - `controller_crashloop` (error, operator-only) when a guest's crashloop_since MOVES;
 //   - `controller_slow_crashloop` (warning, operator-only) when a guest's slow_crashloop_since MOVES
 //     (R-539, operator ruling 3 of 2026-09-16, agent v0.132.0): five restarts inside 24 hours, the
-//     loop the 15-minute brake cannot see because each restart falls outside its window.
+//     loop the 15-minute brake cannot see because each restart falls outside its window;
+//   - `controller_unresponsive` (warning, operator-only) when unresponsive_since MOVES (R-1344), and
+//     `controller_unresponsive_restart` (warning, operator-only) when unresponsive_restart_at MOVES (agent ≥ 0.171.0) —
+//     that restart is NOT also mailed as controller_restarted_by_agent.
 //
 // TIMESTAMPS, NOT COUNTERS. The agent's record is in-memory, so an agent restart zeroes
 // restarts_total; keying on a counter would read that as nothing (fine) but a later 1 would not
@@ -39,10 +42,11 @@ type ControllerSupervisorChecker struct {
 }
 
 type supSeen struct {
-	lastRestartAt      string
-	crashloopSince     string
-	slowCrashloopSince string
-	unresponsiveSince  string
+	lastRestartAt         string
+	crashloopSince        string
+	slowCrashloopSince    string
+	unresponsiveSince     string
+	unresponsiveRestartAt string
 }
 
 // ControllerSupervisorGuest mirrors the agent's hub.ControllerSupervisorGuest wire shape.
@@ -62,6 +66,10 @@ type ControllerSupervisorGuest struct {
 	// sweeps. ALARM ONLY — no restart until the measurement licenses one. Absent on older agents → never an event.
 	Unresponsive      bool   `json:"unresponsive"`
 	UnresponsiveSince string `json:"unresponsive_since"`
+	// R-1344 restart (decided 2026-10-30, agent ≥ 0.171.0): how long after the alarm the agent restarts a controller that
+	// still does not answer (0 = an older agent: alarm only), and the last such restart (equals last_restart_at for it).
+	UnresponsiveRestartAfterS int    `json:"unresponsive_restart_after_s"`
+	UnresponsiveRestartAt     string `json:"unresponsive_restart_at"`
 }
 
 // ParseControllerSupervisor extracts the stanza from a host report body. A missing or malformed
@@ -104,7 +112,7 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests
 		key := fmt.Sprintf("%s/%d", hostID, g.VMID)
 		prev, known := c.seen[key]
 		c.seen[key] = supSeen{lastRestartAt: g.LastRestartAt, crashloopSince: g.CrashloopSince, slowCrashloopSince: g.SlowCrashloopSince,
-			unresponsiveSince: g.UnresponsiveSince}
+			unresponsiveSince: g.UnresponsiveSince, unresponsiveRestartAt: g.UnresponsiveRestartAt}
 		if !known {
 			if g.Crashloop && g.CrashloopSince != "" {
 				c.emit(customerID, hostID, g, "controller_crashloop")
@@ -117,7 +125,12 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests
 			}
 			continue
 		}
-		if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt {
+		// A restart for not answering moves last_restart_at too: it is ONE act, mailed once, as the warning.
+		unrespRestart := g.UnresponsiveRestartAt != "" && g.UnresponsiveRestartAt != prev.unresponsiveRestartAt
+		if unrespRestart {
+			c.emit(customerID, hostID, g, "controller_unresponsive_restart")
+		}
+		if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt && !(unrespRestart && g.LastRestartAt == g.UnresponsiveRestartAt) {
 			c.emit(customerID, hostID, g, "controller_restarted_by_agent")
 		}
 		if g.CrashloopSince != "" && g.CrashloopSince != prev.crashloopSince {
@@ -149,8 +162,17 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll
 			hostID, g.VMID, g.Restarts24h, g.LastReason)
 	case "controller_unresponsive":
 		severity = "warning"
-		message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). The agent does NOT restart it yet (R-1344: the measurement must first show a busy night never looks like this) — use „Restart controller\" on the host page if it stays so.",
-			hostID, g.VMID, g.UnresponsiveSince)
+		if g.UnresponsiveRestartAfterS > 0 {
+			message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). The agent restarts it if it still does not answer %s after this alarm — unless a backup, a restore or an update is running, or the restart brake holds; then use „Restart controller\" on the host page.",
+				hostID, g.VMID, g.UnresponsiveSince, humanSeconds(g.UnresponsiveRestartAfterS))
+		} else {
+			message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). This agent does NOT restart it itself (an agent older than 0.171.0) — use „Restart controller\" on the host page if it stays so.",
+				hostID, g.VMID, g.UnresponsiveSince)
+		}
+	case "controller_unresponsive_restart":
+		severity = "warning"
+		message = fmt.Sprintf("Host %s guest %d: the controller ran but did not answer its health check (since %s), so the agent restarted it at %s. Look at the box if this repeats. Reason: %s",
+			hostID, g.VMID, g.UnresponsiveSince, g.UnresponsiveRestartAt, g.LastReason)
 	default:
 		return
 	}
@@ -159,7 +181,7 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll
 		"last_restart_at": g.LastRestartAt, "last_reason": g.LastReason,
 		"crashloop_since": g.CrashloopSince, "parked": g.Parked,
 		"restarts_24h": g.Restarts24h, "slow_crashloop_since": g.SlowCrashloopSince,
-		"unresponsive_since": g.UnresponsiveSince,
+		"unresponsive_since": g.UnresponsiveSince, "unresponsive_restart_at": g.UnresponsiveRestartAt,
 	})
 	c.logger.Printf("[INFO] Controller supervisor: %s %s/%d (%s)", eventType, hostID, g.VMID, g.LastReason)
 	if _, err := c.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil {
@@ -170,3 +192,14 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll
 		c.onEvent(customerID, eventType, severity, message, string(details), "hub")
 	}
 }
+
+// humanSeconds says a whole number of minutes as „2 minutes", anything else as „90 seconds".
+func humanSeconds(n int) string {
+	switch {
+	case n == 60:
+		return "1 minute"
+	case n%60 == 0:
+		return fmt.Sprintf("%d minutes", n/60)
+	}
+	return fmt.Sprintf("%d seconds", n)
+}

hub/internal/monitor/controller_supervisor_test.go

@@ -3,6 +3,7 @@ package monitor
 import (
 	"io"
 	"log"
+	"strings"
 	"sync"
 	"testing"
 
@@ -175,3 +176,48 @@ func TestControllerSupervisorChecker_Unresponsive(t *testing.T) {
 		t.Fatalf("first sight while unresponsive: %v", evs)
 	}
 }
+
+// R-1344 restart (decided 2026-10-30, agent ≥ 0.171.0): a restart for not answering reaches the operator ONCE, as
+// controller_unresponsive_restart (warning) — not also as the info controller_restarted_by_agent, though last_restart_at
+// moved with it; a plain not-running restart afterwards is still controller_restarted_by_agent. RED-PROOF: seen failing
+// before the branch existed.
+func TestControllerSupervisorChecker_UnresponsiveRestart(t *testing.T) {
+	st := newCapStore(t)
+	rec := &evRec{}
+	c := NewControllerSupervisorChecker(st, rec.fn, log.New(io.Discard, "", 0))
+	st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120}]`), store.HostReportDenorm{})
+	c.Check()
+	rec.take() // first sight while unresponsive: the alarm (pinned above)
+	st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":1,"last_restart_at":"2026-10-30T10:02:00Z","last_reason":"the controller ran but did not answer its health check for 10 sweeps","crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T10:02:00Z"}]`), store.HostReportDenorm{})
+	c.Check()
+	if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_unresponsive_restart/warning" {
+		t.Fatalf("got %v, want exactly [controller_unresponsive_restart/warning]", evs)
+	}
+	c.Check()
+	if evs := rec.take(); len(evs) != 0 {
+		t.Fatalf("the same restart emitted again: %v", evs)
+	}
+	st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":2,"last_restart_at":"2026-10-30T11:00:00Z","last_reason":"controller container exited on 2 consecutive sweeps","crashloop":false,"parked":false,"unresponsive":false,"unresponsive_since":"2026-10-30T10:00:00Z","unresponsive_restart_after_s":120,"unresponsive_restart_at":"2026-10-30T10:02:00Z"}]`), store.HostReportDenorm{})
+	c.Check()
+	if evs := rec.take(); len(evs) != 1 || evs[0] != "controller_restarted_by_agent/info" {
+		t.Fatalf("a not-running restart after it: %v", evs)
+	}
+}
+
+// The alarm says what the agent will do — and it must not promise a restart an older agent never makes (the CLAUDE.md
+// corollary: when the behaviour changes, the alarm text changes with it).
+func TestControllerSupervisorChecker_UnresponsiveTextFollowsTheAgent(t *testing.T) {
+	for _, tc := range []struct{ field, want, never string }{
+		{`,"unresponsive_restart_after_s":120`, "restarts it if it still does not answer 2 minutes after this alarm", "does NOT restart"},
+		{``, "does NOT restart it", "restarts it if"},
+	} {
+		st := newCapStore(t)
+		var msg string
+		c := NewControllerSupervisorChecker(st, func(_, _, _, m, _, _ string) { msg = m }, log.New(io.Discard, "", 0))
+		st.SaveHostReport("h1", "c1", supReport(`[{"vmid":9201,"restarts_total":0,"crashloop":false,"parked":false,"unresponsive":true,"unresponsive_since":"2026-10-30T10:00:00Z"`+tc.field+`}]`), store.HostReportDenorm{})
+		c.Check()
+		if !strings.Contains(msg, tc.want) || strings.Contains(msg, tc.never) {
+			t.Fatalf("field %q: the alarm reads %q — want %q, never %q", tc.field, msg, tc.want, tc.never)
+		}
+	}
+}

hub/internal/notify/dispatcher.go

@@ -744,8 +744,10 @@ var recordOnlyEvents = map[string]bool{
 // is customer-grade and would be DISCARDED by a template). Turning the fallback into a gate would
 // change behaviour well outside this concern.
 var operatorOnlyEvents = map[string]bool{
-	// R-1344 (2026-10-29): the controller runs but does not answer (agent stanza, alarm only) — the operator's.
-	"controller_unresponsive": true,
+	// R-1344 (2026-10-29): the controller runs but does not answer (agent stanza) — the operator's; and (2026-10-30) the
+	// agent's restart of it.
+	"controller_unresponsive":         true,
+	"controller_unresponsive_restart": true,
 	// R-97a. A customer can take no action on a failed whole-guest backup, and being told it failed
 	// while it is still retrying behind the R-88 breaker is alarming without being actionable.
 	"whole_guest_backup_failed": true,