v0.132.0: the slow crash-loop counter (R-539, operator ruling 3 of 2026-09-16)

Beside the unchanged 3-in-15-minutes brake, a second counter: restarts the
supervisor performed in the last 24 hours. At the fifth the heartbeat stanza
sets slow_crashloop_since (moving at most once per 24 h), slow_crashloop and
restarts_24h; hub v0.117.0 mints controller_slow_crashloop (warning,
operator-only) when the timestamp moves. It never stops restarting.

Persisted per guest (tmp+rename, 0600) so an agent restart or reboot does not
reset it - unlike the fast record, whose reason for staying in memory (a
persisted give-up outliving the fix) does not apply to a counter that only
warns. Deliberate kills count. The startup line prints the new limits.

Red-proofs seen failing: no counter; the once-per-24h guard removed ('the
operator would be mailed per restart'); the save removed ('Restarts24h:1'
after an agent restart). Negative control: restarts 7 h apart never raise it.
go build/vet/test ./... green, 30 packages.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

internal/localapi/controllersupervisor_test.go

@@ -243,9 +243,108 @@ func TestControllerSupervisorStanza_WireShape(t *testing.T) {
 		t.Fatal(err)
 	}
 	g := m["guests"][0]
-	for _, k := range []string{"vmid", "restarts_total", "last_restart_at", "last_reason", "crashloop", "parked"} {
+	for _, k := range []string{"vmid", "restarts_total", "last_restart_at", "last_reason", "crashloop", "parked", "restarts_24h", "slow_crashloop"} {
 		if _, ok := g[k]; !ok {
 			t.Fatalf("stanza lacks %q — the hub keys on it: %s", k, b)
 		}
 	}
 }
+
+
+// ---- R-539 (operator ruling 3 of 2026-09-16): the SLOW crash loop ----------------------------------
+
+// supKillOnce kills the controller and lets the supervisor restart it (two confirming sweeps), then
+// moves the clock on by gap. The container comes back "running", so each restart is a separate act.
+func supKillOnce(t *testing.T, s *Server, ex *supExec, clk *supClock, gap time.Duration) {
+	t.Helper()
+	before := ex.count(9201)
+	ex.mu.Lock()
+	ex.status[9201] = "exited"
+	ex.mu.Unlock()
+	s.ControllerSupervisorTick(context.Background())
+	clk.t = clk.t.Add(controllerSupervisorInterval)
+	s.ControllerSupervisorTick(context.Background())
+	if ex.count(9201) != before+1 {
+		t.Fatalf("kill was not followed by exactly one restart (restarts %d → %d)", before, ex.count(9201))
+	}
+	clk.t = clk.t.Add(gap)
+}
+
+// The consequence: a controller that dies every 20 minutes — never three times inside the 15-minute
+// brake — raises slow_crashloop on the FIFTH restart in 24 hours, and the raise does not move again on
+// the sixth (the hub mails on movement; once per 24 hours is the ruling).
+//
+// RED-PROOF: without the slow counter the stanza never sets slow_crashloop → "five restarts 20 minutes
+// apart did not raise slow_crashloop — this is R-539".
+func TestControllerSupervisor_SlowCrashloop(t *testing.T) {
+	ex := &supExec{status: map[int]string{9201: "running"}, onRestart: "running"}
+	s, clk, _ := supServer(t, ex, runningGuest(9201), 9201)
+	ctx := context.Background()
+
+	for i := 1; i <= 4; i++ {
+		supKillOnce(t, s, ex, clk, 20*time.Minute)
+	}
+	g := s.ControllerSupervisorStatus(ctx).Guests[0]
+	if g.Crashloop {
+		t.Fatalf("the 15-minute brake fired on restarts 20 minutes apart — the fixture is wrong: %+v", g)
+	}
+	if g.SlowCrashloop || g.SlowCrashloopSince != "" {
+		t.Fatalf("slow_crashloop raised after only 4 restarts: %+v", g)
+	}
+	supKillOnce(t, s, ex, clk, 20*time.Minute)
+	g = s.ControllerSupervisorStatus(ctx).Guests[0]
+	if !g.SlowCrashloop || g.SlowCrashloopSince == "" || g.Restarts24h != 5 {
+		t.Fatalf("five restarts 20 minutes apart did not raise slow_crashloop — this is R-539: %+v", g)
+	}
+	first := g.SlowCrashloopSince
+	supKillOnce(t, s, ex, clk, 20*time.Minute)
+	g = s.ControllerSupervisorStatus(ctx).Guests[0]
+	if g.SlowCrashloopSince != first {
+		t.Fatalf("the raise moved again on the 6th restart (%q → %q) — the operator would be mailed per restart", first, g.SlowCrashloopSince)
+	}
+	if !g.SlowCrashloop {
+		t.Fatalf("slow_crashloop cleared while the loop continues: %+v", g)
+	}
+}
+
+// The negative control: restarts that never reach five inside any 24 hours never raise it.
+func TestControllerSupervisor_SpreadRestartsNeverSlowCrashloop(t *testing.T) {
+	ex := &supExec{status: map[int]string{9201: "running"}, onRestart: "running"}
+	s, clk, _ := supServer(t, ex, runningGuest(9201), 9201)
+	for i := 0; i < 8; i++ { // eight restarts, 7 hours apart: at most 4 inside any 24 hours
+		supKillOnce(t, s, ex, clk, 7*time.Hour)
+	}
+	g := s.ControllerSupervisorStatus(context.Background()).Guests[0]
+	if g.SlowCrashloop || g.SlowCrashloopSince != "" {
+		t.Fatalf("restarts 7 hours apart raised slow_crashloop: %+v", g)
+	}
+	if g.Restarts24h > 4 {
+		t.Fatalf("restarts_24h=%d — the 24-hour window is not pruning", g.Restarts24h)
+	}
+}
+
+// An agent restart must not reset the slow counter (the ruling; a box whose AGENT also restarts would
+// otherwise never reach five). The same state directory, a fresh Server.
+//
+// RED-PROOF: keep the counter in memory only → the second Server starts at 0 → "the agent restart
+// reset the slow counter".
+func TestControllerSupervisor_SlowCounterSurvivesAgentRestart(t *testing.T) {
+	ex := &supExec{status: map[int]string{9201: "running"}, onRestart: "running"}
+	s, clk, dir := supServer(t, ex, runningGuest(9201), 9201)
+	for i := 0; i < 4; i++ {
+		supKillOnce(t, s, ex, clk, 20*time.Minute)
+	}
+	s2 := &Server{
+		staleLock:    s.staleLock,
+		guestExec:    ex,
+		guestsDir:    dir,
+		swapInFlight: map[int]bool{},
+		logger:       s.logger,
+		now:          clk.now,
+	}
+	supKillOnce(t, s2, ex, clk, 20*time.Minute)
+	g := s2.ControllerSupervisorStatus(context.Background()).Guests[0]
+	if g.Restarts24h != 5 || !g.SlowCrashloop {
+		t.Fatalf("the agent restart reset the slow counter: %+v", g)
+	}
+}