hub: Restart controller (the agent's closed list on the host-report reply) and Clear hold (D64 = B, the controller's list) on the host page, each behind a confirm; controller_unresponsive (R-1344, alarm only); 04 §6.1 + 09 §3 decision 283; wire gate declares both new wires; R-1344 filed — tests red first
gates / gates (push) Successful in 3m23s

This commit is contained in:
admin committed 2026-10-29 12:31:33 +01:00
1 parent 70862fa7b3
commit 3d0c907b56
20 files changed
+663 -20

No files matched your search

+18 -1
View File
@@ -42,6 +42,7 @@ type supSeen struct {
lastRestartAt string
crashloopSince string
slowCrashloopSince string
unresponsiveSince string
}
// ControllerSupervisorGuest mirrors the agent's hub.ControllerSupervisorGuest wire shape.
@@ -57,6 +58,10 @@ type ControllerSupervisorGuest struct {
Restarts24h int `json:"restarts_24h"`
SlowCrashloop bool `json:"slow_crashloop"`
SlowCrashloopSince string `json:"slow_crashloop_since"`
// R-1344 (agent ≥ the 2026-10-29 release): the controller runs but Docker's health check has failed for several
// sweeps. ALARM ONLY — no restart until the measurement licenses one. Absent on older agents → never an event.
Unresponsive bool `json:"unresponsive"`
UnresponsiveSince string `json:"unresponsive_since"`
}
// ParseControllerSupervisor extracts the stanza from a host report body. A missing or malformed
@@ -98,7 +103,8 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests
for _, g := range guests {
key := fmt.Sprintf("%s/%d", hostID, g.VMID)
prev, known := c.seen[key]
c.seen[key] = supSeen{lastRestartAt: g.LastRestartAt, crashloopSince: g.CrashloopSince, slowCrashloopSince: g.SlowCrashloopSince}
c.seen[key] = supSeen{lastRestartAt: g.LastRestartAt, crashloopSince: g.CrashloopSince, slowCrashloopSince: g.SlowCrashloopSince,
unresponsiveSince: g.UnresponsiveSince}
if !known {
if g.Crashloop && g.CrashloopSince != "" {
c.emit(customerID, hostID, g, "controller_crashloop")
@@ -106,6 +112,9 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests
if g.SlowCrashloop && g.SlowCrashloopSince != "" {
c.emit(customerID, hostID, g, "controller_slow_crashloop")
}
if g.Unresponsive && g.UnresponsiveSince != "" {
c.emit(customerID, hostID, g, "controller_unresponsive")
}
continue
}
if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt {
@@ -117,6 +126,9 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests
if g.SlowCrashloopSince != "" && g.SlowCrashloopSince != prev.slowCrashloopSince {
c.emit(customerID, hostID, g, "controller_slow_crashloop")
}
if g.UnresponsiveSince != "" && g.UnresponsiveSince != prev.unresponsiveSince {
c.emit(customerID, hostID, g, "controller_unresponsive")
}
}
}
@@ -135,6 +147,10 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll
severity = "warning"
message = fmt.Sprintf("Host %s guest %d: the controller keeps dying — the agent restarted it %d times in 24 hours (each one too far apart for the 15-minute brake). It is still being restarted; this is the warning that it will not stay up. Last reason: %s",
hostID, g.VMID, g.Restarts24h, g.LastReason)
case "controller_unresponsive":
severity = "warning"
message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). The agent does NOT restart it yet (R-1344: the measurement must first show a busy night never looks like this) — use „Restart controller\" on the host page if it stays so.",
hostID, g.VMID, g.UnresponsiveSince)
default:
return
}
@@ -143,6 +159,7 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll
"last_restart_at": g.LastRestartAt, "last_reason": g.LastReason,
"crashloop_since": g.CrashloopSince, "parked": g.Parked,
"restarts_24h": g.Restarts24h, "slow_crashloop_since": g.SlowCrashloopSince,
"unresponsive_since": g.UnresponsiveSince,
})
c.logger.Printf("[INFO] Controller supervisor: %s %s/%d (%s)", eventType, hostID, g.VMID, g.LastReason)
if _, err := c.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil {