hub: Restart controller (the agent's closed list on the host-report reply) and Clear hold (D64 = B, the controller's list) on the host page, each behind a confirm; controller_unresponsive (R-1344, alarm only); 04 §6.1 + 09 §3 decision 283; wire gate declares both new wires; R-1344 filed — tests red first
gates / gates (push) Successful in 3m23s
gates / gates (push) Successful in 3m23s
This commit is contained in:
1 parent
70862fa7b3
commit
3d0c907b56
20 files changed
+663
-20
No files matched your search
@@ -42,6 +42,7 @@ type supSeen struct {
|
||||
lastRestartAt string
|
||||
crashloopSince string
|
||||
slowCrashloopSince string
|
||||
unresponsiveSince string
|
||||
}
|
||||
|
||||
// ControllerSupervisorGuest mirrors the agent's hub.ControllerSupervisorGuest wire shape.
|
||||
@@ -57,6 +58,10 @@ type ControllerSupervisorGuest struct {
|
||||
Restarts24h int `json:"restarts_24h"`
|
||||
SlowCrashloop bool `json:"slow_crashloop"`
|
||||
SlowCrashloopSince string `json:"slow_crashloop_since"`
|
||||
// R-1344 (agent ≥ the 2026-10-29 release): the controller runs but Docker's health check has failed for several
|
||||
// sweeps. ALARM ONLY — no restart until the measurement licenses one. Absent on older agents → never an event.
|
||||
Unresponsive bool `json:"unresponsive"`
|
||||
UnresponsiveSince string `json:"unresponsive_since"`
|
||||
}
|
||||
|
||||
// ParseControllerSupervisor extracts the stanza from a host report body. A missing or malformed
|
||||
@@ -98,7 +103,8 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests
|
||||
for _, g := range guests {
|
||||
key := fmt.Sprintf("%s/%d", hostID, g.VMID)
|
||||
prev, known := c.seen[key]
|
||||
c.seen[key] = supSeen{lastRestartAt: g.LastRestartAt, crashloopSince: g.CrashloopSince, slowCrashloopSince: g.SlowCrashloopSince}
|
||||
c.seen[key] = supSeen{lastRestartAt: g.LastRestartAt, crashloopSince: g.CrashloopSince, slowCrashloopSince: g.SlowCrashloopSince,
|
||||
unresponsiveSince: g.UnresponsiveSince}
|
||||
if !known {
|
||||
if g.Crashloop && g.CrashloopSince != "" {
|
||||
c.emit(customerID, hostID, g, "controller_crashloop")
|
||||
@@ -106,6 +112,9 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests
|
||||
if g.SlowCrashloop && g.SlowCrashloopSince != "" {
|
||||
c.emit(customerID, hostID, g, "controller_slow_crashloop")
|
||||
}
|
||||
if g.Unresponsive && g.UnresponsiveSince != "" {
|
||||
c.emit(customerID, hostID, g, "controller_unresponsive")
|
||||
}
|
||||
continue
|
||||
}
|
||||
if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt {
|
||||
@@ -117,6 +126,9 @@ func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests
|
||||
if g.SlowCrashloopSince != "" && g.SlowCrashloopSince != prev.slowCrashloopSince {
|
||||
c.emit(customerID, hostID, g, "controller_slow_crashloop")
|
||||
}
|
||||
if g.UnresponsiveSince != "" && g.UnresponsiveSince != prev.unresponsiveSince {
|
||||
c.emit(customerID, hostID, g, "controller_unresponsive")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,6 +147,10 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll
|
||||
severity = "warning"
|
||||
message = fmt.Sprintf("Host %s guest %d: the controller keeps dying — the agent restarted it %d times in 24 hours (each one too far apart for the 15-minute brake). It is still being restarted; this is the warning that it will not stay up. Last reason: %s",
|
||||
hostID, g.VMID, g.Restarts24h, g.LastReason)
|
||||
case "controller_unresponsive":
|
||||
severity = "warning"
|
||||
message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). The agent does NOT restart it yet (R-1344: the measurement must first show a busy night never looks like this) — use „Restart controller\" on the host page if it stays so.",
|
||||
hostID, g.VMID, g.UnresponsiveSince)
|
||||
default:
|
||||
return
|
||||
}
|
||||
@@ -143,6 +159,7 @@ func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g Controll
|
||||
"last_restart_at": g.LastRestartAt, "last_reason": g.LastReason,
|
||||
"crashloop_since": g.CrashloopSince, "parked": g.Parked,
|
||||
"restarts_24h": g.Restarts24h, "slow_crashloop_since": g.SlowCrashloopSince,
|
||||
"unresponsive_since": g.UnresponsiveSince,
|
||||
})
|
||||
c.logger.Printf("[INFO] Controller supervisor: %s %s/%d (%s)", eventType, hostID, g.VMID, g.LastReason)
|
||||
if _, err := c.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil {
|
||||
|
||||
Reference in new issue
Block a user