206 lines
9.8 KiB
Go
206 lines
9.8 KiB
Go
package monitor
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"sync"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// R-523 (felhom-agent v0.131.0). The agent's in-guest controller supervisor restarts a controller
|
|
// container that is not running and, after 3 restarts in 15 minutes, gives up for 30 minutes. The
|
|
// agent has no event channel of its own — its heartbeat is the channel — so the per-guest record
|
|
// rides the host report as `controller_supervisor`, and this checker turns movement in it into events:
|
|
//
|
|
// - `controller_restarted_by_agent` (info) when a guest's last_restart_at MOVES to a new value;
|
|
// - `controller_crashloop` (error, operator-only) when a guest's crashloop_since MOVES;
|
|
// - `controller_slow_crashloop` (warning, operator-only) when a guest's slow_crashloop_since MOVES
|
|
// (R-539, operator ruling 3 of 2026-09-16, agent v0.132.0): five restarts inside 24 hours, the
|
|
// loop the 15-minute brake cannot see because each restart falls outside its window;
|
|
// - `controller_unresponsive` (warning, operator-only) when unresponsive_since MOVES (R-1344), and
|
|
// `controller_unresponsive_restart` (warning, operator-only) when unresponsive_restart_at MOVES (agent ≥ 0.171.0) —
|
|
// that restart is NOT also mailed as controller_restarted_by_agent.
|
|
//
|
|
// TIMESTAMPS, NOT COUNTERS. The agent's record is in-memory, so an agent restart zeroes
|
|
// restarts_total; keying on a counter would read that as nothing (fine) but a later 1 would not
|
|
// exceed the remembered 3 (a lost event). A timestamp that changes is a new act whatever the counter
|
|
// says.
|
|
//
|
|
// First observation (hub restart, new host): SEED SILENTLY, except a guest that is IN a crash-loop at
|
|
// first sight emits once — a hub restarted during an outage must not stay silent about it (the
|
|
// HostCapabilityChecker F2 rule). Both types are pinned in allowedEventTypes and operatorOnlyEvents
|
|
// by controller_supervisor_event_test.go in the api package.
|
|
type ControllerSupervisorChecker struct {
|
|
store *store.Store
|
|
logger *log.Logger
|
|
onEvent EventNotifyFunc
|
|
|
|
mu sync.Mutex
|
|
seen map[string]supSeen // hostID/vmid → last timestamps seen
|
|
}
|
|
|
|
type supSeen struct {
|
|
lastRestartAt string
|
|
crashloopSince string
|
|
slowCrashloopSince string
|
|
unresponsiveSince string
|
|
unresponsiveRestartAt string
|
|
}
|
|
|
|
// ControllerSupervisorGuest mirrors the agent's hub.ControllerSupervisorGuest wire shape.
|
|
type ControllerSupervisorGuest struct {
|
|
VMID int `json:"vmid"`
|
|
RestartsTotal int `json:"restarts_total"`
|
|
LastRestartAt string `json:"last_restart_at"`
|
|
LastReason string `json:"last_reason"`
|
|
Crashloop bool `json:"crashloop"`
|
|
CrashloopSince string `json:"crashloop_since"`
|
|
Parked bool `json:"parked"`
|
|
// R-539 (agent v0.132.0). Absent on older agents → zero values → never an event.
|
|
Restarts24h int `json:"restarts_24h"`
|
|
SlowCrashloop bool `json:"slow_crashloop"`
|
|
SlowCrashloopSince string `json:"slow_crashloop_since"`
|
|
// R-1344 (agent ≥ the 2026-10-29 release): the controller runs but Docker's health check has failed for several
|
|
// sweeps. ALARM ONLY — no restart until the measurement licenses one. Absent on older agents → never an event.
|
|
Unresponsive bool `json:"unresponsive"`
|
|
UnresponsiveSince string `json:"unresponsive_since"`
|
|
// R-1344 restart (decided 2026-10-30, agent ≥ 0.171.0): how long after the alarm the agent restarts a controller that
|
|
// still does not answer (0 = an older agent: alarm only), and the last such restart (equals last_restart_at for it).
|
|
UnresponsiveRestartAfterS int `json:"unresponsive_restart_after_s"`
|
|
UnresponsiveRestartAt string `json:"unresponsive_restart_at"`
|
|
}
|
|
|
|
// ParseControllerSupervisor extracts the stanza from a host report body. A missing or malformed
|
|
// stanza (a pre-v0.131.0 agent) yields nil — never an event.
|
|
func ParseControllerSupervisor(reportJSON string) []ControllerSupervisorGuest {
|
|
var body struct {
|
|
CS *struct {
|
|
Guests []ControllerSupervisorGuest `json:"guests"`
|
|
} `json:"controller_supervisor"`
|
|
}
|
|
if err := json.Unmarshal([]byte(reportJSON), &body); err != nil || body.CS == nil {
|
|
return nil
|
|
}
|
|
return body.CS.Guests
|
|
}
|
|
|
|
func NewControllerSupervisorChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *ControllerSupervisorChecker {
|
|
return &ControllerSupervisorChecker{store: s, logger: logger, onEvent: onEvent, seen: map[string]supSeen{}}
|
|
}
|
|
|
|
// Check reads every host's latest report and emits on movement. Same 60 s sweep as its siblings.
|
|
func (c *ControllerSupervisorChecker) Check() {
|
|
rows, err := c.store.GetLatestHostReports()
|
|
if err != nil {
|
|
c.logger.Printf("[WARN] Controller supervisor check failed: %v", err)
|
|
return
|
|
}
|
|
for _, row := range rows {
|
|
if c.store.IsCustomerBlocked(row.CustomerID) {
|
|
continue
|
|
}
|
|
c.observe(row.HostID, row.CustomerID, ParseControllerSupervisor(row.ReportJSON))
|
|
}
|
|
}
|
|
|
|
func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests []ControllerSupervisorGuest) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
for _, g := range guests {
|
|
key := fmt.Sprintf("%s/%d", hostID, g.VMID)
|
|
prev, known := c.seen[key]
|
|
c.seen[key] = supSeen{lastRestartAt: g.LastRestartAt, crashloopSince: g.CrashloopSince, slowCrashloopSince: g.SlowCrashloopSince,
|
|
unresponsiveSince: g.UnresponsiveSince, unresponsiveRestartAt: g.UnresponsiveRestartAt}
|
|
if !known {
|
|
if g.Crashloop && g.CrashloopSince != "" {
|
|
c.emit(customerID, hostID, g, "controller_crashloop")
|
|
}
|
|
if g.SlowCrashloop && g.SlowCrashloopSince != "" {
|
|
c.emit(customerID, hostID, g, "controller_slow_crashloop")
|
|
}
|
|
if g.Unresponsive && g.UnresponsiveSince != "" {
|
|
c.emit(customerID, hostID, g, "controller_unresponsive")
|
|
}
|
|
continue
|
|
}
|
|
// A restart for not answering moves last_restart_at too: it is ONE act, mailed once, as the warning.
|
|
unrespRestart := g.UnresponsiveRestartAt != "" && g.UnresponsiveRestartAt != prev.unresponsiveRestartAt
|
|
if unrespRestart {
|
|
c.emit(customerID, hostID, g, "controller_unresponsive_restart")
|
|
}
|
|
if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt && !(unrespRestart && g.LastRestartAt == g.UnresponsiveRestartAt) {
|
|
c.emit(customerID, hostID, g, "controller_restarted_by_agent")
|
|
}
|
|
if g.CrashloopSince != "" && g.CrashloopSince != prev.crashloopSince {
|
|
c.emit(customerID, hostID, g, "controller_crashloop")
|
|
}
|
|
if g.SlowCrashloopSince != "" && g.SlowCrashloopSince != prev.slowCrashloopSince {
|
|
c.emit(customerID, hostID, g, "controller_slow_crashloop")
|
|
}
|
|
if g.UnresponsiveSince != "" && g.UnresponsiveSince != prev.unresponsiveSince {
|
|
c.emit(customerID, hostID, g, "controller_unresponsive")
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g ControllerSupervisorGuest, eventType string) {
|
|
var severity, message string
|
|
switch eventType {
|
|
case "controller_restarted_by_agent":
|
|
severity = "info"
|
|
message = fmt.Sprintf("Host %s guest %d: the agent restarted the controller (%s) — restart #%d since the agent started",
|
|
hostID, g.VMID, g.LastReason, g.RestartsTotal)
|
|
case "controller_crashloop":
|
|
severity = "error"
|
|
message = fmt.Sprintf("Host %s guest %d: the controller will not stay up — the agent stopped restarting it after repeated attempts (since %s) and will try again in 30 minutes. Last reason: %s",
|
|
hostID, g.VMID, g.CrashloopSince, g.LastReason)
|
|
case "controller_slow_crashloop":
|
|
severity = "warning"
|
|
message = fmt.Sprintf("Host %s guest %d: the controller keeps dying — the agent restarted it %d times in 24 hours (each one too far apart for the 15-minute brake). It is still being restarted; this is the warning that it will not stay up. Last reason: %s",
|
|
hostID, g.VMID, g.Restarts24h, g.LastReason)
|
|
case "controller_unresponsive":
|
|
severity = "warning"
|
|
if g.UnresponsiveRestartAfterS > 0 {
|
|
message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). The agent restarts it if it still does not answer %s after this alarm — unless a backup, a restore or an update is running, or the restart brake holds; then use „Restart controller\" on the host page.",
|
|
hostID, g.VMID, g.UnresponsiveSince, humanSeconds(g.UnresponsiveRestartAfterS))
|
|
} else {
|
|
message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). This agent does NOT restart it itself (an agent older than 0.171.0) — use „Restart controller\" on the host page if it stays so.",
|
|
hostID, g.VMID, g.UnresponsiveSince)
|
|
}
|
|
case "controller_unresponsive_restart":
|
|
severity = "warning"
|
|
message = fmt.Sprintf("Host %s guest %d: the controller ran but did not answer its health check (since %s), so the agent restarted it at %s. Look at the box if this repeats. Reason: %s",
|
|
hostID, g.VMID, g.UnresponsiveSince, g.UnresponsiveRestartAt, g.LastReason)
|
|
default:
|
|
return
|
|
}
|
|
details, _ := json.Marshal(map[string]any{
|
|
"host_id": hostID, "vmid": g.VMID, "restarts_total": g.RestartsTotal,
|
|
"last_restart_at": g.LastRestartAt, "last_reason": g.LastReason,
|
|
"crashloop_since": g.CrashloopSince, "parked": g.Parked,
|
|
"restarts_24h": g.Restarts24h, "slow_crashloop_since": g.SlowCrashloopSince,
|
|
"unresponsive_since": g.UnresponsiveSince, "unresponsive_restart_at": g.UnresponsiveRestartAt,
|
|
})
|
|
c.logger.Printf("[INFO] Controller supervisor: %s %s/%d (%s)", eventType, hostID, g.VMID, g.LastReason)
|
|
if _, err := c.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil {
|
|
c.logger.Printf("[WARN] save %s for %s: %v", eventType, hostID, err)
|
|
return
|
|
}
|
|
if c.onEvent != nil {
|
|
c.onEvent(customerID, eventType, severity, message, string(details), "hub")
|
|
}
|
|
}
|
|
|
|
// humanSeconds says a whole number of minutes as „2 minutes", anything else as „90 seconds".
|
|
func humanSeconds(n int) string {
|
|
switch {
|
|
case n == 60:
|
|
return "1 minute"
|
|
case n%60 == 0:
|
|
return fmt.Sprintf("%d minutes", n/60)
|
|
}
|
|
return fmt.Sprintf("%d seconds", n)
|
|
}
|