package monitor import ( "encoding/json" "fmt" "log" "sync" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // R-523 (felhom-agent v0.131.0). The agent's in-guest controller supervisor restarts a controller // container that is not running and, after 3 restarts in 15 minutes, gives up for 30 minutes. The // agent has no event channel of its own — its heartbeat is the channel — so the per-guest record // rides the host report as `controller_supervisor`, and this checker turns movement in it into events: // // - `controller_restarted_by_agent` (info) when a guest's last_restart_at MOVES to a new value; // - `controller_crashloop` (error, operator-only) when a guest's crashloop_since MOVES; // - `controller_slow_crashloop` (warning, operator-only) when a guest's slow_crashloop_since MOVES // (R-539, operator ruling 3 of 2026-09-16, agent v0.132.0): five restarts inside 24 hours, the // loop the 15-minute brake cannot see because each restart falls outside its window; // - `controller_unresponsive` (warning, operator-only) when unresponsive_since MOVES (R-1344), and // `controller_unresponsive_restart` (warning, operator-only) when unresponsive_restart_at MOVES (agent ≥ 0.171.0) — // that restart is NOT also mailed as controller_restarted_by_agent. // // TIMESTAMPS, NOT COUNTERS. The agent's record is in-memory, so an agent restart zeroes // restarts_total; keying on a counter would read that as nothing (fine) but a later 1 would not // exceed the remembered 3 (a lost event). A timestamp that changes is a new act whatever the counter // says. // // First observation (hub restart, new host): SEED SILENTLY, except a guest that is IN a crash-loop at // first sight emits once — a hub restarted during an outage must not stay silent about it (the // HostCapabilityChecker F2 rule). Both types are pinned in allowedEventTypes and operatorOnlyEvents // by controller_supervisor_event_test.go in the api package. type ControllerSupervisorChecker struct { store *store.Store logger *log.Logger onEvent EventNotifyFunc mu sync.Mutex seen map[string]supSeen // hostID/vmid → last timestamps seen } type supSeen struct { lastRestartAt string crashloopSince string slowCrashloopSince string unresponsiveSince string unresponsiveRestartAt string } // ControllerSupervisorGuest mirrors the agent's hub.ControllerSupervisorGuest wire shape. type ControllerSupervisorGuest struct { VMID int `json:"vmid"` RestartsTotal int `json:"restarts_total"` LastRestartAt string `json:"last_restart_at"` LastReason string `json:"last_reason"` Crashloop bool `json:"crashloop"` CrashloopSince string `json:"crashloop_since"` Parked bool `json:"parked"` // R-539 (agent v0.132.0). Absent on older agents → zero values → never an event. Restarts24h int `json:"restarts_24h"` SlowCrashloop bool `json:"slow_crashloop"` SlowCrashloopSince string `json:"slow_crashloop_since"` // R-1344 (agent ≥ the 2026-10-29 release): the controller runs but Docker's health check has failed for several // sweeps. ALARM ONLY — no restart until the measurement licenses one. Absent on older agents → never an event. Unresponsive bool `json:"unresponsive"` UnresponsiveSince string `json:"unresponsive_since"` // R-1344 restart (decided 2026-10-30, agent ≥ 0.171.0): how long after the alarm the agent restarts a controller that // still does not answer (0 = an older agent: alarm only), and the last such restart (equals last_restart_at for it). UnresponsiveRestartAfterS int `json:"unresponsive_restart_after_s"` UnresponsiveRestartAt string `json:"unresponsive_restart_at"` } // ParseControllerSupervisor extracts the stanza from a host report body. A missing or malformed // stanza (a pre-v0.131.0 agent) yields nil — never an event. func ParseControllerSupervisor(reportJSON string) []ControllerSupervisorGuest { var body struct { CS *struct { Guests []ControllerSupervisorGuest `json:"guests"` } `json:"controller_supervisor"` } if err := json.Unmarshal([]byte(reportJSON), &body); err != nil || body.CS == nil { return nil } return body.CS.Guests } func NewControllerSupervisorChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *ControllerSupervisorChecker { return &ControllerSupervisorChecker{store: s, logger: logger, onEvent: onEvent, seen: map[string]supSeen{}} } // Check reads every host's latest report and emits on movement. Same 60 s sweep as its siblings. func (c *ControllerSupervisorChecker) Check() { rows, err := c.store.GetLatestHostReports() if err != nil { c.logger.Printf("[WARN] Controller supervisor check failed: %v", err) return } for _, row := range rows { if c.store.IsCustomerBlocked(row.CustomerID) { continue } c.observe(row.HostID, row.CustomerID, ParseControllerSupervisor(row.ReportJSON)) } } func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests []ControllerSupervisorGuest) { c.mu.Lock() defer c.mu.Unlock() for _, g := range guests { key := fmt.Sprintf("%s/%d", hostID, g.VMID) prev, known := c.seen[key] c.seen[key] = supSeen{lastRestartAt: g.LastRestartAt, crashloopSince: g.CrashloopSince, slowCrashloopSince: g.SlowCrashloopSince, unresponsiveSince: g.UnresponsiveSince, unresponsiveRestartAt: g.UnresponsiveRestartAt} if !known { if g.Crashloop && g.CrashloopSince != "" { c.emit(customerID, hostID, g, "controller_crashloop") } if g.SlowCrashloop && g.SlowCrashloopSince != "" { c.emit(customerID, hostID, g, "controller_slow_crashloop") } if g.Unresponsive && g.UnresponsiveSince != "" { c.emit(customerID, hostID, g, "controller_unresponsive") } continue } // A restart for not answering moves last_restart_at too: it is ONE act, mailed once, as the warning. unrespRestart := g.UnresponsiveRestartAt != "" && g.UnresponsiveRestartAt != prev.unresponsiveRestartAt if unrespRestart { c.emit(customerID, hostID, g, "controller_unresponsive_restart") } if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt && !(unrespRestart && g.LastRestartAt == g.UnresponsiveRestartAt) { c.emit(customerID, hostID, g, "controller_restarted_by_agent") } if g.CrashloopSince != "" && g.CrashloopSince != prev.crashloopSince { c.emit(customerID, hostID, g, "controller_crashloop") } if g.SlowCrashloopSince != "" && g.SlowCrashloopSince != prev.slowCrashloopSince { c.emit(customerID, hostID, g, "controller_slow_crashloop") } if g.UnresponsiveSince != "" && g.UnresponsiveSince != prev.unresponsiveSince { c.emit(customerID, hostID, g, "controller_unresponsive") } } } func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g ControllerSupervisorGuest, eventType string) { var severity, message string switch eventType { case "controller_restarted_by_agent": severity = "info" message = fmt.Sprintf("Host %s guest %d: the agent restarted the controller (%s) — restart #%d since the agent started", hostID, g.VMID, g.LastReason, g.RestartsTotal) case "controller_crashloop": severity = "error" message = fmt.Sprintf("Host %s guest %d: the controller will not stay up — the agent stopped restarting it after repeated attempts (since %s) and will try again in 30 minutes. Last reason: %s", hostID, g.VMID, g.CrashloopSince, g.LastReason) case "controller_slow_crashloop": severity = "warning" message = fmt.Sprintf("Host %s guest %d: the controller keeps dying — the agent restarted it %d times in 24 hours (each one too far apart for the 15-minute brake). It is still being restarted; this is the warning that it will not stay up. Last reason: %s", hostID, g.VMID, g.Restarts24h, g.LastReason) case "controller_unresponsive": severity = "warning" if g.UnresponsiveRestartAfterS > 0 { message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). The agent restarts it if it still does not answer %s after this alarm — unless a backup, a restore or an update is running, or the restart brake holds; then use „Restart controller\" on the host page.", hostID, g.VMID, g.UnresponsiveSince, humanSeconds(g.UnresponsiveRestartAfterS)) } else { message = fmt.Sprintf("Host %s guest %d: the controller runs but does not answer its health check (since %s). This agent does NOT restart it itself (an agent older than 0.171.0) — use „Restart controller\" on the host page if it stays so.", hostID, g.VMID, g.UnresponsiveSince) } case "controller_unresponsive_restart": severity = "warning" message = fmt.Sprintf("Host %s guest %d: the controller ran but did not answer its health check (since %s), so the agent restarted it at %s. Look at the box if this repeats. Reason: %s", hostID, g.VMID, g.UnresponsiveSince, g.UnresponsiveRestartAt, g.LastReason) default: return } details, _ := json.Marshal(map[string]any{ "host_id": hostID, "vmid": g.VMID, "restarts_total": g.RestartsTotal, "last_restart_at": g.LastRestartAt, "last_reason": g.LastReason, "crashloop_since": g.CrashloopSince, "parked": g.Parked, "restarts_24h": g.Restarts24h, "slow_crashloop_since": g.SlowCrashloopSince, "unresponsive_since": g.UnresponsiveSince, "unresponsive_restart_at": g.UnresponsiveRestartAt, }) c.logger.Printf("[INFO] Controller supervisor: %s %s/%d (%s)", eventType, hostID, g.VMID, g.LastReason) if _, err := c.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil { c.logger.Printf("[WARN] save %s for %s: %v", eventType, hostID, err) return } if c.onEvent != nil { c.onEvent(customerID, eventType, severity, message, string(details), "hub") } } // humanSeconds says a whole number of minutes as „2 minutes", anything else as „90 seconds". func humanSeconds(n int) string { switch { case n == 60: return "1 minute" case n%60 == 0: return fmt.Sprintf("%d minutes", n/60) } return fmt.Sprintf("%d seconds", n) }