package monitor import ( "encoding/json" "fmt" "log" "sync" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // R-523 (felhom-agent v0.131.0). The agent's in-guest controller supervisor restarts a controller // container that is not running and, after 3 restarts in 15 minutes, gives up for 30 minutes. The // agent has no event channel of its own — its heartbeat is the channel — so the per-guest record // rides the host report as `controller_supervisor`, and this checker turns movement in it into events: // // - `controller_restarted_by_agent` (info) when a guest's last_restart_at MOVES to a new value; // - `controller_crashloop` (error, operator-only) when a guest's crashloop_since MOVES. // // TIMESTAMPS, NOT COUNTERS. The agent's record is in-memory, so an agent restart zeroes // restarts_total; keying on a counter would read that as nothing (fine) but a later 1 would not // exceed the remembered 3 (a lost event). A timestamp that changes is a new act whatever the counter // says. // // First observation (hub restart, new host): SEED SILENTLY, except a guest that is IN a crash-loop at // first sight emits once — a hub restarted during an outage must not stay silent about it (the // HostCapabilityChecker F2 rule). Both types are pinned in allowedEventTypes and operatorOnlyEvents // by controller_supervisor_event_test.go in the api package. type ControllerSupervisorChecker struct { store *store.Store logger *log.Logger onEvent EventNotifyFunc mu sync.Mutex seen map[string]supSeen // hostID/vmid → last timestamps seen } type supSeen struct { lastRestartAt string crashloopSince string } // ControllerSupervisorGuest mirrors the agent's hub.ControllerSupervisorGuest wire shape. type ControllerSupervisorGuest struct { VMID int `json:"vmid"` RestartsTotal int `json:"restarts_total"` LastRestartAt string `json:"last_restart_at"` LastReason string `json:"last_reason"` Crashloop bool `json:"crashloop"` CrashloopSince string `json:"crashloop_since"` Parked bool `json:"parked"` } // ParseControllerSupervisor extracts the stanza from a host report body. A missing or malformed // stanza (a pre-v0.131.0 agent) yields nil — never an event. func ParseControllerSupervisor(reportJSON string) []ControllerSupervisorGuest { var body struct { CS *struct { Guests []ControllerSupervisorGuest `json:"guests"` } `json:"controller_supervisor"` } if err := json.Unmarshal([]byte(reportJSON), &body); err != nil || body.CS == nil { return nil } return body.CS.Guests } func NewControllerSupervisorChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *ControllerSupervisorChecker { return &ControllerSupervisorChecker{store: s, logger: logger, onEvent: onEvent, seen: map[string]supSeen{}} } // Check reads every host's latest report and emits on movement. Same 60 s sweep as its siblings. func (c *ControllerSupervisorChecker) Check() { rows, err := c.store.GetLatestHostReports() if err != nil { c.logger.Printf("[WARN] Controller supervisor check failed: %v", err) return } for _, row := range rows { if c.store.IsCustomerBlocked(row.CustomerID) { continue } c.observe(row.HostID, row.CustomerID, ParseControllerSupervisor(row.ReportJSON)) } } func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests []ControllerSupervisorGuest) { c.mu.Lock() defer c.mu.Unlock() for _, g := range guests { key := fmt.Sprintf("%s/%d", hostID, g.VMID) prev, known := c.seen[key] c.seen[key] = supSeen{lastRestartAt: g.LastRestartAt, crashloopSince: g.CrashloopSince} if !known { if g.Crashloop && g.CrashloopSince != "" { c.emit(customerID, hostID, g, "controller_crashloop") } continue } if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt { c.emit(customerID, hostID, g, "controller_restarted_by_agent") } if g.CrashloopSince != "" && g.CrashloopSince != prev.crashloopSince { c.emit(customerID, hostID, g, "controller_crashloop") } } } func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g ControllerSupervisorGuest, eventType string) { var severity, message string switch eventType { case "controller_restarted_by_agent": severity = "info" message = fmt.Sprintf("Host %s guest %d: the agent restarted the controller (%s) — restart #%d since the agent started", hostID, g.VMID, g.LastReason, g.RestartsTotal) case "controller_crashloop": severity = "error" message = fmt.Sprintf("Host %s guest %d: the controller will not stay up — the agent stopped restarting it after repeated attempts (since %s) and will try again in 30 minutes. Last reason: %s", hostID, g.VMID, g.CrashloopSince, g.LastReason) default: return } details, _ := json.Marshal(map[string]any{ "host_id": hostID, "vmid": g.VMID, "restarts_total": g.RestartsTotal, "last_restart_at": g.LastRestartAt, "last_reason": g.LastReason, "crashloop_since": g.CrashloopSince, "parked": g.Parked, }) c.logger.Printf("[INFO] Controller supervisor: %s %s/%d (%s)", eventType, hostID, g.VMID, g.LastReason) if _, err := c.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil { c.logger.Printf("[WARN] save %s for %s: %v", eventType, hostID, err) return } if c.onEvent != nil { c.onEvent(customerID, eventType, severity, message, string(details), "hub") } }