hub v0.114.0: self-bind auto-send while a customer waits for a box (R-509); node_* bypass the quiet hour (ruling 2026-09-15); PBS re-issue adopts an endpoint token (R-511); controller supervisor events (R-523); event registers

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-09-15 10:05:41 +02:00
parent a028a9a7f5
commit 07959e61b5
16 changed files with 693 additions and 9 deletions
@@ -0,0 +1,136 @@
+package monitor
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"sync"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// R-523 (felhom-agent v0.131.0). The agent's in-guest controller supervisor restarts a controller
+// container that is not running and, after 3 restarts in 15 minutes, gives up for 30 minutes. The
+// agent has no event channel of its own — its heartbeat is the channel — so the per-guest record
+// rides the host report as `controller_supervisor`, and this checker turns movement in it into events:
+//
+//   - `controller_restarted_by_agent` (info) when a guest's last_restart_at MOVES to a new value;
+//   - `controller_crashloop` (error, operator-only) when a guest's crashloop_since MOVES.
+//
+// TIMESTAMPS, NOT COUNTERS. The agent's record is in-memory, so an agent restart zeroes
+// restarts_total; keying on a counter would read that as nothing (fine) but a later 1 would not
+// exceed the remembered 3 (a lost event). A timestamp that changes is a new act whatever the counter
+// says.
+//
+// First observation (hub restart, new host): SEED SILENTLY, except a guest that is IN a crash-loop at
+// first sight emits once — a hub restarted during an outage must not stay silent about it (the
+// HostCapabilityChecker F2 rule). Both types are pinned in allowedEventTypes and operatorOnlyEvents
+// by controller_supervisor_event_test.go in the api package.
+type ControllerSupervisorChecker struct {
+	store   *store.Store
+	logger  *log.Logger
+	onEvent EventNotifyFunc
+
+	mu   sync.Mutex
+	seen map[string]supSeen // hostID/vmid → last timestamps seen
+}
+
+type supSeen struct {
+	lastRestartAt  string
+	crashloopSince string
+}
+
+// ControllerSupervisorGuest mirrors the agent's hub.ControllerSupervisorGuest wire shape.
+type ControllerSupervisorGuest struct {
+	VMID           int    `json:"vmid"`
+	RestartsTotal  int    `json:"restarts_total"`
+	LastRestartAt  string `json:"last_restart_at"`
+	LastReason     string `json:"last_reason"`
+	Crashloop      bool   `json:"crashloop"`
+	CrashloopSince string `json:"crashloop_since"`
+	Parked         bool   `json:"parked"`
+}
+
+// ParseControllerSupervisor extracts the stanza from a host report body. A missing or malformed
+// stanza (a pre-v0.131.0 agent) yields nil — never an event.
+func ParseControllerSupervisor(reportJSON string) []ControllerSupervisorGuest {
+	var body struct {
+		CS *struct {
+			Guests []ControllerSupervisorGuest `json:"guests"`
+		} `json:"controller_supervisor"`
+	}
+	if err := json.Unmarshal([]byte(reportJSON), &body); err != nil || body.CS == nil {
+		return nil
+	}
+	return body.CS.Guests
+}
+
+func NewControllerSupervisorChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *ControllerSupervisorChecker {
+	return &ControllerSupervisorChecker{store: s, logger: logger, onEvent: onEvent, seen: map[string]supSeen{}}
+}
+
+// Check reads every host's latest report and emits on movement. Same 60 s sweep as its siblings.
+func (c *ControllerSupervisorChecker) Check() {
+	rows, err := c.store.GetLatestHostReports()
+	if err != nil {
+		c.logger.Printf("[WARN] Controller supervisor check failed: %v", err)
+		return
+	}
+	for _, row := range rows {
+		if c.store.IsCustomerBlocked(row.CustomerID) {
+			continue
+		}
+		c.observe(row.HostID, row.CustomerID, ParseControllerSupervisor(row.ReportJSON))
+	}
+}
+
+func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests []ControllerSupervisorGuest) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for _, g := range guests {
+		key := fmt.Sprintf("%s/%d", hostID, g.VMID)
+		prev, known := c.seen[key]
+		c.seen[key] = supSeen{lastRestartAt: g.LastRestartAt, crashloopSince: g.CrashloopSince}
+		if !known {
+			if g.Crashloop && g.CrashloopSince != "" {
+				c.emit(customerID, hostID, g, "controller_crashloop")
+			}
+			continue
+		}
+		if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt {
+			c.emit(customerID, hostID, g, "controller_restarted_by_agent")
+		}
+		if g.CrashloopSince != "" && g.CrashloopSince != prev.crashloopSince {
+			c.emit(customerID, hostID, g, "controller_crashloop")
+		}
+	}
+}
+
+func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g ControllerSupervisorGuest, eventType string) {
+	var severity, message string
+	switch eventType {
+	case "controller_restarted_by_agent":
+		severity = "info"
+		message = fmt.Sprintf("Host %s guest %d: the agent restarted the controller (%s) — restart #%d since the agent started",
+			hostID, g.VMID, g.LastReason, g.RestartsTotal)
+	case "controller_crashloop":
+		severity = "error"
+		message = fmt.Sprintf("Host %s guest %d: the controller will not stay up — the agent stopped restarting it after repeated attempts (since %s) and will try again in 30 minutes. Last reason: %s",
+			hostID, g.VMID, g.CrashloopSince, g.LastReason)
+	default:
+		return
+	}
+	details, _ := json.Marshal(map[string]any{
+		"host_id": hostID, "vmid": g.VMID, "restarts_total": g.RestartsTotal,
+		"last_restart_at": g.LastRestartAt, "last_reason": g.LastReason,
+		"crashloop_since": g.CrashloopSince, "parked": g.Parked,
+	})
+	c.logger.Printf("[INFO] Controller supervisor: %s %s/%d (%s)", eventType, hostID, g.VMID, g.LastReason)
+	if _, err := c.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil {
+		c.logger.Printf("[WARN] save %s for %s: %v", eventType, hostID, err)
+		return
+	}
+	if c.onEvent != nil {
+		c.onEvent(customerID, eventType, severity, message, string(details), "hub")
+	}
+}