felhom.eu/hub/internal/monitor/controller_supervisor.go

package monitor

import (
	"encoding/json"
	"fmt"
	"log"
	"sync"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// R-523 (felhom-agent v0.131.0). The agent's in-guest controller supervisor restarts a controller
// container that is not running and, after 3 restarts in 15 minutes, gives up for 30 minutes. The
// agent has no event channel of its own — its heartbeat is the channel — so the per-guest record
// rides the host report as `controller_supervisor`, and this checker turns movement in it into events:
//
//   - `controller_restarted_by_agent` (info) when a guest's last_restart_at MOVES to a new value;
//   - `controller_crashloop` (error, operator-only) when a guest's crashloop_since MOVES.
//
// TIMESTAMPS, NOT COUNTERS. The agent's record is in-memory, so an agent restart zeroes
// restarts_total; keying on a counter would read that as nothing (fine) but a later 1 would not
// exceed the remembered 3 (a lost event). A timestamp that changes is a new act whatever the counter
// says.
//
// First observation (hub restart, new host): SEED SILENTLY, except a guest that is IN a crash-loop at
// first sight emits once — a hub restarted during an outage must not stay silent about it (the
// HostCapabilityChecker F2 rule). Both types are pinned in allowedEventTypes and operatorOnlyEvents
// by controller_supervisor_event_test.go in the api package.
type ControllerSupervisorChecker struct {
	store   *store.Store
	logger  *log.Logger
	onEvent EventNotifyFunc

	mu   sync.Mutex
	seen map[string]supSeen // hostID/vmid → last timestamps seen
}

type supSeen struct {
	lastRestartAt  string
	crashloopSince string
}

// ControllerSupervisorGuest mirrors the agent's hub.ControllerSupervisorGuest wire shape.
type ControllerSupervisorGuest struct {
	VMID           int    `json:"vmid"`
	RestartsTotal  int    `json:"restarts_total"`
	LastRestartAt  string `json:"last_restart_at"`
	LastReason     string `json:"last_reason"`
	Crashloop      bool   `json:"crashloop"`
	CrashloopSince string `json:"crashloop_since"`
	Parked         bool   `json:"parked"`
}

// ParseControllerSupervisor extracts the stanza from a host report body. A missing or malformed
// stanza (a pre-v0.131.0 agent) yields nil — never an event.
func ParseControllerSupervisor(reportJSON string) []ControllerSupervisorGuest {
	var body struct {
		CS *struct {
			Guests []ControllerSupervisorGuest `json:"guests"`
		} `json:"controller_supervisor"`
	}
	if err := json.Unmarshal([]byte(reportJSON), &body); err != nil || body.CS == nil {
		return nil
	}
	return body.CS.Guests
}

func NewControllerSupervisorChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *ControllerSupervisorChecker {
	return &ControllerSupervisorChecker{store: s, logger: logger, onEvent: onEvent, seen: map[string]supSeen{}}
}

// Check reads every host's latest report and emits on movement. Same 60 s sweep as its siblings.
func (c *ControllerSupervisorChecker) Check() {
	rows, err := c.store.GetLatestHostReports()
	if err != nil {
		c.logger.Printf("[WARN] Controller supervisor check failed: %v", err)
		return
	}
	for _, row := range rows {
		if c.store.IsCustomerBlocked(row.CustomerID) {
			continue
		}
		c.observe(row.HostID, row.CustomerID, ParseControllerSupervisor(row.ReportJSON))
	}
}

func (c *ControllerSupervisorChecker) observe(hostID, customerID string, guests []ControllerSupervisorGuest) {
	c.mu.Lock()
	defer c.mu.Unlock()
	for _, g := range guests {
		key := fmt.Sprintf("%s/%d", hostID, g.VMID)
		prev, known := c.seen[key]
		c.seen[key] = supSeen{lastRestartAt: g.LastRestartAt, crashloopSince: g.CrashloopSince}
		if !known {
			if g.Crashloop && g.CrashloopSince != "" {
				c.emit(customerID, hostID, g, "controller_crashloop")
			}
			continue
		}
		if g.LastRestartAt != "" && g.LastRestartAt != prev.lastRestartAt {
			c.emit(customerID, hostID, g, "controller_restarted_by_agent")
		}
		if g.CrashloopSince != "" && g.CrashloopSince != prev.crashloopSince {
			c.emit(customerID, hostID, g, "controller_crashloop")
		}
	}
}

func (c *ControllerSupervisorChecker) emit(customerID, hostID string, g ControllerSupervisorGuest, eventType string) {
	var severity, message string
	switch eventType {
	case "controller_restarted_by_agent":
		severity = "info"
		message = fmt.Sprintf("Host %s guest %d: the agent restarted the controller (%s) — restart #%d since the agent started",
			hostID, g.VMID, g.LastReason, g.RestartsTotal)
	case "controller_crashloop":
		severity = "error"
		message = fmt.Sprintf("Host %s guest %d: the controller will not stay up — the agent stopped restarting it after repeated attempts (since %s) and will try again in 30 minutes. Last reason: %s",
			hostID, g.VMID, g.CrashloopSince, g.LastReason)
	default:
		return
	}
	details, _ := json.Marshal(map[string]any{
		"host_id": hostID, "vmid": g.VMID, "restarts_total": g.RestartsTotal,
		"last_restart_at": g.LastRestartAt, "last_reason": g.LastReason,
		"crashloop_since": g.CrashloopSince, "parked": g.Parked,
	})
	c.logger.Printf("[INFO] Controller supervisor: %s %s/%d (%s)", eventType, hostID, g.VMID, g.LastReason)
	if _, err := c.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil {
		c.logger.Printf("[WARN] save %s for %s: %v", eventType, hostID, err)
		return
	}
	if c.onEvent != nil {
		c.onEvent(customerID, eventType, severity, message, string(details), "hub")
	}
}