package monitor import ( "encoding/json" "log" "sync" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // HostMgmtPlaneChecker raises an operator WARNING when a host's agent-independent break-glass watchdog // AUTO-HEALED a missing /run/sshd privsep dir (TASK G1). The heal itself is silent and login-free (the // point of the watchdog); this surfaces a RECURRING clobber so the operator can find the cause BEFORE // it becomes a full management lockout — complementing HostStalenessChecker (which only catches a box // gone silent). Sibling of HostLeafChecker; runs on the same 60s sweep. // // Design: the state is the host's last-seen privsep_healed_at marker timestamp. The watchdog rewrites // the marker on EACH heal, so a new, different timestamp = a new heal event → one warning. UNLIKE // HostLeafChecker (where the fp is a persistent STATE), a heal marker is an EVENT, so a newly-observed // marker DOES alert — the operator must learn of every auto-heal. To avoid a false alarm at hub // startup on a marker that predates it, construction SEEDS the last-seen timestamp from the newest // reports WITHOUT alerting; thereafter any change to a NEW non-empty timestamp emits exactly one // warning. An empty timestamp (healthy host / old agent) never alerts. type HostMgmtPlaneChecker struct { store *store.Store logger *log.Logger onEvent EventNotifyFunc mu sync.Mutex states map[string]string // hostID → last-seen privsep_healed_at customerOf map[string]string } // NewHostMgmtPlaneChecker seeds per-host baselines from the latest reports. No events on init. func NewHostMgmtPlaneChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostMgmtPlaneChecker { mc := &HostMgmtPlaneChecker{ store: s, logger: logger, onEvent: onEvent, states: make(map[string]string), customerOf: make(map[string]string), } rows, err := s.GetHostMgmtPlaneStates() if err != nil { logger.Printf("[WARN] Host mgmt-plane checker: failed to seed: %v", err) return mc } seeded := 0 for _, row := range rows { if s.IsCustomerBlocked(row.CustomerID) || row.PrivsepHealedAt == "" { continue } mc.customerOf[row.HostID] = row.CustomerID mc.states[row.HostID] = row.PrivsepHealedAt seeded++ } logger.Printf("[INFO] Host mgmt-plane checker initialized: %d host heal-state(s) seeded", seeded) return mc } // Check evaluates all hosts and emits mgmt_plane_healed on a NEW heal timestamp. func (mc *HostMgmtPlaneChecker) Check() { rows, err := mc.store.GetHostMgmtPlaneStates() if err != nil { mc.logger.Printf("[WARN] Host mgmt-plane check failed: %v", err) return } mc.mu.Lock() defer mc.mu.Unlock() seen := make(map[string]bool, len(rows)) for _, row := range rows { if mc.store.IsCustomerBlocked(row.CustomerID) { delete(mc.states, row.HostID) continue } seen[row.HostID] = true mc.customerOf[row.HostID] = row.CustomerID newHealed := row.PrivsepHealedAt if newHealed == mc.states[row.HostID] { continue // unchanged from last-seen (incl. both empty) → nothing to do } mc.states[row.HostID] = newHealed // advance the baseline (incl. back to "" if a reboot cleared it) if newHealed != "" { // A NEW heal timestamp the hub has not seen (construction seeded any that predate it) → alert. mc.emit(row.HostID, row.CustomerID, newHealed) } } for id := range mc.states { if !seen[id] { delete(mc.states, id) } } } // GetState returns the last-seen heal timestamp for a host ("" if none). func (mc *HostMgmtPlaneChecker) GetState(hostID string) string { mc.mu.Lock() defer mc.mu.Unlock() return mc.states[hostID] } func (mc *HostMgmtPlaneChecker) emit(hostID, customerID, healedAt string) { msg := "Host " + hostID + ": the management-plane privsep dir (/run/sshd) was missing and was AUTO-HEALED by the watchdog at " + healedAt + " — a recurring cause can lead to an SSH lockout; investigate (e.g. a unit declaring RuntimeDirectory=sshd)." details, _ := json.Marshal(map[string]string{ "host_id": hostID, "privsep_healed_at": healedAt, }) mc.logger.Printf("[WARN] Host mgmt-plane: %s privsep dir auto-healed at %s (mgmt_plane_healed)", hostID, healedAt) if _, err := mc.store.SaveEvent(customerID, "mgmt_plane_healed", "warning", msg, string(details), "hub"); err != nil { mc.logger.Printf("[WARN] save mgmt_plane_healed for %s: %v", hostID, err) return } if mc.onEvent != nil { mc.onEvent(customerID, "mgmt_plane_healed", "warning", msg, string(details), "hub") } }