felhom.eu/hub/internal/monitor/host_mgmtplane.go

package monitor

import (
	"encoding/json"
	"log"
	"sync"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// HostMgmtPlaneChecker raises an operator WARNING when a host's agent-independent break-glass watchdog
// AUTO-HEALED a missing /run/sshd privsep dir (TASK G1). The heal itself is silent and login-free (the
// point of the watchdog); this surfaces a RECURRING clobber so the operator can find the cause BEFORE
// it becomes a full management lockout — complementing HostStalenessChecker (which only catches a box
// gone silent). Sibling of HostLeafChecker; runs on the same 60s sweep.
//
// Design: the state is the host's last-seen privsep_healed_at marker timestamp. The watchdog rewrites
// the marker on EACH heal, so a new, different timestamp = a new heal event → one warning. UNLIKE
// HostLeafChecker (where the fp is a persistent STATE), a heal marker is an EVENT, so a newly-observed
// marker DOES alert — the operator must learn of every auto-heal. To avoid a false alarm at hub
// startup on a marker that predates it, construction SEEDS the last-seen timestamp from the newest
// reports WITHOUT alerting; thereafter any change to a NEW non-empty timestamp emits exactly one
// warning. An empty timestamp (healthy host / old agent) never alerts.
type HostMgmtPlaneChecker struct {
	store   *store.Store
	logger  *log.Logger
	onEvent EventNotifyFunc

	mu         sync.Mutex
	states     map[string]string // hostID → last-seen privsep_healed_at
	customerOf map[string]string
}

// NewHostMgmtPlaneChecker seeds per-host baselines from the latest reports. No events on init.
func NewHostMgmtPlaneChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostMgmtPlaneChecker {
	mc := &HostMgmtPlaneChecker{
		store:      s,
		logger:     logger,
		onEvent:    onEvent,
		states:     make(map[string]string),
		customerOf: make(map[string]string),
	}
	rows, err := s.GetHostMgmtPlaneStates()
	if err != nil {
		logger.Printf("[WARN] Host mgmt-plane checker: failed to seed: %v", err)
		return mc
	}
	seeded := 0
	for _, row := range rows {
		if s.IsCustomerBlocked(row.CustomerID) || row.PrivsepHealedAt == "" {
			continue
		}
		mc.customerOf[row.HostID] = row.CustomerID
		mc.states[row.HostID] = row.PrivsepHealedAt
		seeded++
	}
	logger.Printf("[INFO] Host mgmt-plane checker initialized: %d host heal-state(s) seeded", seeded)
	return mc
}

// Check evaluates all hosts and emits mgmt_plane_healed on a NEW heal timestamp.
func (mc *HostMgmtPlaneChecker) Check() {
	rows, err := mc.store.GetHostMgmtPlaneStates()
	if err != nil {
		mc.logger.Printf("[WARN] Host mgmt-plane check failed: %v", err)
		return
	}
	mc.mu.Lock()
	defer mc.mu.Unlock()

	seen := make(map[string]bool, len(rows))
	for _, row := range rows {
		if mc.store.IsCustomerBlocked(row.CustomerID) {
			delete(mc.states, row.HostID)
			continue
		}
		seen[row.HostID] = true
		mc.customerOf[row.HostID] = row.CustomerID
		newHealed := row.PrivsepHealedAt
		if newHealed == mc.states[row.HostID] {
			continue // unchanged from last-seen (incl. both empty) → nothing to do
		}
		mc.states[row.HostID] = newHealed // advance the baseline (incl. back to "" if a reboot cleared it)
		if newHealed != "" {
			// A NEW heal timestamp the hub has not seen (construction seeded any that predate it) → alert.
			mc.emit(row.HostID, row.CustomerID, newHealed)
		}
	}

	for id := range mc.states {
		if !seen[id] {
			delete(mc.states, id)
		}
	}
}

// GetState returns the last-seen heal timestamp for a host ("" if none).
func (mc *HostMgmtPlaneChecker) GetState(hostID string) string {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	return mc.states[hostID]
}

func (mc *HostMgmtPlaneChecker) emit(hostID, customerID, healedAt string) {
	msg := "Host " + hostID + ": the management-plane privsep dir (/run/sshd) was missing and was AUTO-HEALED by the watchdog at " + healedAt +
		" — a recurring cause can lead to an SSH lockout; investigate (e.g. a unit declaring RuntimeDirectory=sshd)."
	details, _ := json.Marshal(map[string]string{
		"host_id":           hostID,
		"privsep_healed_at": healedAt,
	})
	mc.logger.Printf("[WARN] Host mgmt-plane: %s privsep dir auto-healed at %s (mgmt_plane_healed)", hostID, healedAt)
	if _, err := mc.store.SaveEvent(customerID, "mgmt_plane_healed", "warning", msg, string(details), "hub"); err != nil {
		mc.logger.Printf("[WARN] save mgmt_plane_healed for %s: %v", hostID, err)
		return
	}
	if mc.onEvent != nil {
		mc.onEvent(customerID, "mgmt_plane_healed", "warning", msg, string(details), "hub")
	}
}