felhom.eu/hub/internal/monitor/host_oob.go

package monitor

import (
	"encoding/json"
	"log"
	"sync"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// HostOOBChecker raises an operator WARNING when a host's OOB access path is DEGRADED — felhom-sshd
// down (while the operator peer is configured, i.e. OOB is meant to work) OR its config is invalid.
// It answers "can the operator get into this box right now, and if not, why" proactively, from the
// hub. Transition-based (ok↔degraded, one event per transition — the HostCapabilityChecker shape), so
// a persistent problem alerts ONCE, not every 60s sweep, and a recovery is noted.
//
// A host with no oob stanza (pre-H1 / feature off) is never evaluated. A degraded state requires the
// operator peer to be configured — a box where OOB was never set up is not "broken".
type HostOOBChecker struct {
	store   *store.Store
	logger  *log.Logger
	onEvent EventNotifyFunc

	mu         sync.Mutex
	degraded   map[string]bool // hostID → currently-degraded
	customerOf map[string]string
}

// NewHostOOBChecker seeds per-host degraded state from the latest reports WITHOUT alerting (a problem
// present at startup alerts on the first transition-in evaluated after seed = never re-alerts a
// steady bad state; matches HostCapabilityChecker). Actually seeds silent, then Check transitions.
func NewHostOOBChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostOOBChecker {
	c := &HostOOBChecker{
		store:      s,
		logger:     logger,
		onEvent:    onEvent,
		degraded:   make(map[string]bool),
		customerOf: make(map[string]string),
	}
	rows, err := s.GetHostOOBStates()
	if err != nil {
		logger.Printf("[WARN] Host OOB checker: failed to seed: %v", err)
		return c
	}
	seeded := 0
	for _, row := range rows {
		if s.IsCustomerBlocked(row.CustomerID) || !row.Present {
			continue
		}
		c.customerOf[row.HostID] = row.CustomerID
		if oobDegraded(row) {
			c.degraded[row.HostID] = true // seed the bad state so we don't re-alert it on cycle 1
			seeded++
		}
	}
	logger.Printf("[INFO] Host OOB checker initialized: %d host(s) seeded degraded", seeded)
	return c
}

// oobDegraded is the degraded predicate: config invalid, OR (OOB meant to work — operator peer
// configured — AND felhom-sshd is not active/reachable OR the operator's key is not installed).
//
// THE KEY CLAUSE IS NEW (R-260, G-1) AND IT IS THE POINT. Until 2026-08-08 this predicate tested
// five things and the sixth — whether the credential that actually grants entry exists — never
// arrived, because the hub's decoder had no field for it. A box with the service active, reachable,
// a valid config and a configured peer reported `ok` with NO OPERATOR KEY INSTALLED. That is not a
// wrong answer; it is an answer to a question nobody was asking.
//
// It is deliberately gated on OperatorPeerConfigured, exactly like the reachability clause: a box
// where OOB was never set up is not "broken", and widening this check beyond the fact that is now
// arriving is how a check stops being read.
func oobDegraded(r store.HostOOBRow) bool {
	return oobDegradedReason(r) != ""
}

// oobDegradedReason returns the SPECIFIC reason a host's operator access is degraded, or "" when it
// is not. The reason is separated from the boolean because the operator reads the alert at 07:00 and
// needs to know WHICH of the things this checks is wrong — "out-of-band access degraded" is true and
// useless.
func oobDegradedReason(r store.HostOOBRow) string {
	if !r.Present {
		return ""
	}
	if r.ConfigInvalid {
		return "felhom-sshd config invalid (sshd -t fails)"
	}
	if !r.OperatorPeerConfigured {
		return ""
	}
	if !r.FelhomSshdActive {
		return "felhom-sshd is not active"
	}
	if !r.Reachable {
		return "felhom-sshd unreachable (local dial to the OOB port fails)"
	}
	// The operator key, and the two ways it can be wrong. Both are reported distinctly, and NEITHER
	// is a silent ok — an absence read as "the key is installed" is precisely the defect this clause
	// was added to end, arriving through the version door instead.
	if !r.OperatorKeyReported {
		// Unreachable for any released agent: `operator_key_configured` and the `oob` stanza that
		// carries it shipped together in agent v0.72.0 (2026-07-05), so a stanza without the field
		// cannot come from a version anyone runs — the vouched floor is far above it. Handled
		// explicitly anyway, because "cannot happen" is the kind of claim this project has been
		// burned by, and pinned by TestOOBDegraded_StanzaWithoutKeyField_IsNotOK.
		return "the agent reports operator access but is too old to say whether the operator key is " +
			"installed (pre-v0.72.0) — treat entry as UNPROVEN, not working"
	}
	if !r.OperatorKeyConfigured {
		return "the operator's authorized_key is NOT installed — felhom-sshd is up and answering, " +
			"and nobody can log in through it"
	}
	return ""
}

// Check evaluates all hosts and emits oob_degraded / oob_recovered on transitions.
func (c *HostOOBChecker) Check() {
	rows, err := c.store.GetHostOOBStates()
	if err != nil {
		c.logger.Printf("[WARN] Host OOB check failed: %v", err)
		return
	}
	c.mu.Lock()
	defer c.mu.Unlock()

	seen := make(map[string]bool, len(rows))
	for _, row := range rows {
		if c.store.IsCustomerBlocked(row.CustomerID) {
			delete(c.degraded, row.HostID)
			continue
		}
		if !row.Present {
			continue // no oob stanza → not evaluated
		}
		seen[row.HostID] = true
		c.customerOf[row.HostID] = row.CustomerID
		bad := oobDegraded(row)
		was := c.degraded[row.HostID]
		switch {
		case bad && !was:
			c.degraded[row.HostID] = true
			c.emit(row, "oob_degraded", "warning")
		case !bad && was:
			delete(c.degraded, row.HostID)
			c.emit(row, "oob_recovered", "info")
		}
	}
	for id := range c.degraded {
		if !seen[id] {
			delete(c.degraded, id)
		}
	}
}

// IsDegraded reports the current tracked state for a host (test/UI helper).
func (c *HostOOBChecker) IsDegraded(hostID string) bool {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.degraded[hostID]
}

func (c *HostOOBChecker) emit(row store.HostOOBRow, eventType, severity string) {
	var msg string
	if eventType == "oob_degraded" {
		// The reason comes from the predicate itself, so the message can never name a different
		// fault from the one that fired. The old form derived it separately and could only ever say
		// "unreachable" or "config invalid" — it had no vocabulary for a missing operator key,
		// which is the fault this checker most needs to be able to name (R-260).
		reason := oobDegradedReason(row)
		if reason == "" {
			reason = "operator access degraded"
		}
		msg = "Host " + row.HostID + ": OPERATOR ACCESS DEGRADED — " + reason +
			". The break-glass net (auto-heal + vaulted root@pam console) is still under the box."
	} else {
		msg = "Host " + row.HostID + ": operator access recovered (felhom-sshd reachable again)."
	}
	det := map[string]any{
		"host_id":          row.HostID,
		"felhom_sshd_port": row.FelhomSshdPort,
		"active":           row.FelhomSshdActive,
		"reachable":        row.Reachable,
		"config_invalid":   row.ConfigInvalid,
		// R-260: carried for context, NOT consulted by the predicate.
		"operator_key_configured": row.OperatorKeyConfigured,
		"operator_key_reported":   row.OperatorKeyReported,
	}
	if row.WGHandshakeAgeS != nil {
		det["wg_handshake_age_s"] = *row.WGHandshakeAgeS
	}
	if row.HealedAt != "" {
		det["healed_at"] = row.HealedAt
	}
	details, _ := json.Marshal(det)
	c.logger.Printf("[%s] Host OOB: %s (%s)", map[string]string{"warning": "WARN", "info": "INFO"}[severity], row.HostID, eventType)
	if _, err := c.store.SaveEvent(row.CustomerID, eventType, severity, msg, string(details), "hub"); err != nil {
		c.logger.Printf("[WARN] save %s for %s: %v", eventType, row.HostID, err)
		return
	}
	if c.onEvent != nil {
		c.onEvent(row.CustomerID, eventType, severity, msg, string(details), "hub")
	}
}