package monitor import ( "encoding/json" "log" "sync" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // HostOOBChecker raises an operator WARNING when a host's OOB access path is DEGRADED — felhom-sshd // down (while the operator peer is configured, i.e. OOB is meant to work) OR its config is invalid. // It answers "can the operator get into this box right now, and if not, why" proactively, from the // hub. Transition-based (ok↔degraded, one event per transition — the HostCapabilityChecker shape), so // a persistent problem alerts ONCE, not every 60s sweep, and a recovery is noted. // // A host with no oob stanza (pre-H1 / feature off) is never evaluated. A degraded state requires the // operator peer to be configured — a box where OOB was never set up is not "broken". type HostOOBChecker struct { store *store.Store logger *log.Logger onEvent EventNotifyFunc mu sync.Mutex degraded map[string]bool // hostID → currently-degraded customerOf map[string]string } // NewHostOOBChecker seeds per-host degraded state from the latest reports WITHOUT alerting (a problem // present at startup alerts on the first transition-in evaluated after seed = never re-alerts a // steady bad state; matches HostCapabilityChecker). Actually seeds silent, then Check transitions. func NewHostOOBChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostOOBChecker { c := &HostOOBChecker{ store: s, logger: logger, onEvent: onEvent, degraded: make(map[string]bool), customerOf: make(map[string]string), } rows, err := s.GetHostOOBStates() if err != nil { logger.Printf("[WARN] Host OOB checker: failed to seed: %v", err) return c } seeded := 0 for _, row := range rows { if s.IsCustomerBlocked(row.CustomerID) || !row.Present { continue } c.customerOf[row.HostID] = row.CustomerID if oobDegraded(row) { c.degraded[row.HostID] = true // seed the bad state so we don't re-alert it on cycle 1 seeded++ } } logger.Printf("[INFO] Host OOB checker initialized: %d host(s) seeded degraded", seeded) return c } // oobDegraded is the degraded predicate: config invalid, OR (OOB meant to work — operator peer // configured — AND felhom-sshd is not active/reachable OR the operator's key is not installed). // // THE KEY CLAUSE IS NEW (R-260, G-1) AND IT IS THE POINT. Until 2026-08-08 this predicate tested // five things and the sixth — whether the credential that actually grants entry exists — never // arrived, because the hub's decoder had no field for it. A box with the service active, reachable, // a valid config and a configured peer reported `ok` with NO OPERATOR KEY INSTALLED. That is not a // wrong answer; it is an answer to a question nobody was asking. // // It is deliberately gated on OperatorPeerConfigured, exactly like the reachability clause: a box // where OOB was never set up is not "broken", and widening this check beyond the fact that is now // arriving is how a check stops being read. func oobDegraded(r store.HostOOBRow) bool { return oobDegradedReason(r) != "" } // oobDegradedReason returns the SPECIFIC reason a host's operator access is degraded, or "" when it // is not. The reason is separated from the boolean because the operator reads the alert at 07:00 and // needs to know WHICH of the things this checks is wrong — "out-of-band access degraded" is true and // useless. func oobDegradedReason(r store.HostOOBRow) string { if !r.Present { return "" } if r.ConfigInvalid { return "felhom-sshd config invalid (sshd -t fails)" } if !r.OperatorPeerConfigured { return "" } if !r.FelhomSshdActive { return "felhom-sshd is not active" } if !r.Reachable { return "felhom-sshd unreachable (local dial to the OOB port fails)" } // The operator key, and the two ways it can be wrong. Both are reported distinctly, and NEITHER // is a silent ok — an absence read as "the key is installed" is precisely the defect this clause // was added to end, arriving through the version door instead. if !r.OperatorKeyReported { // Unreachable for any released agent: `operator_key_configured` and the `oob` stanza that // carries it shipped together in agent v0.72.0 (2026-07-05), so a stanza without the field // cannot come from a version anyone runs — the vouched floor is far above it. Handled // explicitly anyway, because "cannot happen" is the kind of claim this project has been // burned by, and pinned by TestOOBDegraded_StanzaWithoutKeyField_IsNotOK. return "the agent reports operator access but is too old to say whether the operator key is " + "installed (pre-v0.72.0) — treat entry as UNPROVEN, not working" } if !r.OperatorKeyConfigured { return "the operator's authorized_key is NOT installed — felhom-sshd is up and answering, " + "and nobody can log in through it" } return "" } // Check evaluates all hosts and emits oob_degraded / oob_recovered on transitions. func (c *HostOOBChecker) Check() { rows, err := c.store.GetHostOOBStates() if err != nil { c.logger.Printf("[WARN] Host OOB check failed: %v", err) return } c.mu.Lock() defer c.mu.Unlock() seen := make(map[string]bool, len(rows)) for _, row := range rows { if c.store.IsCustomerBlocked(row.CustomerID) { delete(c.degraded, row.HostID) continue } if !row.Present { continue // no oob stanza → not evaluated } seen[row.HostID] = true c.customerOf[row.HostID] = row.CustomerID bad := oobDegraded(row) was := c.degraded[row.HostID] switch { case bad && !was: c.degraded[row.HostID] = true c.emit(row, "oob_degraded", "warning") case !bad && was: delete(c.degraded, row.HostID) c.emit(row, "oob_recovered", "info") } } for id := range c.degraded { if !seen[id] { delete(c.degraded, id) } } } // IsDegraded reports the current tracked state for a host (test/UI helper). func (c *HostOOBChecker) IsDegraded(hostID string) bool { c.mu.Lock() defer c.mu.Unlock() return c.degraded[hostID] } func (c *HostOOBChecker) emit(row store.HostOOBRow, eventType, severity string) { var msg string if eventType == "oob_degraded" { // The reason comes from the predicate itself, so the message can never name a different // fault from the one that fired. The old form derived it separately and could only ever say // "unreachable" or "config invalid" — it had no vocabulary for a missing operator key, // which is the fault this checker most needs to be able to name (R-260). reason := oobDegradedReason(row) if reason == "" { reason = "operator access degraded" } msg = "Host " + row.HostID + ": OPERATOR ACCESS DEGRADED — " + reason + ". The break-glass net (auto-heal + vaulted root@pam console) is still under the box." } else { msg = "Host " + row.HostID + ": operator access recovered (felhom-sshd reachable again)." } det := map[string]any{ "host_id": row.HostID, "felhom_sshd_port": row.FelhomSshdPort, "active": row.FelhomSshdActive, "reachable": row.Reachable, "config_invalid": row.ConfigInvalid, // R-260: carried for context, NOT consulted by the predicate. "operator_key_configured": row.OperatorKeyConfigured, "operator_key_reported": row.OperatorKeyReported, } if row.WGHandshakeAgeS != nil { det["wg_handshake_age_s"] = *row.WGHandshakeAgeS } if row.HealedAt != "" { det["healed_at"] = row.HealedAt } details, _ := json.Marshal(det) c.logger.Printf("[%s] Host OOB: %s (%s)", map[string]string{"warning": "WARN", "info": "INFO"}[severity], row.HostID, eventType) if _, err := c.store.SaveEvent(row.CustomerID, eventType, severity, msg, string(details), "hub"); err != nil { c.logger.Printf("[WARN] save %s for %s: %v", eventType, row.HostID, err) return } if c.onEvent != nil { c.onEvent(row.CustomerID, eventType, severity, msg, string(details), "hub") } }