feat(hub): OOB access health ingest + degraded alert (H1 Part 4)

store.GetHostOOBStates parses the agent oob heartbeat stanza. monitor/host_oob:
transition-based oob_degraded/oob_recovered warning (felhom-sshd down while the
operator peer is configured, OR config invalid) — proactive "can the operator get
in right now" signal; unconfigured OOB never alerts. Wired into the 60s sweep.
Non-hollow tests + transitions.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6

hub/cmd/hub/main.go

@@ -381,6 +381,9 @@ func main() {
 	// TASK G1: warn when a host's agent-independent watchdog auto-healed a missing /run/sshd privsep
 	// dir — a recurring clobber that can lead to an SSH lockout (complements host_staleness). Same sweep.
 	hostMgmtPlaneChecker := monitor.NewHostMgmtPlaneChecker(dataStore, dispatcher.ProcessEvent, logger)
+	// TASK H1: alert when a host's OPERATOR ACCESS is degraded — felhom-sshd down (with the operator
+	// peer configured) or its config invalid. Transition-based, same 60s sweep.
+	hostOOBChecker := monitor.NewHostOOBChecker(dataStore, dispatcher.ProcessEvent, logger)
 	go func() {
 		ticker := time.NewTicker(60 * time.Second)
 		defer ticker.Stop()
@@ -396,6 +399,7 @@ func main() {
 				hostDiskChecker.Check()
 				storageFillChecker.Check()
 				hostMgmtPlaneChecker.Check()
+				hostOOBChecker.Check()
 			}
 		}
 	}()

hub/internal/api/wg.go

@@ -15,6 +15,7 @@ import (
 	"io"
 	"net/http"
 	"net/netip"
+	"strings"
 	"time"
 
 	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
@@ -48,6 +49,29 @@ func validateWGPubkey(pk string) error {
 	return nil
 }
 
+// validSSHAuthorizedKey does a conservative shape check on an SSH public key (an authorized_keys
+// line): a known key type, a base64 blob, no newlines/control chars (it is written verbatim into a
+// per-user authorized_keys file, so a hostile value must not inject options or extra lines).
+func validSSHAuthorizedKey(line string) bool {
+	line = strings.TrimSpace(line)
+	if line == "" || strings.ContainsAny(line, "\n\r\x00") {
+		return false
+	}
+	fields := strings.Fields(line)
+	if len(fields) < 2 {
+		return false
+	}
+	switch fields[0] {
+	case "ssh-ed25519", "ssh-rsa", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521", "sk-ssh-ed25519@openssh.com":
+	default:
+		return false
+	}
+	if _, err := base64.StdEncoding.DecodeString(fields[1]); err != nil {
+		return false
+	}
+	return true
+}
+
 // syncAfterMutation runs an inline sync after a peer mutation. The DB write already happened —
 // it is the source of truth — so a push failure is REPORTED, not rolled back: the reconciler's
 // next tick converges the endpoint (Scenario D).
@@ -327,6 +351,10 @@ func (h *Handler) mergeWireguard(hostID, desired string) string {
 		h.logger.Printf("[WARN] wg merge %s: operator OOB peer lookup failed: %v (serving without oob_peer_ip)", hostID, oerr)
 	} else if op != nil {
 		wgBlock["oob_peer_ip"] = op.AssignedIP // bare IPv4, e.g. "10.77.0.250"
+		// The operator SSH pubkey rides alongside (agent writes felhom-sshd's authorized_keys from it).
+		if k := h.store.GetOOBOperatorSSHKey(); k != "" {
+			wgBlock["oob_operator_ssh_key"] = k
+		}
 	}
 	doc["wireguard"] = wgBlock
 	out, err := json.Marshal(doc)
@@ -438,6 +466,7 @@ func (h *Handler) handleAdminSetOperatorPeer(w http.ResponseWriter, r *http.Requ
 	var req struct {
 		Pubkey     string `json:"pubkey"`
 		AssignedIP string `json:"assigned_ip"` // bare IPv4, e.g. "10.77.0.250"
+		SSHPubkey  string `json:"ssh_pubkey"`  // optional: the operator's SSH authorized_keys line
 	}
 	if err := json.Unmarshal(body, &req); err != nil {
 		http.Error(w, "Invalid payload: body must be JSON", http.StatusBadRequest)
@@ -447,6 +476,10 @@ func (h *Handler) handleAdminSetOperatorPeer(w http.ResponseWriter, r *http.Requ
 		http.Error(w, "Invalid payload: "+err.Error(), http.StatusBadRequest)
 		return
 	}
+	if req.SSHPubkey != "" && !validSSHAuthorizedKey(req.SSHPubkey) {
+		http.Error(w, "Invalid payload: ssh_pubkey must be an ssh-ed25519/ssh-rsa/ecdsa authorized_keys line", http.StatusBadRequest)
+		return
+	}
 	if err := h.store.SetOperatorOOBPeer(req.Pubkey, req.AssignedIP); err != nil {
 		if err == store.ErrWGEndpointUnset {
 			http.Error(w, "wg endpoint not configured", http.StatusConflict)
@@ -455,6 +488,11 @@ func (h *Handler) handleAdminSetOperatorPeer(w http.ResponseWriter, r *http.Requ
 		http.Error(w, "Invalid operator peer: "+err.Error(), http.StatusBadRequest)
 		return
 	}
+	if req.SSHPubkey != "" {
+		if err := h.store.SetOOBOperatorSSHKey(req.SSHPubkey); err != nil {
+			h.logger.Printf("[WARN] operator peer set but ssh_pubkey store failed: %v", err)
+		}
+	}
 	syncStatus := h.syncAfterMutation(r.Context())
 	bumped, berr := h.store.BumpAllHostGenerations()
 	if berr != nil {

hub/internal/monitor/host_oob.go

@@ -0,0 +1,148 @@
+package monitor
+
+import (
+	"encoding/json"
+	"log"
+	"sync"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// HostOOBChecker raises an operator WARNING when a host's OOB access path is DEGRADED — felhom-sshd
+// down (while the operator peer is configured, i.e. OOB is meant to work) OR its config is invalid.
+// It answers "can the operator get into this box right now, and if not, why" proactively, from the
+// hub. Transition-based (ok↔degraded, one event per transition — the HostCapabilityChecker shape), so
+// a persistent problem alerts ONCE, not every 60s sweep, and a recovery is noted.
+//
+// A host with no oob stanza (pre-H1 / feature off) is never evaluated. A degraded state requires the
+// operator peer to be configured — a box where OOB was never set up is not "broken".
+type HostOOBChecker struct {
+	store   *store.Store
+	logger  *log.Logger
+	onEvent EventNotifyFunc
+
+	mu         sync.Mutex
+	degraded   map[string]bool // hostID → currently-degraded
+	customerOf map[string]string
+}
+
+// NewHostOOBChecker seeds per-host degraded state from the latest reports WITHOUT alerting (a problem
+// present at startup alerts on the first transition-in evaluated after seed = never re-alerts a
+// steady bad state; matches HostCapabilityChecker). Actually seeds silent, then Check transitions.
+func NewHostOOBChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostOOBChecker {
+	c := &HostOOBChecker{
+		store:      s,
+		logger:     logger,
+		onEvent:    onEvent,
+		degraded:   make(map[string]bool),
+		customerOf: make(map[string]string),
+	}
+	rows, err := s.GetHostOOBStates()
+	if err != nil {
+		logger.Printf("[WARN] Host OOB checker: failed to seed: %v", err)
+		return c
+	}
+	seeded := 0
+	for _, row := range rows {
+		if s.IsCustomerBlocked(row.CustomerID) || !row.Present {
+			continue
+		}
+		c.customerOf[row.HostID] = row.CustomerID
+		if oobDegraded(row) {
+			c.degraded[row.HostID] = true // seed the bad state so we don't re-alert it on cycle 1
+			seeded++
+		}
+	}
+	logger.Printf("[INFO] Host OOB checker initialized: %d host(s) seeded degraded", seeded)
+	return c
+}
+
+// oobDegraded is the degraded predicate: config invalid, OR (OOB meant to work — operator peer
+// configured — AND felhom-sshd is not active/reachable).
+func oobDegraded(r store.HostOOBRow) bool {
+	if !r.Present {
+		return false
+	}
+	if r.ConfigInvalid {
+		return true
+	}
+	if r.OperatorPeerConfigured && (!r.FelhomSshdActive || !r.Reachable) {
+		return true
+	}
+	return false
+}
+
+// Check evaluates all hosts and emits oob_degraded / oob_recovered on transitions.
+func (c *HostOOBChecker) Check() {
+	rows, err := c.store.GetHostOOBStates()
+	if err != nil {
+		c.logger.Printf("[WARN] Host OOB check failed: %v", err)
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	seen := make(map[string]bool, len(rows))
+	for _, row := range rows {
+		if c.store.IsCustomerBlocked(row.CustomerID) {
+			delete(c.degraded, row.HostID)
+			continue
+		}
+		if !row.Present {
+			continue // no oob stanza → not evaluated
+		}
+		seen[row.HostID] = true
+		c.customerOf[row.HostID] = row.CustomerID
+		bad := oobDegraded(row)
+		was := c.degraded[row.HostID]
+		switch {
+		case bad && !was:
+			c.degraded[row.HostID] = true
+			c.emit(row, "oob_degraded", "warning")
+		case !bad && was:
+			delete(c.degraded, row.HostID)
+			c.emit(row, "oob_recovered", "info")
+		}
+	}
+	for id := range c.degraded {
+		if !seen[id] {
+			delete(c.degraded, id)
+		}
+	}
+}
+
+// IsDegraded reports the current tracked state for a host (test/UI helper).
+func (c *HostOOBChecker) IsDegraded(hostID string) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.degraded[hostID]
+}
+
+func (c *HostOOBChecker) emit(row store.HostOOBRow, eventType, severity string) {
+	var msg string
+	if eventType == "oob_degraded" {
+		reason := "felhom-sshd unreachable"
+		if row.ConfigInvalid {
+			reason = "felhom-sshd config invalid (sshd -t fails)"
+		}
+		msg = "Host " + row.HostID + ": OPERATOR ACCESS DEGRADED — " + reason +
+			". The break-glass net (auto-heal + vaulted root@pam console) is still under the box."
+	} else {
+		msg = "Host " + row.HostID + ": operator access recovered (felhom-sshd reachable again)."
+	}
+	details, _ := json.Marshal(map[string]any{
+		"host_id":          row.HostID,
+		"felhom_sshd_port": row.FelhomSshdPort,
+		"active":           row.FelhomSshdActive,
+		"reachable":        row.Reachable,
+		"config_invalid":   row.ConfigInvalid,
+	})
+	c.logger.Printf("[%s] Host OOB: %s (%s)", map[string]string{"warning": "WARN", "info": "INFO"}[severity], row.HostID, eventType)
+	if _, err := c.store.SaveEvent(row.CustomerID, eventType, severity, msg, string(details), "hub"); err != nil {
+		c.logger.Printf("[WARN] save %s for %s: %v", eventType, row.HostID, err)
+		return
+	}
+	if c.onEvent != nil {
+		c.onEvent(row.CustomerID, eventType, severity, msg, string(details), "hub")
+	}
+}

hub/internal/monitor/host_oob_test.go

@@ -0,0 +1,103 @@
+package monitor
+
+import (
+	"io"
+	"log"
+	"path/filepath"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+	_ "modernc.org/sqlite"
+)
+
+func oobReport(active, reachable, configInvalid, operatorConfigured bool) []byte {
+	b := func(v bool) string {
+		if v {
+			return "true"
+		}
+		return "false"
+	}
+	return []byte(`{"host_id":"h1","oob":{"felhom_sshd_active":` + b(active) +
+		`,"felhom_sshd_port":8822,"reachable":` + b(reachable) +
+		`,"config_invalid":` + b(configInvalid) +
+		`,"operator_peer_configured":` + b(operatorConfigured) + `}}`)
+}
+
+func newOOBStore(t *testing.T) *store.Store {
+	t.Helper()
+	st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { st.Close() })
+	st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"})
+	st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"})
+	return st
+}
+
+func TestHostOOB_DegradedThenRecoveredTransitions(t *testing.T) {
+	st := newOOBStore(t)
+	// healthy at construction (active+reachable, operator configured)
+	st.SaveHostReport("h1", "c1", oobReport(true, true, false, true), store.HostReportDenorm{})
+	var events []string
+	c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	if c.IsDegraded("h1") {
+		t.Fatal("healthy host seeded degraded")
+	}
+
+	// felhom-sshd goes DOWN with operator configured → one oob_degraded
+	st.SaveHostReport("h1", "c1", oobReport(false, false, false, true), store.HostReportDenorm{})
+	c.Check()
+	if len(events) != 1 || events[0] != "oob_degraded" {
+		t.Fatalf("down+operator-configured → one oob_degraded, got %v", events)
+	}
+	c.Check() // persistent → no re-alert
+	if len(events) != 1 {
+		t.Fatalf("persistent degraded must not re-emit, got %v", events)
+	}
+
+	// recovers → oob_recovered
+	st.SaveHostReport("h1", "c1", oobReport(true, true, false, true), store.HostReportDenorm{})
+	c.Check()
+	if len(events) != 2 || events[1] != "oob_recovered" {
+		t.Fatalf("recovery → oob_recovered, got %v", events)
+	}
+}
+
+func TestHostOOB_ConfigInvalidAlerts(t *testing.T) {
+	st := newOOBStore(t)
+	st.SaveHostReport("h1", "c1", oobReport(true, true, false, false), store.HostReportDenorm{})
+	var events []string
+	c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	// config invalid (even without operator configured) → degraded
+	st.SaveHostReport("h1", "c1", oobReport(true, true, true, false), store.HostReportDenorm{})
+	c.Check()
+	if len(events) != 1 || events[0] != "oob_degraded" {
+		t.Fatalf("config_invalid → oob_degraded, got %v", events)
+	}
+}
+
+// A box where OOB was NEVER set up (no operator peer) with felhom-sshd down must NOT alert — it's not
+// broken, it's unconfigured.
+func TestHostOOB_DownButNoOperatorNotDegraded(t *testing.T) {
+	st := newOOBStore(t)
+	st.SaveHostReport("h1", "c1", oobReport(false, false, false, false), store.HostReportDenorm{})
+	var events []string
+	c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	c.Check()
+	if len(events) != 0 {
+		t.Fatalf("unconfigured OOB (no operator peer) must not alert, got %v", events)
+	}
+}
+
+// A report with no oob stanza (pre-H1 agent) is never evaluated.
+func TestHostOOB_NoStanzaIgnored(t *testing.T) {
+	st := newOOBStore(t)
+	st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{})
+	var events []string
+	c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	c.Check()
+	if len(events) != 0 {
+		t.Fatalf("no oob stanza must not alert, got %v", events)
+	}
+}

hub/internal/store/host_oob.go

@@ -0,0 +1,59 @@
+package store
+
+import "encoding/json"
+
+// HostOOBRow is the latest operator-access (OOB) state per host (TASK H1), parsed from the newest
+// host_report. Present is false when the agent sent no oob stanza (pre-H1 / feature off) → never
+// alerted.
+type HostOOBRow struct {
+	HostID                 string
+	CustomerID             string
+	Present                bool
+	FelhomSshdActive       bool
+	FelhomSshdPort         int
+	Reachable              bool
+	ConfigInvalid          bool
+	OperatorPeerConfigured bool
+}
+
+// GetHostOOBStates returns the latest oob stanza per host (mirrors GetHostMgmtPlaneStates). A report
+// without the stanza yields Present=false; malformed JSON degrades to zero values, never an error.
+func (s *Store) GetHostOOBStates() ([]HostOOBRow, error) {
+	rows, err := s.db.Query(`
+		SELECT hr.host_id, hr.customer_id, hr.report_json
+		FROM host_reports hr
+		JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest
+		  ON hr.id = latest.mx`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []HostOOBRow
+	for rows.Next() {
+		var r HostOOBRow
+		var reportJSON string
+		if err := rows.Scan(&r.HostID, &r.CustomerID, &reportJSON); err != nil {
+			return nil, err
+		}
+		var body struct {
+			OOB *struct {
+				FelhomSshdActive       bool `json:"felhom_sshd_active"`
+				FelhomSshdPort         int  `json:"felhom_sshd_port"`
+				Reachable              bool `json:"reachable"`
+				ConfigInvalid          bool `json:"config_invalid"`
+				OperatorPeerConfigured bool `json:"operator_peer_configured"`
+			} `json:"oob"`
+		}
+		_ = json.Unmarshal([]byte(reportJSON), &body)
+		if body.OOB != nil {
+			r.Present = true
+			r.FelhomSshdActive = body.OOB.FelhomSshdActive
+			r.FelhomSshdPort = body.OOB.FelhomSshdPort
+			r.Reachable = body.OOB.Reachable
+			r.ConfigInvalid = body.OOB.ConfigInvalid
+			r.OperatorPeerConfigured = body.OOB.OperatorPeerConfigured
+		}
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}

hub/internal/store/wg_operator.go

@@ -95,6 +95,16 @@ func (s *Store) SetOperatorOOBPeer(pubkey, assignedIP string) error {
 	return tx.Commit()
 }
 
+// oobOperatorSSHKeyKey is the hub_settings key for the fleet operator SSH public key (H1).
+const oobOperatorSSHKeyKey = "oob_operator_ssh_pubkey"
+
+// SetOOBOperatorSSHKey stores the operator's SSH PUBLIC key (an authorized_keys line, delivered to
+// every box via the desired-state so felhom-sshd honours the operator login). "" clears it.
+func (s *Store) SetOOBOperatorSSHKey(pubkey string) error { return s.setSetting(oobOperatorSSHKeyKey, pubkey) }
+
+// GetOOBOperatorSSHKey returns the operator SSH public key ("" if unset).
+func (s *Store) GetOOBOperatorSSHKey() string { return s.getSetting(oobOperatorSSHKeyKey) }
+
 func shortKey(k string) string {
 	if len(k) > 12 {
 		return k[:12] + "…"