diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index cf08194..9c0b8a5 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -381,6 +381,9 @@ func main() { // TASK G1: warn when a host's agent-independent watchdog auto-healed a missing /run/sshd privsep // dir — a recurring clobber that can lead to an SSH lockout (complements host_staleness). Same sweep. hostMgmtPlaneChecker := monitor.NewHostMgmtPlaneChecker(dataStore, dispatcher.ProcessEvent, logger) + // TASK H1: alert when a host's OPERATOR ACCESS is degraded — felhom-sshd down (with the operator + // peer configured) or its config invalid. Transition-based, same 60s sweep. + hostOOBChecker := monitor.NewHostOOBChecker(dataStore, dispatcher.ProcessEvent, logger) go func() { ticker := time.NewTicker(60 * time.Second) defer ticker.Stop() @@ -396,6 +399,7 @@ func main() { hostDiskChecker.Check() storageFillChecker.Check() hostMgmtPlaneChecker.Check() + hostOOBChecker.Check() } } }() diff --git a/hub/internal/api/wg.go b/hub/internal/api/wg.go index 9a61ac6..19b9809 100644 --- a/hub/internal/api/wg.go +++ b/hub/internal/api/wg.go @@ -15,6 +15,7 @@ import ( "io" "net/http" "net/netip" + "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" @@ -48,6 +49,29 @@ func validateWGPubkey(pk string) error { return nil } +// validSSHAuthorizedKey does a conservative shape check on an SSH public key (an authorized_keys +// line): a known key type, a base64 blob, no newlines/control chars (it is written verbatim into a +// per-user authorized_keys file, so a hostile value must not inject options or extra lines). +func validSSHAuthorizedKey(line string) bool { + line = strings.TrimSpace(line) + if line == "" || strings.ContainsAny(line, "\n\r\x00") { + return false + } + fields := strings.Fields(line) + if len(fields) < 2 { + return false + } + switch fields[0] { + case "ssh-ed25519", "ssh-rsa", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521", "sk-ssh-ed25519@openssh.com": + default: + return false + } + if _, err := base64.StdEncoding.DecodeString(fields[1]); err != nil { + return false + } + return true +} + // syncAfterMutation runs an inline sync after a peer mutation. The DB write already happened — // it is the source of truth — so a push failure is REPORTED, not rolled back: the reconciler's // next tick converges the endpoint (Scenario D). @@ -327,6 +351,10 @@ func (h *Handler) mergeWireguard(hostID, desired string) string { h.logger.Printf("[WARN] wg merge %s: operator OOB peer lookup failed: %v (serving without oob_peer_ip)", hostID, oerr) } else if op != nil { wgBlock["oob_peer_ip"] = op.AssignedIP // bare IPv4, e.g. "10.77.0.250" + // The operator SSH pubkey rides alongside (agent writes felhom-sshd's authorized_keys from it). + if k := h.store.GetOOBOperatorSSHKey(); k != "" { + wgBlock["oob_operator_ssh_key"] = k + } } doc["wireguard"] = wgBlock out, err := json.Marshal(doc) @@ -438,6 +466,7 @@ func (h *Handler) handleAdminSetOperatorPeer(w http.ResponseWriter, r *http.Requ var req struct { Pubkey string `json:"pubkey"` AssignedIP string `json:"assigned_ip"` // bare IPv4, e.g. "10.77.0.250" + SSHPubkey string `json:"ssh_pubkey"` // optional: the operator's SSH authorized_keys line } if err := json.Unmarshal(body, &req); err != nil { http.Error(w, "Invalid payload: body must be JSON", http.StatusBadRequest) @@ -447,6 +476,10 @@ func (h *Handler) handleAdminSetOperatorPeer(w http.ResponseWriter, r *http.Requ http.Error(w, "Invalid payload: "+err.Error(), http.StatusBadRequest) return } + if req.SSHPubkey != "" && !validSSHAuthorizedKey(req.SSHPubkey) { + http.Error(w, "Invalid payload: ssh_pubkey must be an ssh-ed25519/ssh-rsa/ecdsa authorized_keys line", http.StatusBadRequest) + return + } if err := h.store.SetOperatorOOBPeer(req.Pubkey, req.AssignedIP); err != nil { if err == store.ErrWGEndpointUnset { http.Error(w, "wg endpoint not configured", http.StatusConflict) @@ -455,6 +488,11 @@ func (h *Handler) handleAdminSetOperatorPeer(w http.ResponseWriter, r *http.Requ http.Error(w, "Invalid operator peer: "+err.Error(), http.StatusBadRequest) return } + if req.SSHPubkey != "" { + if err := h.store.SetOOBOperatorSSHKey(req.SSHPubkey); err != nil { + h.logger.Printf("[WARN] operator peer set but ssh_pubkey store failed: %v", err) + } + } syncStatus := h.syncAfterMutation(r.Context()) bumped, berr := h.store.BumpAllHostGenerations() if berr != nil { diff --git a/hub/internal/monitor/host_oob.go b/hub/internal/monitor/host_oob.go new file mode 100644 index 0000000..0f7941d --- /dev/null +++ b/hub/internal/monitor/host_oob.go @@ -0,0 +1,148 @@ +package monitor + +import ( + "encoding/json" + "log" + "sync" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// HostOOBChecker raises an operator WARNING when a host's OOB access path is DEGRADED — felhom-sshd +// down (while the operator peer is configured, i.e. OOB is meant to work) OR its config is invalid. +// It answers "can the operator get into this box right now, and if not, why" proactively, from the +// hub. Transition-based (ok↔degraded, one event per transition — the HostCapabilityChecker shape), so +// a persistent problem alerts ONCE, not every 60s sweep, and a recovery is noted. +// +// A host with no oob stanza (pre-H1 / feature off) is never evaluated. A degraded state requires the +// operator peer to be configured — a box where OOB was never set up is not "broken". +type HostOOBChecker struct { + store *store.Store + logger *log.Logger + onEvent EventNotifyFunc + + mu sync.Mutex + degraded map[string]bool // hostID → currently-degraded + customerOf map[string]string +} + +// NewHostOOBChecker seeds per-host degraded state from the latest reports WITHOUT alerting (a problem +// present at startup alerts on the first transition-in evaluated after seed = never re-alerts a +// steady bad state; matches HostCapabilityChecker). Actually seeds silent, then Check transitions. +func NewHostOOBChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostOOBChecker { + c := &HostOOBChecker{ + store: s, + logger: logger, + onEvent: onEvent, + degraded: make(map[string]bool), + customerOf: make(map[string]string), + } + rows, err := s.GetHostOOBStates() + if err != nil { + logger.Printf("[WARN] Host OOB checker: failed to seed: %v", err) + return c + } + seeded := 0 + for _, row := range rows { + if s.IsCustomerBlocked(row.CustomerID) || !row.Present { + continue + } + c.customerOf[row.HostID] = row.CustomerID + if oobDegraded(row) { + c.degraded[row.HostID] = true // seed the bad state so we don't re-alert it on cycle 1 + seeded++ + } + } + logger.Printf("[INFO] Host OOB checker initialized: %d host(s) seeded degraded", seeded) + return c +} + +// oobDegraded is the degraded predicate: config invalid, OR (OOB meant to work — operator peer +// configured — AND felhom-sshd is not active/reachable). +func oobDegraded(r store.HostOOBRow) bool { + if !r.Present { + return false + } + if r.ConfigInvalid { + return true + } + if r.OperatorPeerConfigured && (!r.FelhomSshdActive || !r.Reachable) { + return true + } + return false +} + +// Check evaluates all hosts and emits oob_degraded / oob_recovered on transitions. +func (c *HostOOBChecker) Check() { + rows, err := c.store.GetHostOOBStates() + if err != nil { + c.logger.Printf("[WARN] Host OOB check failed: %v", err) + return + } + c.mu.Lock() + defer c.mu.Unlock() + + seen := make(map[string]bool, len(rows)) + for _, row := range rows { + if c.store.IsCustomerBlocked(row.CustomerID) { + delete(c.degraded, row.HostID) + continue + } + if !row.Present { + continue // no oob stanza → not evaluated + } + seen[row.HostID] = true + c.customerOf[row.HostID] = row.CustomerID + bad := oobDegraded(row) + was := c.degraded[row.HostID] + switch { + case bad && !was: + c.degraded[row.HostID] = true + c.emit(row, "oob_degraded", "warning") + case !bad && was: + delete(c.degraded, row.HostID) + c.emit(row, "oob_recovered", "info") + } + } + for id := range c.degraded { + if !seen[id] { + delete(c.degraded, id) + } + } +} + +// IsDegraded reports the current tracked state for a host (test/UI helper). +func (c *HostOOBChecker) IsDegraded(hostID string) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.degraded[hostID] +} + +func (c *HostOOBChecker) emit(row store.HostOOBRow, eventType, severity string) { + var msg string + if eventType == "oob_degraded" { + reason := "felhom-sshd unreachable" + if row.ConfigInvalid { + reason = "felhom-sshd config invalid (sshd -t fails)" + } + msg = "Host " + row.HostID + ": OPERATOR ACCESS DEGRADED — " + reason + + ". The break-glass net (auto-heal + vaulted root@pam console) is still under the box." + } else { + msg = "Host " + row.HostID + ": operator access recovered (felhom-sshd reachable again)." + } + details, _ := json.Marshal(map[string]any{ + "host_id": row.HostID, + "felhom_sshd_port": row.FelhomSshdPort, + "active": row.FelhomSshdActive, + "reachable": row.Reachable, + "config_invalid": row.ConfigInvalid, + }) + c.logger.Printf("[%s] Host OOB: %s (%s)", map[string]string{"warning": "WARN", "info": "INFO"}[severity], row.HostID, eventType) + if _, err := c.store.SaveEvent(row.CustomerID, eventType, severity, msg, string(details), "hub"); err != nil { + c.logger.Printf("[WARN] save %s for %s: %v", eventType, row.HostID, err) + return + } + if c.onEvent != nil { + c.onEvent(row.CustomerID, eventType, severity, msg, string(details), "hub") + } +} diff --git a/hub/internal/monitor/host_oob_test.go b/hub/internal/monitor/host_oob_test.go new file mode 100644 index 0000000..751ea55 --- /dev/null +++ b/hub/internal/monitor/host_oob_test.go @@ -0,0 +1,103 @@ +package monitor + +import ( + "io" + "log" + "path/filepath" + "testing" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" + _ "modernc.org/sqlite" +) + +func oobReport(active, reachable, configInvalid, operatorConfigured bool) []byte { + b := func(v bool) string { + if v { + return "true" + } + return "false" + } + return []byte(`{"host_id":"h1","oob":{"felhom_sshd_active":` + b(active) + + `,"felhom_sshd_port":8822,"reachable":` + b(reachable) + + `,"config_invalid":` + b(configInvalid) + + `,"operator_peer_configured":` + b(operatorConfigured) + `}}`) +} + +func newOOBStore(t *testing.T) *store.Store { + t.Helper() + st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"}) + st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}) + return st +} + +func TestHostOOB_DegradedThenRecoveredTransitions(t *testing.T) { + st := newOOBStore(t) + // healthy at construction (active+reachable, operator configured) + st.SaveHostReport("h1", "c1", oobReport(true, true, false, true), store.HostReportDenorm{}) + var events []string + c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + if c.IsDegraded("h1") { + t.Fatal("healthy host seeded degraded") + } + + // felhom-sshd goes DOWN with operator configured → one oob_degraded + st.SaveHostReport("h1", "c1", oobReport(false, false, false, true), store.HostReportDenorm{}) + c.Check() + if len(events) != 1 || events[0] != "oob_degraded" { + t.Fatalf("down+operator-configured → one oob_degraded, got %v", events) + } + c.Check() // persistent → no re-alert + if len(events) != 1 { + t.Fatalf("persistent degraded must not re-emit, got %v", events) + } + + // recovers → oob_recovered + st.SaveHostReport("h1", "c1", oobReport(true, true, false, true), store.HostReportDenorm{}) + c.Check() + if len(events) != 2 || events[1] != "oob_recovered" { + t.Fatalf("recovery → oob_recovered, got %v", events) + } +} + +func TestHostOOB_ConfigInvalidAlerts(t *testing.T) { + st := newOOBStore(t) + st.SaveHostReport("h1", "c1", oobReport(true, true, false, false), store.HostReportDenorm{}) + var events []string + c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + // config invalid (even without operator configured) → degraded + st.SaveHostReport("h1", "c1", oobReport(true, true, true, false), store.HostReportDenorm{}) + c.Check() + if len(events) != 1 || events[0] != "oob_degraded" { + t.Fatalf("config_invalid → oob_degraded, got %v", events) + } +} + +// A box where OOB was NEVER set up (no operator peer) with felhom-sshd down must NOT alert — it's not +// broken, it's unconfigured. +func TestHostOOB_DownButNoOperatorNotDegraded(t *testing.T) { + st := newOOBStore(t) + st.SaveHostReport("h1", "c1", oobReport(false, false, false, false), store.HostReportDenorm{}) + var events []string + c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + c.Check() + if len(events) != 0 { + t.Fatalf("unconfigured OOB (no operator peer) must not alert, got %v", events) + } +} + +// A report with no oob stanza (pre-H1 agent) is never evaluated. +func TestHostOOB_NoStanzaIgnored(t *testing.T) { + st := newOOBStore(t) + st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{}) + var events []string + c := NewHostOOBChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + c.Check() + if len(events) != 0 { + t.Fatalf("no oob stanza must not alert, got %v", events) + } +} diff --git a/hub/internal/store/host_oob.go b/hub/internal/store/host_oob.go new file mode 100644 index 0000000..5c35778 --- /dev/null +++ b/hub/internal/store/host_oob.go @@ -0,0 +1,59 @@ +package store + +import "encoding/json" + +// HostOOBRow is the latest operator-access (OOB) state per host (TASK H1), parsed from the newest +// host_report. Present is false when the agent sent no oob stanza (pre-H1 / feature off) → never +// alerted. +type HostOOBRow struct { + HostID string + CustomerID string + Present bool + FelhomSshdActive bool + FelhomSshdPort int + Reachable bool + ConfigInvalid bool + OperatorPeerConfigured bool +} + +// GetHostOOBStates returns the latest oob stanza per host (mirrors GetHostMgmtPlaneStates). A report +// without the stanza yields Present=false; malformed JSON degrades to zero values, never an error. +func (s *Store) GetHostOOBStates() ([]HostOOBRow, error) { + rows, err := s.db.Query(` + SELECT hr.host_id, hr.customer_id, hr.report_json + FROM host_reports hr + JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest + ON hr.id = latest.mx`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []HostOOBRow + for rows.Next() { + var r HostOOBRow + var reportJSON string + if err := rows.Scan(&r.HostID, &r.CustomerID, &reportJSON); err != nil { + return nil, err + } + var body struct { + OOB *struct { + FelhomSshdActive bool `json:"felhom_sshd_active"` + FelhomSshdPort int `json:"felhom_sshd_port"` + Reachable bool `json:"reachable"` + ConfigInvalid bool `json:"config_invalid"` + OperatorPeerConfigured bool `json:"operator_peer_configured"` + } `json:"oob"` + } + _ = json.Unmarshal([]byte(reportJSON), &body) + if body.OOB != nil { + r.Present = true + r.FelhomSshdActive = body.OOB.FelhomSshdActive + r.FelhomSshdPort = body.OOB.FelhomSshdPort + r.Reachable = body.OOB.Reachable + r.ConfigInvalid = body.OOB.ConfigInvalid + r.OperatorPeerConfigured = body.OOB.OperatorPeerConfigured + } + out = append(out, r) + } + return out, rows.Err() +} diff --git a/hub/internal/store/wg_operator.go b/hub/internal/store/wg_operator.go index 28042f4..1d78dc0 100644 --- a/hub/internal/store/wg_operator.go +++ b/hub/internal/store/wg_operator.go @@ -95,6 +95,16 @@ func (s *Store) SetOperatorOOBPeer(pubkey, assignedIP string) error { return tx.Commit() } +// oobOperatorSSHKeyKey is the hub_settings key for the fleet operator SSH public key (H1). +const oobOperatorSSHKeyKey = "oob_operator_ssh_pubkey" + +// SetOOBOperatorSSHKey stores the operator's SSH PUBLIC key (an authorized_keys line, delivered to +// every box via the desired-state so felhom-sshd honours the operator login). "" clears it. +func (s *Store) SetOOBOperatorSSHKey(pubkey string) error { return s.setSetting(oobOperatorSSHKeyKey, pubkey) } + +// GetOOBOperatorSSHKey returns the operator SSH public key ("" if unset). +func (s *Store) GetOOBOperatorSSHKey() string { return s.getSetting(oobOperatorSSHKeyKey) } + func shortKey(k string) string { if len(k) > 12 { return k[:12] + "…"