diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index ef03172..2bcc543 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,5 +1,24 @@ # Felhom Hub — Changelog +## v0.34.0 — break-glass recovery vault + mgmt_plane surfacing (TASK G1) (2026-07-05) + +The hub half of the management-plane break-glass system (prerequisite for felhom-sshd / H1; agent half += felhom-agent v0.71.0). Closes the recovery gap from +`documentation/audits/SPIKE-felhom-sshd-2026-07-05.md` §8/#9. + +- **Break-glass credential vault** (`store.host_recovery` + `internal/store/host_recovery.go`): a + per-host root@pam console password, stored at rest, operator-retrievable — the human fallback for + reaching the PVE web console (pveproxy, a failure domain distinct from sshd) when both the sshd path + and the agent-independent auto-heal have failed. `PUT /hosts/{id}/recovery-credential` (SELF-scoped + host key — day-0 vaults it) + `GET /admin/hosts/{id}/recovery-credential` (GLOBAL key only — a host + key cannot read its own console password back). Secret discipline: never logged (username + length + only); red-proofed that the password never reaches the hub log. +- **mgmt_plane surfacing** (`internal/monitor/host_mgmtplane.go`, on the 60s sweep): parses the agent's + additive `mgmt_plane` heartbeat stanza and raises a `mgmt_plane_healed` WARNING when the watchdog + auto-healed a missing `/run/sshd` (new `privsep_healed_at`) — a recurring clobber surfaces BEFORE it + becomes a lockout, complementing host_staleness. Trust-on-first-report (seed, then alert on change), + mirroring HostLeafChecker. + ## v0.33.0 — S2 offsite connectivity: box-facing WG registration + wireguard desired-state block + /offsite UI (2026-07-04) Doc 06 roadmap row S2 (commits `fcf84a0`/`ba52005`/`13203c2`); the S2 architectural decision: diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index 297557a..cf08194 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -378,6 +378,9 @@ func main() { // target (dump/backup volume, data drive, lvmthin pool, PBS datastore). Born/persistent; excludes the // root-backed builtin (hostDiskChecker owns root, no double-alert); emits natural `critical`. Same sweep. storageFillChecker := monitor.NewStorageFillChecker(dataStore, cfg.Alerting.StorageFillWarnPercent, cfg.Alerting.StorageFillCritPercent, dispatcher.ProcessEvent, logger) + // TASK G1: warn when a host's agent-independent watchdog auto-healed a missing /run/sshd privsep + // dir — a recurring clobber that can lead to an SSH lockout (complements host_staleness). Same sweep. + hostMgmtPlaneChecker := monitor.NewHostMgmtPlaneChecker(dataStore, dispatcher.ProcessEvent, logger) go func() { ticker := time.NewTicker(60 * time.Second) defer ticker.Stop() @@ -392,6 +395,7 @@ func main() { hostLeafChecker.Check() hostDiskChecker.Check() storageFillChecker.Check() + hostMgmtPlaneChecker.Check() } } }() diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index 02309c5..82d9045 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -155,6 +155,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/escrow"): hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/escrow") h.handleHostEscrowPut(w, r, hostID) + // G1 break-glass: day-0 vaults the root@pam console credential (self-scoped host key); the + // operator retrieves it via the /admin/ path (global key only). + case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/recovery-credential"): + hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/recovery-credential") + h.handleHostRecoveryCredentialPut(w, r, hostID) + case r.Method == http.MethodGet && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-credential"): + hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/admin/hosts/"), "/recovery-credential") + h.handleAdminGetRecoveryCredential(w, r, hostID) // DR capstone (slice 10D). Recovery-mode toggle (global key); re-enroll + restore-directive // (gated on recovery mode — no old key needed, the box is lost). case r.Method == http.MethodPut && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-mode"): @@ -883,6 +891,95 @@ func (h *Handler) handleHostEscrowPut(w http.ResponseWriter, r *http.Request, pa w.Write([]byte(`{"status":"ok"}`)) } +// handleHostRecoveryCredentialPut vaults a host's break-glass root@pam console credential (TASK G1). +// SELF-SCOPED (a host key writes only its own; global may write any) — day-0 posts it with the +// host api_key. The secret is stored at rest and NEVER logged (only the username + a length are +// logged). This is the human fallback for when both the sshd path AND the agent-independent +// auto-heal have failed: the operator retrieves it to reach the PVE web console (pveproxy — a +// failure domain distinct from sshd). +func (h *Handler) handleHostRecoveryCredentialPut(w http.ResponseWriter, r *http.Request, pathHostID string) { + authHostID, _, isGlobal, ok := h.checkAuthHost(r) + if !ok { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + if pathHostID == "" { + http.Error(w, "Missing host_id", http.StatusBadRequest) + return + } + if !isGlobal && authHostID != pathHostID { + http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden) + return + } + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<16)) // 64 KiB cap; a username+password is tiny + if err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return + } + var req struct { + Username string `json:"username"` + Password string `json:"password"` + } + if err := json.Unmarshal(body, &req); err != nil || req.Username == "" || req.Password == "" { + http.Error(w, "Invalid payload: username + password required", http.StatusBadRequest) + return + } + // The host must exist (mint-first) — a per-host key already proves it; the global path re-checks. + if isGlobal { + host, herr := h.store.GetHost(pathHostID) + if herr != nil || host == nil { + http.Error(w, "Unknown host_id", http.StatusBadRequest) + return + } + } + if err := h.store.SaveHostRecoveryCredential(pathHostID, req.Username, req.Password); err != nil { + h.logger.Printf("[ERROR] Failed to vault recovery credential for host %s: %v", pathHostID, err) + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + // SECRET DISCIPLINE: log the username + a length only — NEVER the password. + h.logger.Printf("[INFO] vaulted break-glass recovery credential for host %s (user=%s, secret %d chars)", + pathHostID, req.Username, len(req.Password)) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status":"ok"}`)) +} + +// handleAdminGetRecoveryCredential returns a host's vaulted break-glass credential to the OPERATOR +// (global key only — a per-host key must NOT read its own console password back out). This is the +// authenticated retrieval path the break-glass runbook uses. The response body carries the secret by +// necessity; it is never written to the hub log. +func (h *Handler) handleAdminGetRecoveryCredential(w http.ResponseWriter, r *http.Request, pathHostID string) { + _, _, isGlobal, ok := h.checkAuthHost(r) + if !ok || !isGlobal { + http.Error(w, "Unauthorized", http.StatusUnauthorized) // operator/global key ONLY + return + } + if pathHostID == "" { + http.Error(w, "Missing host_id", http.StatusBadRequest) + return + } + cred, err := h.store.GetHostRecoveryCredential(pathHostID) + if err != nil { + h.logger.Printf("[ERROR] Failed to read recovery credential for host %s: %v", pathHostID, err) + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + if cred == nil { + http.Error(w, "No recovery credential vaulted for this host", http.StatusNotFound) + return + } + h.logger.Printf("[INFO] operator retrieved break-glass recovery credential for host %s (user=%s)", pathHostID, cred.Username) + resp, _ := json.Marshal(map[string]string{ + "host_id": cred.HostID, + "username": cred.Username, + "password": cred.Secret, + "set_at": cred.SetAt.UTC().Format(time.RFC3339), + }) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write(resp) +} + // handleGetDesiredState serves a host its authoritative desired-state (slice 10A). Per-host key, // SELF-SCOPED: a host reads ONLY its own (the global operator key may read any). The agent fetches // this when the heartbeat envelope's desired_generation has advanced past its cached one. The diff --git a/hub/internal/api/host_recovery_test.go b/hub/internal/api/host_recovery_test.go new file mode 100644 index 0000000..e162e6c --- /dev/null +++ b/hub/internal/api/host_recovery_test.go @@ -0,0 +1,74 @@ +package api + +import ( + "bytes" + "log" + "path/filepath" + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" + _ "modernc.org/sqlite" +) + +func TestRecoveryCredential_VaultSelfScopedAndOperatorRetrieval(t *testing.T) { + h, st, _ := newTestHandler(t) + st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"}) + st.UpsertHost(&store.Host{HostID: "h2", CustomerID: "c1", APIKey: "HKEY2"}) + + body := `{"username":"root@pam","password":"Str0ng-Break-Glass"}` + + // self-scoped write: h1's key vaults h1 → 200 + if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "HKEY", body); rr.Code != 200 { + t.Fatalf("self vault = %d: %s", rr.Code, rr.Body.String()) + } + // a host key CANNOT vault ANOTHER host → 403 + if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "HKEY2", body); rr.Code != 403 { + t.Fatalf("cross-host vault must be 403, got %d", rr.Code) + } + // unauthenticated → 401 + if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "", body); rr.Code != 401 { + t.Fatalf("unauth vault must be 401, got %d", rr.Code) + } + + // operator retrieval requires the GLOBAL key; a host key is refused (401 — can't read its own back) + if rr := do(h, "GET", "/admin/hosts/h1/recovery-credential", "HKEY", ""); rr.Code != 401 { + t.Fatalf("host key reading recovery credential must be 401, got %d", rr.Code) + } + rr := do(h, "GET", "/admin/hosts/h1/recovery-credential", globalKey, "") + if rr.Code != 200 { + t.Fatalf("operator retrieval = %d: %s", rr.Code, rr.Body.String()) + } + if !strings.Contains(rr.Body.String(), "Str0ng-Break-Glass") || !strings.Contains(rr.Body.String(), "root@pam") { + t.Fatalf("retrieval must return the vaulted credential, got %s", rr.Body.String()) + } + // a host with no credential → 404 + if rr := do(h, "GET", "/admin/hosts/h2/recovery-credential", globalKey, ""); rr.Code != 404 { + t.Fatalf("no-credential host must be 404, got %d", rr.Code) + } +} + +// SECRET DISCIPLINE (red-proof for trap 3): the password must NEVER appear in the hub log — on the +// vault write NOR the operator retrieval. Companion: if a handler logged the password, this fails. +func TestRecoveryCredential_PasswordNeverLogged(t *testing.T) { + var buf bytes.Buffer + path := filepath.Join(t.TempDir(), "test.db") + st, err := store.New(path, log.New(&buf, "", 0)) + if err != nil { + t.Fatalf("store.New: %v", err) + } + defer st.Close() + h := New(st, globalKey, "", "", nil, log.New(&buf, "", 0)) + st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"}) + + const secret = "SuperSecret-DoNotLog-42" + if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "HKEY", `{"username":"root@pam","password":"`+secret+`"}`); rr.Code != 200 { + t.Fatalf("vault = %d", rr.Code) + } + if rr := do(h, "GET", "/admin/hosts/h1/recovery-credential", globalKey, ""); rr.Code != 200 { + t.Fatalf("retrieve = %d", rr.Code) + } + if strings.Contains(buf.String(), secret) { + t.Fatalf("the recovery password LEAKED into the hub log:\n%s", buf.String()) + } +} diff --git a/hub/internal/monitor/host_mgmtplane.go b/hub/internal/monitor/host_mgmtplane.go new file mode 100644 index 0000000..63b080d --- /dev/null +++ b/hub/internal/monitor/host_mgmtplane.go @@ -0,0 +1,122 @@ +package monitor + +import ( + "encoding/json" + "log" + "sync" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// HostMgmtPlaneChecker raises an operator WARNING when a host's agent-independent break-glass watchdog +// AUTO-HEALED a missing /run/sshd privsep dir (TASK G1). The heal itself is silent and login-free (the +// point of the watchdog); this surfaces a RECURRING clobber so the operator can find the cause BEFORE +// it becomes a full management lockout — complementing HostStalenessChecker (which only catches a box +// gone silent). Sibling of HostLeafChecker; runs on the same 60s sweep. +// +// Design (mirrors HostLeafChecker's trust-on-first-report): the state is the host's last-seen +// privsep_healed_at marker timestamp. The watchdog rewrites the marker on EACH heal, so a new, different +// timestamp = a new heal event → one warning. The first observation of a non-empty timestamp seeds the +// baseline WITHOUT alerting (it may be a heal from before the hub was watching — avoid a false alarm on +// startup; a genuinely recurring cause re-heals and re-alerts on the next occurrence). An empty +// timestamp (healthy host / old agent) never alerts and never overwrites a baseline. +type HostMgmtPlaneChecker struct { + store *store.Store + logger *log.Logger + onEvent EventNotifyFunc + + mu sync.Mutex + states map[string]string // hostID → last-seen privsep_healed_at + customerOf map[string]string +} + +// NewHostMgmtPlaneChecker seeds per-host baselines from the latest reports. No events on init. +func NewHostMgmtPlaneChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostMgmtPlaneChecker { + mc := &HostMgmtPlaneChecker{ + store: s, + logger: logger, + onEvent: onEvent, + states: make(map[string]string), + customerOf: make(map[string]string), + } + rows, err := s.GetHostMgmtPlaneStates() + if err != nil { + logger.Printf("[WARN] Host mgmt-plane checker: failed to seed: %v", err) + return mc + } + seeded := 0 + for _, row := range rows { + if s.IsCustomerBlocked(row.CustomerID) || row.PrivsepHealedAt == "" { + continue + } + mc.customerOf[row.HostID] = row.CustomerID + mc.states[row.HostID] = row.PrivsepHealedAt + seeded++ + } + logger.Printf("[INFO] Host mgmt-plane checker initialized: %d host heal-state(s) seeded", seeded) + return mc +} + +// Check evaluates all hosts and emits mgmt_plane_healed on a NEW heal timestamp. +func (mc *HostMgmtPlaneChecker) Check() { + rows, err := mc.store.GetHostMgmtPlaneStates() + if err != nil { + mc.logger.Printf("[WARN] Host mgmt-plane check failed: %v", err) + return + } + mc.mu.Lock() + defer mc.mu.Unlock() + + seen := make(map[string]bool, len(rows)) + for _, row := range rows { + if mc.store.IsCustomerBlocked(row.CustomerID) { + delete(mc.states, row.HostID) + continue + } + seen[row.HostID] = true + if row.PrivsepHealedAt == "" { + continue // no heal marker → healthy / old agent → no alert, no baseline change + } + mc.customerOf[row.HostID] = row.CustomerID + old := mc.states[row.HostID] + if old == "" { + mc.states[row.HostID] = row.PrivsepHealedAt // first observation → seed, no event + continue + } + if old == row.PrivsepHealedAt { + continue // same heal already alerted + } + mc.states[row.HostID] = row.PrivsepHealedAt + mc.emit(row.HostID, row.CustomerID, row.PrivsepHealedAt) + } + + for id := range mc.states { + if !seen[id] { + delete(mc.states, id) + } + } +} + +// GetState returns the last-seen heal timestamp for a host ("" if none). +func (mc *HostMgmtPlaneChecker) GetState(hostID string) string { + mc.mu.Lock() + defer mc.mu.Unlock() + return mc.states[hostID] +} + +func (mc *HostMgmtPlaneChecker) emit(hostID, customerID, healedAt string) { + msg := "Host " + hostID + ": the management-plane privsep dir (/run/sshd) was missing and was AUTO-HEALED by the watchdog at " + healedAt + + " — a recurring cause can lead to an SSH lockout; investigate (e.g. a unit declaring RuntimeDirectory=sshd)." + details, _ := json.Marshal(map[string]string{ + "host_id": hostID, + "privsep_healed_at": healedAt, + }) + mc.logger.Printf("[WARN] Host mgmt-plane: %s privsep dir auto-healed at %s (mgmt_plane_healed)", hostID, healedAt) + if _, err := mc.store.SaveEvent(customerID, "mgmt_plane_healed", "warning", msg, string(details), "hub"); err != nil { + mc.logger.Printf("[WARN] save mgmt_plane_healed for %s: %v", hostID, err) + return + } + if mc.onEvent != nil { + mc.onEvent(customerID, "mgmt_plane_healed", "warning", msg, string(details), "hub") + } +} diff --git a/hub/internal/monitor/host_mgmtplane_test.go b/hub/internal/monitor/host_mgmtplane_test.go new file mode 100644 index 0000000..be4637e --- /dev/null +++ b/hub/internal/monitor/host_mgmtplane_test.go @@ -0,0 +1,90 @@ +package monitor + +import ( + "io" + "log" + "path/filepath" + "testing" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" + _ "modernc.org/sqlite" +) + +func reportWithHeal(healedAt string) []byte { + if healedAt == "" { + return []byte(`{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true}}`) + } + return []byte(`{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true,"healed_recently":true,"privsep_healed_at":"` + healedAt + `"}}`) +} + +func newMgmtStore(t *testing.T) *store.Store { + t.Helper() + st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"}) + st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}) + return st +} + +// A NEW heal timestamp fires exactly one mgmt_plane_healed; the first observation only seeds; a repeat +// of the same timestamp does not re-fire. Companion TestHostMgmtPlaneChecker_NoHealNoEvent proves it's +// the heal, not the sweep, that fires it (drop the emit → this test fails). +func TestHostMgmtPlaneChecker_NewHealAlertsOnce(t *testing.T) { + st := newMgmtStore(t) + // first report already carries a heal marker → seed baseline, NO event on construction. + st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T16:42:17Z"), store.HostReportDenorm{}) + var events []string + mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + if mc.GetState("h1") != "2026-07-05T16:42:17Z" { + t.Fatalf("seed = %q", mc.GetState("h1")) + } + if len(events) != 0 { + t.Fatalf("construction must not emit, got %v", events) + } + + // a NEW heal (different timestamp) → one warning. + st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T18:00:00Z"), store.HostReportDenorm{}) + mc.Check() + if len(events) != 1 || events[0] != "mgmt_plane_healed" { + t.Fatalf("new heal → one mgmt_plane_healed, got %v", events) + } + // same timestamp again → no duplicate. + mc.Check() + if len(events) != 1 { + t.Fatalf("same heal must not re-emit, got %v", events) + } +} + +func TestHostMgmtPlaneChecker_NoHealNoEvent(t *testing.T) { + st := newMgmtStore(t) + st.SaveHostReport("h1", "c1", reportWithHeal(""), store.HostReportDenorm{}) // healthy, no marker + var events []string + mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + mc.Check() + mc.Check() + if len(events) != 0 { + t.Fatalf("a healthy host (no heal marker) must never alert, got %v", events) + } + if mc.GetState("h1") != "" { + t.Fatalf("no marker → no baseline, got %q", mc.GetState("h1")) + } +} + +// A recurring clobber: heal at T1 (seed), heal again at T2 (alert), heal again at T3 (alert) — each +// distinct heal surfaces, which is the whole point (find the recurring cause before a lockout). +func TestHostMgmtPlaneChecker_RecurringHealsEachAlert(t *testing.T) { + st := newMgmtStore(t) + st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T10:00:00Z"), store.HostReportDenorm{}) + var events []string + mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0)) + st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T11:00:00Z"), store.HostReportDenorm{}) + mc.Check() + st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T12:00:00Z"), store.HostReportDenorm{}) + mc.Check() + if len(events) != 2 { + t.Fatalf("two distinct new heals after seed → two alerts, got %d (%v)", len(events), events) + } +} diff --git a/hub/internal/store/host_recovery.go b/hub/internal/store/host_recovery.go new file mode 100644 index 0000000..0a5357c --- /dev/null +++ b/hub/internal/store/host_recovery.go @@ -0,0 +1,110 @@ +package store + +import ( + "database/sql" + "encoding/json" + "time" +) + +// HostRecoveryCredential is the break-glass PVE console credential for a host (TASK G1). Secret is +// the root@pam password — a hub-held secret, operator-retrievable (NOT zero-knowledge like escrow). +type HostRecoveryCredential struct { + HostID string + Username string + Secret string + SetAt time.Time +} + +// SaveHostRecoveryCredential upserts a host's break-glass credential (last-write-wins: day-0 sets it, +// --rotate re-sets). The secret is stored as-is at rest; the hub NEVER logs it and only ever returns +// it over the operator-authenticated retrieval path. +func (s *Store) SaveHostRecoveryCredential(hostID, username, secret string) error { + _, err := s.db.Exec(` + INSERT INTO host_recovery (host_id, username, secret, set_at, updated_at) + VALUES (?, ?, ?, datetime('now'), datetime('now')) + ON CONFLICT(host_id) DO UPDATE SET + username = excluded.username, + secret = excluded.secret, + updated_at = datetime('now')`, + hostID, username, secret) + return err +} + +// GetHostRecoveryCredential returns a host's break-glass credential, or (nil, nil) if none is vaulted. +func (s *Store) GetHostRecoveryCredential(hostID string) (*HostRecoveryCredential, error) { + var c HostRecoveryCredential + var setAt string + err := s.db.QueryRow( + `SELECT host_id, username, secret, set_at FROM host_recovery WHERE host_id = ?`, hostID). + Scan(&c.HostID, &c.Username, &c.Secret, &setAt) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + c.SetAt = parseSQLiteTime(setAt) + return &c, nil +} + +// HasHostRecoveryCredential reports whether a host already has a vaulted credential (day-0 idempotency: +// don't regenerate/re-set on a re-run unless --rotate). +func (s *Store) HasHostRecoveryCredential(hostID string) (bool, error) { + var one int + err := s.db.QueryRow(`SELECT 1 FROM host_recovery WHERE host_id = ?`, hostID).Scan(&one) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +// HostMgmtPlaneRow is the latest management-plane state per host (TASK G1), parsed from the newest +// host_report. PrivsepHealedAt is the watchdog heal-marker timestamp ("" when never healed / old agent). +type HostMgmtPlaneRow struct { + HostID string + CustomerID string + PrivsepDirOK bool + SshdReachable bool + PrivsepHealedAt string +} + +// GetHostMgmtPlaneStates returns the latest mgmt_plane stanza per host (mirrors +// GetHostLeafFingerprints). A report without the stanza (old agent, feature off) yields zero values → +// no alert. Malformed JSON degrades to zero values, never an error for that host. +func (s *Store) GetHostMgmtPlaneStates() ([]HostMgmtPlaneRow, error) { + rows, err := s.db.Query(` + SELECT hr.host_id, hr.customer_id, hr.report_json + FROM host_reports hr + JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest + ON hr.id = latest.mx`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []HostMgmtPlaneRow + for rows.Next() { + var r HostMgmtPlaneRow + var reportJSON string + if err := rows.Scan(&r.HostID, &r.CustomerID, &reportJSON); err != nil { + return nil, err + } + var body struct { + MgmtPlane *struct { + PrivsepDirOK bool `json:"privsep_dir_ok"` + SshdReachable bool `json:"sshd_reachable"` + PrivsepHealedAt string `json:"privsep_healed_at"` + } `json:"mgmt_plane"` + } + _ = json.Unmarshal([]byte(reportJSON), &body) // malformed/old → nil mgmt_plane → zero values + if body.MgmtPlane != nil { + r.PrivsepDirOK = body.MgmtPlane.PrivsepDirOK + r.SshdReachable = body.MgmtPlane.SshdReachable + r.PrivsepHealedAt = body.MgmtPlane.PrivsepHealedAt + } + out = append(out, r) + } + return out, rows.Err() +} diff --git a/hub/internal/store/host_recovery_test.go b/hub/internal/store/host_recovery_test.go new file mode 100644 index 0000000..2ec4bc4 --- /dev/null +++ b/hub/internal/store/host_recovery_test.go @@ -0,0 +1,83 @@ +package store + +import "testing" + +func TestHostRecoveryCredential_RoundTripUpsertAndAbsent(t *testing.T) { + s := newTestStore(t) + if err := s.UpsertHost(&Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil { + t.Fatalf("UpsertHost: %v", err) + } + + // absent → (nil, nil) + Has=false + got, err := s.GetHostRecoveryCredential("h1") + if err != nil || got != nil { + t.Fatalf("absent cred: got %+v / %v (want nil,nil)", got, err) + } + has, _ := s.HasHostRecoveryCredential("h1") + if has { + t.Fatal("HasHostRecoveryCredential must be false before any vault") + } + + // vault → round-trips + if err := s.SaveHostRecoveryCredential("h1", "root@pam", "s3cret-Aa1"); err != nil { + t.Fatalf("SaveHostRecoveryCredential: %v", err) + } + got, err = s.GetHostRecoveryCredential("h1") + if err != nil || got == nil { + t.Fatalf("GetHostRecoveryCredential: %+v / %v", got, err) + } + if got.Username != "root@pam" || got.Secret != "s3cret-Aa1" { + t.Fatalf("round-trip mismatch: %+v", got) + } + if has, _ := s.HasHostRecoveryCredential("h1"); !has { + t.Fatal("HasHostRecoveryCredential must be true after vault") + } + + // upsert (rotate) → overwrites last-write-wins + if err := s.SaveHostRecoveryCredential("h1", "root@pam", "rotated-Bb2"); err != nil { + t.Fatalf("re-vault: %v", err) + } + got, _ = s.GetHostRecoveryCredential("h1") + if got.Secret != "rotated-Bb2" { + t.Fatalf("rotate did not overwrite: %+v", got) + } +} + +func TestGetHostMgmtPlaneStates_ParsesHealMarker(t *testing.T) { + s := newTestStore(t) + if err := s.UpsertHost(&Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil { + t.Fatalf("UpsertHost: %v", err) + } + // a report WITH a heal marker + report := `{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true,"healed_recently":true,"privsep_healed_at":"2026-07-05T16:42:17Z"}}` + if err := s.SaveHostReport("h1", "c1", []byte(report), HostReportDenorm{}); err != nil { + t.Fatalf("SaveHostReport: %v", err) + } + rows, err := s.GetHostMgmtPlaneStates() + if err != nil { + t.Fatalf("GetHostMgmtPlaneStates: %v", err) + } + var found bool + for _, r := range rows { + if r.HostID == "h1" { + found = true + if !r.PrivsepDirOK || r.PrivsepHealedAt != "2026-07-05T16:42:17Z" { + t.Fatalf("parsed row wrong: %+v", r) + } + } + } + if !found { + t.Fatal("h1 not in mgmt-plane states") + } + + // a report WITHOUT the stanza (old agent) → zero values, no crash + if err := s.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), HostReportDenorm{}); err != nil { + t.Fatalf("SaveHostReport2: %v", err) + } + rows, _ = s.GetHostMgmtPlaneStates() + for _, r := range rows { + if r.HostID == "h1" && r.PrivsepHealedAt != "" { + t.Fatalf("old-agent report should yield empty healed_at, got %+v", r) + } + } +} diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index 4d5e563..a4d5527 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -398,6 +398,24 @@ func (s *Store) migrate() error { return err } + // host_recovery (TASK G1): the break-glass root@pam console credential, vaulted at rest and + // operator-retrievable. UNLIKE host_escrow (opaque, hub-can't-open), this IS a hub-held secret the + // operator retrieves to reach the PVE web console (pveproxy — a failure domain distinct from sshd) + // when both the sshd path AND the agent-independent auto-heal have failed. One row per host, + // last-write-wins (day-0 sets it; --rotate re-sets). Never in desired-state, never logged. + _, err = s.db.Exec(` + CREATE TABLE IF NOT EXISTS host_recovery ( + host_id TEXT PRIMARY KEY, + username TEXT NOT NULL, + secret TEXT NOT NULL, + set_at DATETIME NOT NULL DEFAULT (datetime('now')), + updated_at DATETIME NOT NULL DEFAULT (datetime('now')) + ); + `) + if err != nil { + return err + } + return nil } diff --git a/scripts/felhom-host-install.sh b/scripts/felhom-host-install.sh index 123828c..00ee61e 100644 --- a/scripts/felhom-host-install.sh +++ b/scripts/felhom-host-install.sh @@ -153,6 +153,7 @@ UNINSTALL=false # --uninstall: local host teardown (destroy guest + rem REMOVE_GOLDEN=false # --remove-golden: also delete the golden vzdump during --uninstall ADOPT_POOL=false # --adopt-pool: retrofit an EXISTING Felhom guest into the felhom pool (non-destructive) RESCOPE_ACL=false # --rescope-acl: migrate an existing install from the broad-/ token to the scoped ACL +ROTATE_RECOVERY=false # --rotate-recovery: regenerate + re-vault the break-glass root@pam password (TASK G1) # --- Gitea (artifact source) + agent install model (BUNDLE slice) --- GITEA_BASE="https://gitea.dooplex.hu" @@ -582,6 +583,23 @@ run_uninstall() { if [[ -f "$agent_cfg" ]]; then run rm -f "$agent_cfg"; else log_skip " $agent_cfg already absent"; fi run rmdir "$(dirname "$agent_cfg")" 2>/dev/null || true + # 4b2. Management-plane break-glass (TASK G1): timer+oneshot+script+tmpfiles. Stop/disable the + # timer, remove all four artifacts + the runtime heal-marker. We do NOT `rmdir /run/sshd` — + # the stock sshd needs it; leaving the (now unit-less) dir in place is correct (a bare kernel + # /run tmpfs recreates it empty on next boot anyway). Tolerate-absent throughout. + if systemctl list-unit-files felhom-mgmt-watchdog.timer >/dev/null 2>&1; then + systemctl is-active --quiet felhom-mgmt-watchdog.timer 2>/dev/null && run systemctl stop felhom-mgmt-watchdog.timer + systemctl is-enabled --quiet felhom-mgmt-watchdog.timer 2>/dev/null && run systemctl disable felhom-mgmt-watchdog.timer + else + log_skip " felhom-mgmt-watchdog.timer not loaded — skip stop/disable" + fi + run systemctl reset-failed felhom-mgmt-watchdog.service 2>/dev/null || true + local wda + for wda in /etc/systemd/system/felhom-mgmt-watchdog.service /etc/systemd/system/felhom-mgmt-watchdog.timer \ + /usr/local/sbin/felhom-mgmt-watchdog /etc/tmpfiles.d/felhom-privsep.conf /run/felhom-mgmt-watchdog.healed; do + if [[ -e "$wda" ]]; then run rm -f "$wda"; fi + done + # 4c. Shared-parent unit + wrapper + /mnt/felhom-drives (agent-installed at runtime; drill R2). # Stop/disable, remove unit + script, unbind + remove the (empty) parent dir. Tolerate-absent. if systemctl list-unit-files felhom-shared-parent.service 2>/dev/null | grep -q felhom-shared-parent; then @@ -782,6 +800,7 @@ while [[ $# -gt 0 ]]; do --remove-golden) REMOVE_GOLDEN=true; shift ;; --adopt-pool) ADOPT_POOL=true; shift ;; --rescope-acl) RESCOPE_ACL=true; shift ;; + --rotate-recovery) ROTATE_RECOVERY=true; shift ;; --acl-storages) read -ra PVE_STORAGES <<< "$2"; shift 2 ;; --dry-run) DRY_RUN=true; shift ;; --resume) RESUME=true; shift ;; @@ -1141,6 +1160,54 @@ step_enroll() { _state_mark enroll } +#------------------------------------------------------------------------------- +# STEP 4b — break-glass credential (TASK G1): generate + set + vault the root@pam console password +#------------------------------------------------------------------------------- +# The human fallback for when BOTH the sshd path AND the agent-independent auto-heal (layers 1+2) have +# failed: a strong root@pam password lets the operator reach the PVE WEB CONSOLE (pveproxy :8006 — a +# failure domain distinct from sshd) and run the one-line /run/sshd fix. Generated with strong entropy, +# set via chpasswd, and vaulted to the hub over the enroll-authenticated channel (host api_key). The +# password is NEVER logged, printed, or written to any file — it goes stdin→chpasswd and stdin→curl +# only (SPIKE-felhom-sshd finding #9 / TASK G1 trap 3). Idempotent: skipped if already vaulted unless +# --rotate-recovery (a re-set would strand the operator's saved copy). +step_break_glass() { + log_step "4b/8 break-glass credential (root@pam console password → hub vault)" + if [[ -z "${HOST_ID:-}" || -z "${HOST_API_KEY:-}" ]]; then + log_warn " no host_id/api_key (enroll skipped?) — cannot vault a recovery credential; skipping" + return 0 + fi + if $DRY_RUN; then + log_dry "openssl rand → strong root@pam password (never logged) ; chpasswd ; PUT $HUB_URL/api/v1/hosts/$HOST_ID/recovery-credential (Bearer host key)" + _state_mark break_glass; return 0 + fi + if _state_has break_glass && ! $ROTATE_RECOVERY; then + log_skip " recovery credential already vaulted (use --rotate-recovery to regenerate)" + return 0 + fi + # Strong password: 24 url-safe bytes (~144 bits). Kept ONLY in a local shell var, never on disk. + local newpw + newpw=$(openssl rand -base64 24 2>/dev/null | tr -d '\n' | tr '+/' '-_') + [[ ${#newpw} -ge 24 ]] || die "failed to generate a strong recovery password" + # Set root@pam (= the Linux root user on PVE) via chpasswd on STDIN — no argv, no log. + if ! printf 'root:%s\n' "$newpw" | chpasswd 2>/dev/null; then + newpw="" # scrub + die "chpasswd failed to set the root@pam recovery password" + fi + # Vault to the hub over the host-key-authenticated channel; password only on stdin (-d @-). + local code + code=$(printf '{"username":"root@pam","password":"%s"}' "$newpw" \ + | curl -sS -o /dev/null -w '%{http_code}' -X PUT \ + "$HUB_URL/api/v1/hosts/$HOST_ID/recovery-credential" \ + -H "Authorization: Bearer $HOST_API_KEY" -H 'Content-Type: application/json' -d @- 2>/dev/null) + newpw="" # scrub the plaintext from the shell var the moment it is vaulted + case "$code" in + 200) log_success " root@pam password set + vaulted to the hub (retrieve via the operator /admin path; never logged here)" ;; + 401|403) die "recovery-credential vault rejected ($code) — host key/authorization problem" ;; + *) die "recovery-credential vault failed (HTTP $code)" ;; + esac + _state_mark break_glass +} + #------------------------------------------------------------------------------- # STEP 5 — agent install: fetch+verify the binary, ensure the service user, sudoers, unit #------------------------------------------------------------------------------- @@ -1331,9 +1398,63 @@ step_agent_install() { fi rm -f "$rbtmp" fi + + # Management-plane break-glass layers 1+2 (TASK G1). Three artifacts that keep the host reachable + # even if a second sshd (H1) removes the SHARED /run/sshd privsep dir (SPIKE-felhom-sshd §8): + # • felhom-privsep.conf (tmpfiles) — layer 1: /run/sshd is boot-persistent, owned by no unit. + # • felhom-mgmt-watchdog (script) — layer 2 heal action (recreate dir + reset-failed sshd). + # • .service + .timer — run it every ~60s, AGENT-INDEPENDENTLY (heals with the + # agent down — the whole point; trap 1). + # Non-fatal if the agent repo predates them (raw fetch 404s → break-glass just stays manual). + # HARD GUARD: refuse ANY fetched unit that declares RuntimeDirectory= — that directive is the very + # incident G1 closes (a second sshd's `RuntimeDirectory=sshd` removed the shared /run/sshd). + install_mgmt_watchdog + _state_mark agent_install } +# install_mgmt_watchdog fetches + installs the G1 break-glass host artifacts (idempotent; enables the +# timer). Split out for readability; called from step_agent_install. Every unit is RuntimeDirectory- +# guarded (trap 2). Non-fatal on a repo that predates the artifacts. +install_mgmt_watchdog() { + if $DRY_RUN; then + log_dry "fetch configs/felhom-privsep.tmpfiles -> /etc/tmpfiles.d/felhom-privsep.conf ; systemd-tmpfiles --create" + log_dry "fetch configs/felhom-mgmt-watchdog.sh -> /usr/local/sbin/felhom-mgmt-watchdog (0755)" + log_dry "fetch configs/felhom-mgmt-watchdog.{service,timer} -> /etc/systemd/system/ ; enable --now felhom-mgmt-watchdog.timer" + return 0 + fi + local wdtmp; wdtmp=$(mktemp -t felhom-mgmt-wd.XXXXXX) + if ! fetch_raw "configs/felhom-mgmt-watchdog.sh" "$wdtmp" 2>/dev/null; then + log_skip " mgmt-watchdog artifacts not in the agent repo yet — break-glass auto-heal stays manual" + rm -f "$wdtmp"; return 0 + fi + sh -n "$wdtmp" || { rm -f "$wdtmp"; die "fetched felhom-mgmt-watchdog.sh failed sh -n — refusing to install"; } + install -m 0755 -o root -g root "$wdtmp" /usr/local/sbin/felhom-mgmt-watchdog + rm -f "$wdtmp" + + # tmpfiles (layer 1) — install + create now so /run/sshd is guaranteed present immediately. + local tftmp; tftmp=$(mktemp -t felhom-privsep.XXXXXX) + fetch_raw "configs/felhom-privsep.tmpfiles" "$tftmp" + install -m 0644 -o root -g root "$tftmp" /etc/tmpfiles.d/felhom-privsep.conf + rm -f "$tftmp" + systemd-tmpfiles --create /etc/tmpfiles.d/felhom-privsep.conf 2>/dev/null || true + + # units (layer 2) — RuntimeDirectory guard on BOTH before install (trap 2 / the incident cause). + local svctmp tmrtmp; svctmp=$(mktemp -t felhom-wd-svc.XXXXXX); tmrtmp=$(mktemp -t felhom-wd-tmr.XXXXXX) + fetch_raw "configs/felhom-mgmt-watchdog.service" "$svctmp" + fetch_raw "configs/felhom-mgmt-watchdog.timer" "$tmrtmp" + if grep -qiE '^[[:space:]]*RuntimeDirectory[[:space:]]*=' "$svctmp" "$tmrtmp"; then + rm -f "$svctmp" "$tmrtmp" + die "mgmt-watchdog unit declares RuntimeDirectory= — that is the incident G1 fixes; refusing to install" + fi + install -m 0644 -o root -g root "$svctmp" /etc/systemd/system/felhom-mgmt-watchdog.service + install -m 0644 -o root -g root "$tmrtmp" /etc/systemd/system/felhom-mgmt-watchdog.timer + rm -f "$svctmp" "$tmrtmp" + systemctl daemon-reload + systemctl enable --now felhom-mgmt-watchdog.timer >/dev/null 2>&1 || true + log_success " installed break-glass layers 1+2 (tmpfiles /run/sshd + agent-independent watchdog timer)" +} + #------------------------------------------------------------------------------- # STEP 6 — write agent config + ensure service healthy #------------------------------------------------------------------------------- @@ -1611,6 +1732,7 @@ fi should_skip token || step_token should_skip grows || step_grows should_skip enroll || step_enroll +should_skip break_glass || step_break_glass should_skip agent_install || step_agent_install should_skip agent_config || step_agent_config should_skip golden || step_golden