feat(hub,install): break-glass recovery vault + mgmt_plane surfacing (TASK G1)

Hub half of the management-plane break-glass (prereq for felhom-sshd/H1; agent half = felhom-agent v0.71.0). Closes SPIKE-felhom-sshd §8/#9. - store.host_recovery + methods: per-host root@pam console password, at-rest, operator-retrievable (the PVE-web-console fallback when sshd + auto-heal both fail). - API: PUT /hosts/{id}/recovery-credential (self-scoped, day-0 vaults) + GET /admin/hosts/{id}/recovery-credential (global key only). Secret never logged (red-proofed). - monitor/host_mgmtplane: parses the agent mgmt_plane stanza, raises mgmt_plane_healed WARNING on a new privsep_healed_at (recurring clobber surfaces before lockout; complements host_staleness). - host-install: step_break_glass generates a strong root@pam password (openssl rand, never logged/filed — stdin to chpasswd + curl), vaults via host key; idempotent unless --rotate-recovery. Installs the G1 host artifacts (tmpfiles + agent-independent watchdog timer), RuntimeDirectory-guarded; uninstall removes them. Hub v0.34.0. Non-hollow tests + red-proofs; full suite green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
2026-07-05 19:03:18 +02:00
parent 2f97ce31dd
commit 05d81810d4
10 changed files with 739 additions and 0 deletions
@@ -0,0 +1,122 @@
+package monitor
+
+import (
+	"encoding/json"
+	"log"
+	"sync"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// HostMgmtPlaneChecker raises an operator WARNING when a host's agent-independent break-glass watchdog
+// AUTO-HEALED a missing /run/sshd privsep dir (TASK G1). The heal itself is silent and login-free (the
+// point of the watchdog); this surfaces a RECURRING clobber so the operator can find the cause BEFORE
+// it becomes a full management lockout — complementing HostStalenessChecker (which only catches a box
+// gone silent). Sibling of HostLeafChecker; runs on the same 60s sweep.
+//
+// Design (mirrors HostLeafChecker's trust-on-first-report): the state is the host's last-seen
+// privsep_healed_at marker timestamp. The watchdog rewrites the marker on EACH heal, so a new, different
+// timestamp = a new heal event → one warning. The first observation of a non-empty timestamp seeds the
+// baseline WITHOUT alerting (it may be a heal from before the hub was watching — avoid a false alarm on
+// startup; a genuinely recurring cause re-heals and re-alerts on the next occurrence). An empty
+// timestamp (healthy host / old agent) never alerts and never overwrites a baseline.
+type HostMgmtPlaneChecker struct {
+	store   *store.Store
+	logger  *log.Logger
+	onEvent EventNotifyFunc
+
+	mu         sync.Mutex
+	states     map[string]string // hostID → last-seen privsep_healed_at
+	customerOf map[string]string
+}
+
+// NewHostMgmtPlaneChecker seeds per-host baselines from the latest reports. No events on init.
+func NewHostMgmtPlaneChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostMgmtPlaneChecker {
+	mc := &HostMgmtPlaneChecker{
+		store:      s,
+		logger:     logger,
+		onEvent:    onEvent,
+		states:     make(map[string]string),
+		customerOf: make(map[string]string),
+	}
+	rows, err := s.GetHostMgmtPlaneStates()
+	if err != nil {
+		logger.Printf("[WARN] Host mgmt-plane checker: failed to seed: %v", err)
+		return mc
+	}
+	seeded := 0
+	for _, row := range rows {
+		if s.IsCustomerBlocked(row.CustomerID) || row.PrivsepHealedAt == "" {
+			continue
+		}
+		mc.customerOf[row.HostID] = row.CustomerID
+		mc.states[row.HostID] = row.PrivsepHealedAt
+		seeded++
+	}
+	logger.Printf("[INFO] Host mgmt-plane checker initialized: %d host heal-state(s) seeded", seeded)
+	return mc
+}
+
+// Check evaluates all hosts and emits mgmt_plane_healed on a NEW heal timestamp.
+func (mc *HostMgmtPlaneChecker) Check() {
+	rows, err := mc.store.GetHostMgmtPlaneStates()
+	if err != nil {
+		mc.logger.Printf("[WARN] Host mgmt-plane check failed: %v", err)
+		return
+	}
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+
+	seen := make(map[string]bool, len(rows))
+	for _, row := range rows {
+		if mc.store.IsCustomerBlocked(row.CustomerID) {
+			delete(mc.states, row.HostID)
+			continue
+		}
+		seen[row.HostID] = true
+		if row.PrivsepHealedAt == "" {
+			continue // no heal marker → healthy / old agent → no alert, no baseline change
+		}
+		mc.customerOf[row.HostID] = row.CustomerID
+		old := mc.states[row.HostID]
+		if old == "" {
+			mc.states[row.HostID] = row.PrivsepHealedAt // first observation → seed, no event
+			continue
+		}
+		if old == row.PrivsepHealedAt {
+			continue // same heal already alerted
+		}
+		mc.states[row.HostID] = row.PrivsepHealedAt
+		mc.emit(row.HostID, row.CustomerID, row.PrivsepHealedAt)
+	}
+
+	for id := range mc.states {
+		if !seen[id] {
+			delete(mc.states, id)
+		}
+	}
+}
+
+// GetState returns the last-seen heal timestamp for a host ("" if none).
+func (mc *HostMgmtPlaneChecker) GetState(hostID string) string {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	return mc.states[hostID]
+}
+
+func (mc *HostMgmtPlaneChecker) emit(hostID, customerID, healedAt string) {
+	msg := "Host " + hostID + ": the management-plane privsep dir (/run/sshd) was missing and was AUTO-HEALED by the watchdog at " + healedAt +
+		" — a recurring cause can lead to an SSH lockout; investigate (e.g. a unit declaring RuntimeDirectory=sshd)."
+	details, _ := json.Marshal(map[string]string{
+		"host_id":           hostID,
+		"privsep_healed_at": healedAt,
+	})
+	mc.logger.Printf("[WARN] Host mgmt-plane: %s privsep dir auto-healed at %s (mgmt_plane_healed)", hostID, healedAt)
+	if _, err := mc.store.SaveEvent(customerID, "mgmt_plane_healed", "warning", msg, string(details), "hub"); err != nil {
+		mc.logger.Printf("[WARN] save mgmt_plane_healed for %s: %v", hostID, err)
+		return
+	}
+	if mc.onEvent != nil {
+		mc.onEvent(customerID, "mgmt_plane_healed", "warning", msg, string(details), "hub")
+	}
+}
@@ -0,0 +1,90 @@
+package monitor
+
+import (
+	"io"
+	"log"
+	"path/filepath"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+	_ "modernc.org/sqlite"
+)
+
+func reportWithHeal(healedAt string) []byte {
+	if healedAt == "" {
+		return []byte(`{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true}}`)
+	}
+	return []byte(`{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true,"healed_recently":true,"privsep_healed_at":"` + healedAt + `"}}`)
+}
+
+func newMgmtStore(t *testing.T) *store.Store {
+	t.Helper()
+	st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { st.Close() })
+	st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"})
+	st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"})
+	return st
+}
+
+// A NEW heal timestamp fires exactly one mgmt_plane_healed; the first observation only seeds; a repeat
+// of the same timestamp does not re-fire. Companion TestHostMgmtPlaneChecker_NoHealNoEvent proves it's
+// the heal, not the sweep, that fires it (drop the emit → this test fails).
+func TestHostMgmtPlaneChecker_NewHealAlertsOnce(t *testing.T) {
+	st := newMgmtStore(t)
+	// first report already carries a heal marker → seed baseline, NO event on construction.
+	st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T16:42:17Z"), store.HostReportDenorm{})
+	var events []string
+	mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	if mc.GetState("h1") != "2026-07-05T16:42:17Z" {
+		t.Fatalf("seed = %q", mc.GetState("h1"))
+	}
+	if len(events) != 0 {
+		t.Fatalf("construction must not emit, got %v", events)
+	}
+
+	// a NEW heal (different timestamp) → one warning.
+	st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T18:00:00Z"), store.HostReportDenorm{})
+	mc.Check()
+	if len(events) != 1 || events[0] != "mgmt_plane_healed" {
+		t.Fatalf("new heal → one mgmt_plane_healed, got %v", events)
+	}
+	// same timestamp again → no duplicate.
+	mc.Check()
+	if len(events) != 1 {
+		t.Fatalf("same heal must not re-emit, got %v", events)
+	}
+}
+
+func TestHostMgmtPlaneChecker_NoHealNoEvent(t *testing.T) {
+	st := newMgmtStore(t)
+	st.SaveHostReport("h1", "c1", reportWithHeal(""), store.HostReportDenorm{}) // healthy, no marker
+	var events []string
+	mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	mc.Check()
+	mc.Check()
+	if len(events) != 0 {
+		t.Fatalf("a healthy host (no heal marker) must never alert, got %v", events)
+	}
+	if mc.GetState("h1") != "" {
+		t.Fatalf("no marker → no baseline, got %q", mc.GetState("h1"))
+	}
+}
+
+// A recurring clobber: heal at T1 (seed), heal again at T2 (alert), heal again at T3 (alert) — each
+// distinct heal surfaces, which is the whole point (find the recurring cause before a lockout).
+func TestHostMgmtPlaneChecker_RecurringHealsEachAlert(t *testing.T) {
+	st := newMgmtStore(t)
+	st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T10:00:00Z"), store.HostReportDenorm{})
+	var events []string
+	mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T11:00:00Z"), store.HostReportDenorm{})
+	mc.Check()
+	st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T12:00:00Z"), store.HostReportDenorm{})
+	mc.Check()
+	if len(events) != 2 {
+		t.Fatalf("two distinct new heals after seed → two alerts, got %d (%v)", len(events), events)
+	}
+}