feat(hub,install): break-glass recovery vault + mgmt_plane surfacing (TASK G1)

Hub half of the management-plane break-glass (prereq for felhom-sshd/H1; agent half = felhom-agent v0.71.0). Closes SPIKE-felhom-sshd §8/#9. - store.host_recovery + methods: per-host root@pam console password, at-rest, operator-retrievable (the PVE-web-console fallback when sshd + auto-heal both fail). - API: PUT /hosts/{id}/recovery-credential (self-scoped, day-0 vaults) + GET /admin/hosts/{id}/recovery-credential (global key only). Secret never logged (red-proofed). - monitor/host_mgmtplane: parses the agent mgmt_plane stanza, raises mgmt_plane_healed WARNING on a new privsep_healed_at (recurring clobber surfaces before lockout; complements host_staleness). - host-install: step_break_glass generates a strong root@pam password (openssl rand, never logged/filed — stdin to chpasswd + curl), vaults via host key; idempotent unless --rotate-recovery. Installs the G1 host artifacts (tmpfiles + agent-independent watchdog timer), RuntimeDirectory-guarded; uninstall removes them. Hub v0.34.0. Non-hollow tests + red-proofs; full suite green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
2026-07-05 19:03:18 +02:00
parent 2f97ce31dd
commit 05d81810d4
10 changed files with 739 additions and 0 deletions
@@ -0,0 +1,110 @@
+package store
+
+import (
+	"database/sql"
+	"encoding/json"
+	"time"
+)
+
+// HostRecoveryCredential is the break-glass PVE console credential for a host (TASK G1). Secret is
+// the root@pam password — a hub-held secret, operator-retrievable (NOT zero-knowledge like escrow).
+type HostRecoveryCredential struct {
+	HostID   string
+	Username string
+	Secret   string
+	SetAt    time.Time
+}
+
+// SaveHostRecoveryCredential upserts a host's break-glass credential (last-write-wins: day-0 sets it,
+// --rotate re-sets). The secret is stored as-is at rest; the hub NEVER logs it and only ever returns
+// it over the operator-authenticated retrieval path.
+func (s *Store) SaveHostRecoveryCredential(hostID, username, secret string) error {
+	_, err := s.db.Exec(`
+		INSERT INTO host_recovery (host_id, username, secret, set_at, updated_at)
+		VALUES (?, ?, ?, datetime('now'), datetime('now'))
+		ON CONFLICT(host_id) DO UPDATE SET
+			username   = excluded.username,
+			secret     = excluded.secret,
+			updated_at = datetime('now')`,
+		hostID, username, secret)
+	return err
+}
+
+// GetHostRecoveryCredential returns a host's break-glass credential, or (nil, nil) if none is vaulted.
+func (s *Store) GetHostRecoveryCredential(hostID string) (*HostRecoveryCredential, error) {
+	var c HostRecoveryCredential
+	var setAt string
+	err := s.db.QueryRow(
+		`SELECT host_id, username, secret, set_at FROM host_recovery WHERE host_id = ?`, hostID).
+		Scan(&c.HostID, &c.Username, &c.Secret, &setAt)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	c.SetAt = parseSQLiteTime(setAt)
+	return &c, nil
+}
+
+// HasHostRecoveryCredential reports whether a host already has a vaulted credential (day-0 idempotency:
+// don't regenerate/re-set on a re-run unless --rotate).
+func (s *Store) HasHostRecoveryCredential(hostID string) (bool, error) {
+	var one int
+	err := s.db.QueryRow(`SELECT 1 FROM host_recovery WHERE host_id = ?`, hostID).Scan(&one)
+	if err == sql.ErrNoRows {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// HostMgmtPlaneRow is the latest management-plane state per host (TASK G1), parsed from the newest
+// host_report. PrivsepHealedAt is the watchdog heal-marker timestamp ("" when never healed / old agent).
+type HostMgmtPlaneRow struct {
+	HostID          string
+	CustomerID      string
+	PrivsepDirOK    bool
+	SshdReachable   bool
+	PrivsepHealedAt string
+}
+
+// GetHostMgmtPlaneStates returns the latest mgmt_plane stanza per host (mirrors
+// GetHostLeafFingerprints). A report without the stanza (old agent, feature off) yields zero values →
+// no alert. Malformed JSON degrades to zero values, never an error for that host.
+func (s *Store) GetHostMgmtPlaneStates() ([]HostMgmtPlaneRow, error) {
+	rows, err := s.db.Query(`
+		SELECT hr.host_id, hr.customer_id, hr.report_json
+		FROM host_reports hr
+		JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest
+		  ON hr.id = latest.mx`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []HostMgmtPlaneRow
+	for rows.Next() {
+		var r HostMgmtPlaneRow
+		var reportJSON string
+		if err := rows.Scan(&r.HostID, &r.CustomerID, &reportJSON); err != nil {
+			return nil, err
+		}
+		var body struct {
+			MgmtPlane *struct {
+				PrivsepDirOK    bool   `json:"privsep_dir_ok"`
+				SshdReachable   bool   `json:"sshd_reachable"`
+				PrivsepHealedAt string `json:"privsep_healed_at"`
+			} `json:"mgmt_plane"`
+		}
+		_ = json.Unmarshal([]byte(reportJSON), &body) // malformed/old → nil mgmt_plane → zero values
+		if body.MgmtPlane != nil {
+			r.PrivsepDirOK = body.MgmtPlane.PrivsepDirOK
+			r.SshdReachable = body.MgmtPlane.SshdReachable
+			r.PrivsepHealedAt = body.MgmtPlane.PrivsepHealedAt
+		}
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}
@@ -0,0 +1,83 @@
+package store
+
+import "testing"
+
+func TestHostRecoveryCredential_RoundTripUpsertAndAbsent(t *testing.T) {
+	s := newTestStore(t)
+	if err := s.UpsertHost(&Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
+		t.Fatalf("UpsertHost: %v", err)
+	}
+
+	// absent → (nil, nil) + Has=false
+	got, err := s.GetHostRecoveryCredential("h1")
+	if err != nil || got != nil {
+		t.Fatalf("absent cred: got %+v / %v (want nil,nil)", got, err)
+	}
+	has, _ := s.HasHostRecoveryCredential("h1")
+	if has {
+		t.Fatal("HasHostRecoveryCredential must be false before any vault")
+	}
+
+	// vault → round-trips
+	if err := s.SaveHostRecoveryCredential("h1", "root@pam", "s3cret-Aa1"); err != nil {
+		t.Fatalf("SaveHostRecoveryCredential: %v", err)
+	}
+	got, err = s.GetHostRecoveryCredential("h1")
+	if err != nil || got == nil {
+		t.Fatalf("GetHostRecoveryCredential: %+v / %v", got, err)
+	}
+	if got.Username != "root@pam" || got.Secret != "s3cret-Aa1" {
+		t.Fatalf("round-trip mismatch: %+v", got)
+	}
+	if has, _ := s.HasHostRecoveryCredential("h1"); !has {
+		t.Fatal("HasHostRecoveryCredential must be true after vault")
+	}
+
+	// upsert (rotate) → overwrites last-write-wins
+	if err := s.SaveHostRecoveryCredential("h1", "root@pam", "rotated-Bb2"); err != nil {
+		t.Fatalf("re-vault: %v", err)
+	}
+	got, _ = s.GetHostRecoveryCredential("h1")
+	if got.Secret != "rotated-Bb2" {
+		t.Fatalf("rotate did not overwrite: %+v", got)
+	}
+}
+
+func TestGetHostMgmtPlaneStates_ParsesHealMarker(t *testing.T) {
+	s := newTestStore(t)
+	if err := s.UpsertHost(&Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
+		t.Fatalf("UpsertHost: %v", err)
+	}
+	// a report WITH a heal marker
+	report := `{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true,"healed_recently":true,"privsep_healed_at":"2026-07-05T16:42:17Z"}}`
+	if err := s.SaveHostReport("h1", "c1", []byte(report), HostReportDenorm{}); err != nil {
+		t.Fatalf("SaveHostReport: %v", err)
+	}
+	rows, err := s.GetHostMgmtPlaneStates()
+	if err != nil {
+		t.Fatalf("GetHostMgmtPlaneStates: %v", err)
+	}
+	var found bool
+	for _, r := range rows {
+		if r.HostID == "h1" {
+			found = true
+			if !r.PrivsepDirOK || r.PrivsepHealedAt != "2026-07-05T16:42:17Z" {
+				t.Fatalf("parsed row wrong: %+v", r)
+			}
+		}
+	}
+	if !found {
+		t.Fatal("h1 not in mgmt-plane states")
+	}
+
+	// a report WITHOUT the stanza (old agent) → zero values, no crash
+	if err := s.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), HostReportDenorm{}); err != nil {
+		t.Fatalf("SaveHostReport2: %v", err)
+	}
+	rows, _ = s.GetHostMgmtPlaneStates()
+	for _, r := range rows {
+		if r.HostID == "h1" && r.PrivsepHealedAt != "" {
+			t.Fatalf("old-agent report should yield empty healed_at, got %+v", r)
+		}
+	}
+}
@@ -398,6 +398,24 @@ func (s *Store) migrate() error {
 		return err
 	}

+	// host_recovery (TASK G1): the break-glass root@pam console credential, vaulted at rest and
+	// operator-retrievable. UNLIKE host_escrow (opaque, hub-can't-open), this IS a hub-held secret the
+	// operator retrieves to reach the PVE web console (pveproxy — a failure domain distinct from sshd)
+	// when both the sshd path AND the agent-independent auto-heal have failed. One row per host,
+	// last-write-wins (day-0 sets it; --rotate re-sets). Never in desired-state, never logged.
+	_, err = s.db.Exec(`
+		CREATE TABLE IF NOT EXISTS host_recovery (
+			host_id     TEXT PRIMARY KEY,
+			username    TEXT NOT NULL,
+			secret      TEXT NOT NULL,
+			set_at      DATETIME NOT NULL DEFAULT (datetime('now')),
+			updated_at  DATETIME NOT NULL DEFAULT (datetime('now'))
+		);
+	`)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }