feat(hub,install): break-glass recovery vault + mgmt_plane surfacing (TASK G1)

Hub half of the management-plane break-glass (prereq for felhom-sshd/H1; agent
half = felhom-agent v0.71.0). Closes SPIKE-felhom-sshd §8/#9.

- store.host_recovery + methods: per-host root@pam console password, at-rest,
  operator-retrievable (the PVE-web-console fallback when sshd + auto-heal both fail).
- API: PUT /hosts/{id}/recovery-credential (self-scoped, day-0 vaults) + GET
  /admin/hosts/{id}/recovery-credential (global key only). Secret never logged
  (red-proofed).
- monitor/host_mgmtplane: parses the agent mgmt_plane stanza, raises
  mgmt_plane_healed WARNING on a new privsep_healed_at (recurring clobber surfaces
  before lockout; complements host_staleness).
- host-install: step_break_glass generates a strong root@pam password (openssl
  rand, never logged/filed — stdin to chpasswd + curl), vaults via host key;
  idempotent unless --rotate-recovery. Installs the G1 host artifacts (tmpfiles +
  agent-independent watchdog timer), RuntimeDirectory-guarded; uninstall removes them.

Hub v0.34.0. Non-hollow tests + red-proofs; full suite green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6

hub/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Felhom Hub — Changelog
 
+## v0.34.0 — break-glass recovery vault + mgmt_plane surfacing (TASK G1) (2026-07-05)
+
+The hub half of the management-plane break-glass system (prerequisite for felhom-sshd / H1; agent half
+= felhom-agent v0.71.0). Closes the recovery gap from
+`documentation/audits/SPIKE-felhom-sshd-2026-07-05.md` §8/#9.
+
+- **Break-glass credential vault** (`store.host_recovery` + `internal/store/host_recovery.go`): a
+  per-host root@pam console password, stored at rest, operator-retrievable — the human fallback for
+  reaching the PVE web console (pveproxy, a failure domain distinct from sshd) when both the sshd path
+  and the agent-independent auto-heal have failed. `PUT /hosts/{id}/recovery-credential` (SELF-scoped
+  host key — day-0 vaults it) + `GET /admin/hosts/{id}/recovery-credential` (GLOBAL key only — a host
+  key cannot read its own console password back). Secret discipline: never logged (username + length
+  only); red-proofed that the password never reaches the hub log.
+- **mgmt_plane surfacing** (`internal/monitor/host_mgmtplane.go`, on the 60s sweep): parses the agent's
+  additive `mgmt_plane` heartbeat stanza and raises a `mgmt_plane_healed` WARNING when the watchdog
+  auto-healed a missing `/run/sshd` (new `privsep_healed_at`) — a recurring clobber surfaces BEFORE it
+  becomes a lockout, complementing host_staleness. Trust-on-first-report (seed, then alert on change),
+  mirroring HostLeafChecker.
+
 ## v0.33.0 — S2 offsite connectivity: box-facing WG registration + wireguard desired-state block + /offsite UI (2026-07-04)
 
 Doc 06 roadmap row S2 (commits `fcf84a0`/`ba52005`/`13203c2`); the S2 architectural decision:

hub/cmd/hub/main.go

@@ -378,6 +378,9 @@ func main() {
 	// target (dump/backup volume, data drive, lvmthin pool, PBS datastore). Born/persistent; excludes the
 	// root-backed builtin (hostDiskChecker owns root, no double-alert); emits natural `critical`. Same sweep.
 	storageFillChecker := monitor.NewStorageFillChecker(dataStore, cfg.Alerting.StorageFillWarnPercent, cfg.Alerting.StorageFillCritPercent, dispatcher.ProcessEvent, logger)
+	// TASK G1: warn when a host's agent-independent watchdog auto-healed a missing /run/sshd privsep
+	// dir — a recurring clobber that can lead to an SSH lockout (complements host_staleness). Same sweep.
+	hostMgmtPlaneChecker := monitor.NewHostMgmtPlaneChecker(dataStore, dispatcher.ProcessEvent, logger)
 	go func() {
 		ticker := time.NewTicker(60 * time.Second)
 		defer ticker.Stop()
@@ -392,6 +395,7 @@ func main() {
 				hostLeafChecker.Check()
 				hostDiskChecker.Check()
 				storageFillChecker.Check()
+				hostMgmtPlaneChecker.Check()
 			}
 		}
 	}()

hub/internal/api/handler.go

@@ -155,6 +155,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/escrow"):
 		hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/escrow")
 		h.handleHostEscrowPut(w, r, hostID)
+	// G1 break-glass: day-0 vaults the root@pam console credential (self-scoped host key); the
+	// operator retrieves it via the /admin/ path (global key only).
+	case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/recovery-credential"):
+		hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/recovery-credential")
+		h.handleHostRecoveryCredentialPut(w, r, hostID)
+	case r.Method == http.MethodGet && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-credential"):
+		hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/admin/hosts/"), "/recovery-credential")
+		h.handleAdminGetRecoveryCredential(w, r, hostID)
 	// DR capstone (slice 10D). Recovery-mode toggle (global key); re-enroll + restore-directive
 	// (gated on recovery mode — no old key needed, the box is lost).
 	case r.Method == http.MethodPut && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-mode"):
@@ -883,6 +891,95 @@ func (h *Handler) handleHostEscrowPut(w http.ResponseWriter, r *http.Request, pa
 	w.Write([]byte(`{"status":"ok"}`))
 }
 
+// handleHostRecoveryCredentialPut vaults a host's break-glass root@pam console credential (TASK G1).
+// SELF-SCOPED (a host key writes only its own; global may write any) — day-0 posts it with the
+// host api_key. The secret is stored at rest and NEVER logged (only the username + a length are
+// logged). This is the human fallback for when both the sshd path AND the agent-independent
+// auto-heal have failed: the operator retrieves it to reach the PVE web console (pveproxy — a
+// failure domain distinct from sshd).
+func (h *Handler) handleHostRecoveryCredentialPut(w http.ResponseWriter, r *http.Request, pathHostID string) {
+	authHostID, _, isGlobal, ok := h.checkAuthHost(r)
+	if !ok {
+		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		return
+	}
+	if pathHostID == "" {
+		http.Error(w, "Missing host_id", http.StatusBadRequest)
+		return
+	}
+	if !isGlobal && authHostID != pathHostID {
+		http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
+		return
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, 1<<16)) // 64 KiB cap; a username+password is tiny
+	if err != nil {
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+	var req struct {
+		Username string `json:"username"`
+		Password string `json:"password"`
+	}
+	if err := json.Unmarshal(body, &req); err != nil || req.Username == "" || req.Password == "" {
+		http.Error(w, "Invalid payload: username + password required", http.StatusBadRequest)
+		return
+	}
+	// The host must exist (mint-first) — a per-host key already proves it; the global path re-checks.
+	if isGlobal {
+		host, herr := h.store.GetHost(pathHostID)
+		if herr != nil || host == nil {
+			http.Error(w, "Unknown host_id", http.StatusBadRequest)
+			return
+		}
+	}
+	if err := h.store.SaveHostRecoveryCredential(pathHostID, req.Username, req.Password); err != nil {
+		h.logger.Printf("[ERROR] Failed to vault recovery credential for host %s: %v", pathHostID, err)
+		http.Error(w, "Internal error", http.StatusInternalServerError)
+		return
+	}
+	// SECRET DISCIPLINE: log the username + a length only — NEVER the password.
+	h.logger.Printf("[INFO] vaulted break-glass recovery credential for host %s (user=%s, secret %d chars)",
+		pathHostID, req.Username, len(req.Password))
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte(`{"status":"ok"}`))
+}
+
+// handleAdminGetRecoveryCredential returns a host's vaulted break-glass credential to the OPERATOR
+// (global key only — a per-host key must NOT read its own console password back out). This is the
+// authenticated retrieval path the break-glass runbook uses. The response body carries the secret by
+// necessity; it is never written to the hub log.
+func (h *Handler) handleAdminGetRecoveryCredential(w http.ResponseWriter, r *http.Request, pathHostID string) {
+	_, _, isGlobal, ok := h.checkAuthHost(r)
+	if !ok || !isGlobal {
+		http.Error(w, "Unauthorized", http.StatusUnauthorized) // operator/global key ONLY
+		return
+	}
+	if pathHostID == "" {
+		http.Error(w, "Missing host_id", http.StatusBadRequest)
+		return
+	}
+	cred, err := h.store.GetHostRecoveryCredential(pathHostID)
+	if err != nil {
+		h.logger.Printf("[ERROR] Failed to read recovery credential for host %s: %v", pathHostID, err)
+		http.Error(w, "Internal error", http.StatusInternalServerError)
+		return
+	}
+	if cred == nil {
+		http.Error(w, "No recovery credential vaulted for this host", http.StatusNotFound)
+		return
+	}
+	h.logger.Printf("[INFO] operator retrieved break-glass recovery credential for host %s (user=%s)", pathHostID, cred.Username)
+	resp, _ := json.Marshal(map[string]string{
+		"host_id":  cred.HostID,
+		"username": cred.Username,
+		"password": cred.Secret,
+		"set_at":   cred.SetAt.UTC().Format(time.RFC3339),
+	})
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	w.Write(resp)
+}
+
 // handleGetDesiredState serves a host its authoritative desired-state (slice 10A). Per-host key,
 // SELF-SCOPED: a host reads ONLY its own (the global operator key may read any). The agent fetches
 // this when the heartbeat envelope's desired_generation has advanced past its cached one. The

hub/internal/api/host_recovery_test.go

@@ -0,0 +1,74 @@
+package api
+
+import (
+	"bytes"
+	"log"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+	_ "modernc.org/sqlite"
+)
+
+func TestRecoveryCredential_VaultSelfScopedAndOperatorRetrieval(t *testing.T) {
+	h, st, _ := newTestHandler(t)
+	st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"})
+	st.UpsertHost(&store.Host{HostID: "h2", CustomerID: "c1", APIKey: "HKEY2"})
+
+	body := `{"username":"root@pam","password":"Str0ng-Break-Glass"}`
+
+	// self-scoped write: h1's key vaults h1 → 200
+	if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "HKEY", body); rr.Code != 200 {
+		t.Fatalf("self vault = %d: %s", rr.Code, rr.Body.String())
+	}
+	// a host key CANNOT vault ANOTHER host → 403
+	if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "HKEY2", body); rr.Code != 403 {
+		t.Fatalf("cross-host vault must be 403, got %d", rr.Code)
+	}
+	// unauthenticated → 401
+	if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "", body); rr.Code != 401 {
+		t.Fatalf("unauth vault must be 401, got %d", rr.Code)
+	}
+
+	// operator retrieval requires the GLOBAL key; a host key is refused (401 — can't read its own back)
+	if rr := do(h, "GET", "/admin/hosts/h1/recovery-credential", "HKEY", ""); rr.Code != 401 {
+		t.Fatalf("host key reading recovery credential must be 401, got %d", rr.Code)
+	}
+	rr := do(h, "GET", "/admin/hosts/h1/recovery-credential", globalKey, "")
+	if rr.Code != 200 {
+		t.Fatalf("operator retrieval = %d: %s", rr.Code, rr.Body.String())
+	}
+	if !strings.Contains(rr.Body.String(), "Str0ng-Break-Glass") || !strings.Contains(rr.Body.String(), "root@pam") {
+		t.Fatalf("retrieval must return the vaulted credential, got %s", rr.Body.String())
+	}
+	// a host with no credential → 404
+	if rr := do(h, "GET", "/admin/hosts/h2/recovery-credential", globalKey, ""); rr.Code != 404 {
+		t.Fatalf("no-credential host must be 404, got %d", rr.Code)
+	}
+}
+
+// SECRET DISCIPLINE (red-proof for trap 3): the password must NEVER appear in the hub log — on the
+// vault write NOR the operator retrieval. Companion: if a handler logged the password, this fails.
+func TestRecoveryCredential_PasswordNeverLogged(t *testing.T) {
+	var buf bytes.Buffer
+	path := filepath.Join(t.TempDir(), "test.db")
+	st, err := store.New(path, log.New(&buf, "", 0))
+	if err != nil {
+		t.Fatalf("store.New: %v", err)
+	}
+	defer st.Close()
+	h := New(st, globalKey, "", "", nil, log.New(&buf, "", 0))
+	st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"})
+
+	const secret = "SuperSecret-DoNotLog-42"
+	if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "HKEY", `{"username":"root@pam","password":"`+secret+`"}`); rr.Code != 200 {
+		t.Fatalf("vault = %d", rr.Code)
+	}
+	if rr := do(h, "GET", "/admin/hosts/h1/recovery-credential", globalKey, ""); rr.Code != 200 {
+		t.Fatalf("retrieve = %d", rr.Code)
+	}
+	if strings.Contains(buf.String(), secret) {
+		t.Fatalf("the recovery password LEAKED into the hub log:\n%s", buf.String())
+	}
+}

hub/internal/monitor/host_mgmtplane.go

@@ -0,0 +1,122 @@
+package monitor
+
+import (
+	"encoding/json"
+	"log"
+	"sync"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// HostMgmtPlaneChecker raises an operator WARNING when a host's agent-independent break-glass watchdog
+// AUTO-HEALED a missing /run/sshd privsep dir (TASK G1). The heal itself is silent and login-free (the
+// point of the watchdog); this surfaces a RECURRING clobber so the operator can find the cause BEFORE
+// it becomes a full management lockout — complementing HostStalenessChecker (which only catches a box
+// gone silent). Sibling of HostLeafChecker; runs on the same 60s sweep.
+//
+// Design (mirrors HostLeafChecker's trust-on-first-report): the state is the host's last-seen
+// privsep_healed_at marker timestamp. The watchdog rewrites the marker on EACH heal, so a new, different
+// timestamp = a new heal event → one warning. The first observation of a non-empty timestamp seeds the
+// baseline WITHOUT alerting (it may be a heal from before the hub was watching — avoid a false alarm on
+// startup; a genuinely recurring cause re-heals and re-alerts on the next occurrence). An empty
+// timestamp (healthy host / old agent) never alerts and never overwrites a baseline.
+type HostMgmtPlaneChecker struct {
+	store   *store.Store
+	logger  *log.Logger
+	onEvent EventNotifyFunc
+
+	mu         sync.Mutex
+	states     map[string]string // hostID → last-seen privsep_healed_at
+	customerOf map[string]string
+}
+
+// NewHostMgmtPlaneChecker seeds per-host baselines from the latest reports. No events on init.
+func NewHostMgmtPlaneChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostMgmtPlaneChecker {
+	mc := &HostMgmtPlaneChecker{
+		store:      s,
+		logger:     logger,
+		onEvent:    onEvent,
+		states:     make(map[string]string),
+		customerOf: make(map[string]string),
+	}
+	rows, err := s.GetHostMgmtPlaneStates()
+	if err != nil {
+		logger.Printf("[WARN] Host mgmt-plane checker: failed to seed: %v", err)
+		return mc
+	}
+	seeded := 0
+	for _, row := range rows {
+		if s.IsCustomerBlocked(row.CustomerID) || row.PrivsepHealedAt == "" {
+			continue
+		}
+		mc.customerOf[row.HostID] = row.CustomerID
+		mc.states[row.HostID] = row.PrivsepHealedAt
+		seeded++
+	}
+	logger.Printf("[INFO] Host mgmt-plane checker initialized: %d host heal-state(s) seeded", seeded)
+	return mc
+}
+
+// Check evaluates all hosts and emits mgmt_plane_healed on a NEW heal timestamp.
+func (mc *HostMgmtPlaneChecker) Check() {
+	rows, err := mc.store.GetHostMgmtPlaneStates()
+	if err != nil {
+		mc.logger.Printf("[WARN] Host mgmt-plane check failed: %v", err)
+		return
+	}
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+
+	seen := make(map[string]bool, len(rows))
+	for _, row := range rows {
+		if mc.store.IsCustomerBlocked(row.CustomerID) {
+			delete(mc.states, row.HostID)
+			continue
+		}
+		seen[row.HostID] = true
+		if row.PrivsepHealedAt == "" {
+			continue // no heal marker → healthy / old agent → no alert, no baseline change
+		}
+		mc.customerOf[row.HostID] = row.CustomerID
+		old := mc.states[row.HostID]
+		if old == "" {
+			mc.states[row.HostID] = row.PrivsepHealedAt // first observation → seed, no event
+			continue
+		}
+		if old == row.PrivsepHealedAt {
+			continue // same heal already alerted
+		}
+		mc.states[row.HostID] = row.PrivsepHealedAt
+		mc.emit(row.HostID, row.CustomerID, row.PrivsepHealedAt)
+	}
+
+	for id := range mc.states {
+		if !seen[id] {
+			delete(mc.states, id)
+		}
+	}
+}
+
+// GetState returns the last-seen heal timestamp for a host ("" if none).
+func (mc *HostMgmtPlaneChecker) GetState(hostID string) string {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	return mc.states[hostID]
+}
+
+func (mc *HostMgmtPlaneChecker) emit(hostID, customerID, healedAt string) {
+	msg := "Host " + hostID + ": the management-plane privsep dir (/run/sshd) was missing and was AUTO-HEALED by the watchdog at " + healedAt +
+		" — a recurring cause can lead to an SSH lockout; investigate (e.g. a unit declaring RuntimeDirectory=sshd)."
+	details, _ := json.Marshal(map[string]string{
+		"host_id":           hostID,
+		"privsep_healed_at": healedAt,
+	})
+	mc.logger.Printf("[WARN] Host mgmt-plane: %s privsep dir auto-healed at %s (mgmt_plane_healed)", hostID, healedAt)
+	if _, err := mc.store.SaveEvent(customerID, "mgmt_plane_healed", "warning", msg, string(details), "hub"); err != nil {
+		mc.logger.Printf("[WARN] save mgmt_plane_healed for %s: %v", hostID, err)
+		return
+	}
+	if mc.onEvent != nil {
+		mc.onEvent(customerID, "mgmt_plane_healed", "warning", msg, string(details), "hub")
+	}
+}

hub/internal/monitor/host_mgmtplane_test.go

@@ -0,0 +1,90 @@
+package monitor
+
+import (
+	"io"
+	"log"
+	"path/filepath"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+	_ "modernc.org/sqlite"
+)
+
+func reportWithHeal(healedAt string) []byte {
+	if healedAt == "" {
+		return []byte(`{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true}}`)
+	}
+	return []byte(`{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true,"healed_recently":true,"privsep_healed_at":"` + healedAt + `"}}`)
+}
+
+func newMgmtStore(t *testing.T) *store.Store {
+	t.Helper()
+	st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { st.Close() })
+	st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"})
+	st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"})
+	return st
+}
+
+// A NEW heal timestamp fires exactly one mgmt_plane_healed; the first observation only seeds; a repeat
+// of the same timestamp does not re-fire. Companion TestHostMgmtPlaneChecker_NoHealNoEvent proves it's
+// the heal, not the sweep, that fires it (drop the emit → this test fails).
+func TestHostMgmtPlaneChecker_NewHealAlertsOnce(t *testing.T) {
+	st := newMgmtStore(t)
+	// first report already carries a heal marker → seed baseline, NO event on construction.
+	st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T16:42:17Z"), store.HostReportDenorm{})
+	var events []string
+	mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	if mc.GetState("h1") != "2026-07-05T16:42:17Z" {
+		t.Fatalf("seed = %q", mc.GetState("h1"))
+	}
+	if len(events) != 0 {
+		t.Fatalf("construction must not emit, got %v", events)
+	}
+
+	// a NEW heal (different timestamp) → one warning.
+	st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T18:00:00Z"), store.HostReportDenorm{})
+	mc.Check()
+	if len(events) != 1 || events[0] != "mgmt_plane_healed" {
+		t.Fatalf("new heal → one mgmt_plane_healed, got %v", events)
+	}
+	// same timestamp again → no duplicate.
+	mc.Check()
+	if len(events) != 1 {
+		t.Fatalf("same heal must not re-emit, got %v", events)
+	}
+}
+
+func TestHostMgmtPlaneChecker_NoHealNoEvent(t *testing.T) {
+	st := newMgmtStore(t)
+	st.SaveHostReport("h1", "c1", reportWithHeal(""), store.HostReportDenorm{}) // healthy, no marker
+	var events []string
+	mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	mc.Check()
+	mc.Check()
+	if len(events) != 0 {
+		t.Fatalf("a healthy host (no heal marker) must never alert, got %v", events)
+	}
+	if mc.GetState("h1") != "" {
+		t.Fatalf("no marker → no baseline, got %q", mc.GetState("h1"))
+	}
+}
+
+// A recurring clobber: heal at T1 (seed), heal again at T2 (alert), heal again at T3 (alert) — each
+// distinct heal surfaces, which is the whole point (find the recurring cause before a lockout).
+func TestHostMgmtPlaneChecker_RecurringHealsEachAlert(t *testing.T) {
+	st := newMgmtStore(t)
+	st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T10:00:00Z"), store.HostReportDenorm{})
+	var events []string
+	mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
+	st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T11:00:00Z"), store.HostReportDenorm{})
+	mc.Check()
+	st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T12:00:00Z"), store.HostReportDenorm{})
+	mc.Check()
+	if len(events) != 2 {
+		t.Fatalf("two distinct new heals after seed → two alerts, got %d (%v)", len(events), events)
+	}
+}

hub/internal/store/host_recovery.go

@@ -0,0 +1,110 @@
+package store
+
+import (
+	"database/sql"
+	"encoding/json"
+	"time"
+)
+
+// HostRecoveryCredential is the break-glass PVE console credential for a host (TASK G1). Secret is
+// the root@pam password — a hub-held secret, operator-retrievable (NOT zero-knowledge like escrow).
+type HostRecoveryCredential struct {
+	HostID   string
+	Username string
+	Secret   string
+	SetAt    time.Time
+}
+
+// SaveHostRecoveryCredential upserts a host's break-glass credential (last-write-wins: day-0 sets it,
+// --rotate re-sets). The secret is stored as-is at rest; the hub NEVER logs it and only ever returns
+// it over the operator-authenticated retrieval path.
+func (s *Store) SaveHostRecoveryCredential(hostID, username, secret string) error {
+	_, err := s.db.Exec(`
+		INSERT INTO host_recovery (host_id, username, secret, set_at, updated_at)
+		VALUES (?, ?, ?, datetime('now'), datetime('now'))
+		ON CONFLICT(host_id) DO UPDATE SET
+			username   = excluded.username,
+			secret     = excluded.secret,
+			updated_at = datetime('now')`,
+		hostID, username, secret)
+	return err
+}
+
+// GetHostRecoveryCredential returns a host's break-glass credential, or (nil, nil) if none is vaulted.
+func (s *Store) GetHostRecoveryCredential(hostID string) (*HostRecoveryCredential, error) {
+	var c HostRecoveryCredential
+	var setAt string
+	err := s.db.QueryRow(
+		`SELECT host_id, username, secret, set_at FROM host_recovery WHERE host_id = ?`, hostID).
+		Scan(&c.HostID, &c.Username, &c.Secret, &setAt)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	c.SetAt = parseSQLiteTime(setAt)
+	return &c, nil
+}
+
+// HasHostRecoveryCredential reports whether a host already has a vaulted credential (day-0 idempotency:
+// don't regenerate/re-set on a re-run unless --rotate).
+func (s *Store) HasHostRecoveryCredential(hostID string) (bool, error) {
+	var one int
+	err := s.db.QueryRow(`SELECT 1 FROM host_recovery WHERE host_id = ?`, hostID).Scan(&one)
+	if err == sql.ErrNoRows {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// HostMgmtPlaneRow is the latest management-plane state per host (TASK G1), parsed from the newest
+// host_report. PrivsepHealedAt is the watchdog heal-marker timestamp ("" when never healed / old agent).
+type HostMgmtPlaneRow struct {
+	HostID          string
+	CustomerID      string
+	PrivsepDirOK    bool
+	SshdReachable   bool
+	PrivsepHealedAt string
+}
+
+// GetHostMgmtPlaneStates returns the latest mgmt_plane stanza per host (mirrors
+// GetHostLeafFingerprints). A report without the stanza (old agent, feature off) yields zero values →
+// no alert. Malformed JSON degrades to zero values, never an error for that host.
+func (s *Store) GetHostMgmtPlaneStates() ([]HostMgmtPlaneRow, error) {
+	rows, err := s.db.Query(`
+		SELECT hr.host_id, hr.customer_id, hr.report_json
+		FROM host_reports hr
+		JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest
+		  ON hr.id = latest.mx`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []HostMgmtPlaneRow
+	for rows.Next() {
+		var r HostMgmtPlaneRow
+		var reportJSON string
+		if err := rows.Scan(&r.HostID, &r.CustomerID, &reportJSON); err != nil {
+			return nil, err
+		}
+		var body struct {
+			MgmtPlane *struct {
+				PrivsepDirOK    bool   `json:"privsep_dir_ok"`
+				SshdReachable   bool   `json:"sshd_reachable"`
+				PrivsepHealedAt string `json:"privsep_healed_at"`
+			} `json:"mgmt_plane"`
+		}
+		_ = json.Unmarshal([]byte(reportJSON), &body) // malformed/old → nil mgmt_plane → zero values
+		if body.MgmtPlane != nil {
+			r.PrivsepDirOK = body.MgmtPlane.PrivsepDirOK
+			r.SshdReachable = body.MgmtPlane.SshdReachable
+			r.PrivsepHealedAt = body.MgmtPlane.PrivsepHealedAt
+		}
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}

hub/internal/store/host_recovery_test.go

@@ -0,0 +1,83 @@
+package store
+
+import "testing"
+
+func TestHostRecoveryCredential_RoundTripUpsertAndAbsent(t *testing.T) {
+	s := newTestStore(t)
+	if err := s.UpsertHost(&Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
+		t.Fatalf("UpsertHost: %v", err)
+	}
+
+	// absent → (nil, nil) + Has=false
+	got, err := s.GetHostRecoveryCredential("h1")
+	if err != nil || got != nil {
+		t.Fatalf("absent cred: got %+v / %v (want nil,nil)", got, err)
+	}
+	has, _ := s.HasHostRecoveryCredential("h1")
+	if has {
+		t.Fatal("HasHostRecoveryCredential must be false before any vault")
+	}
+
+	// vault → round-trips
+	if err := s.SaveHostRecoveryCredential("h1", "root@pam", "s3cret-Aa1"); err != nil {
+		t.Fatalf("SaveHostRecoveryCredential: %v", err)
+	}
+	got, err = s.GetHostRecoveryCredential("h1")
+	if err != nil || got == nil {
+		t.Fatalf("GetHostRecoveryCredential: %+v / %v", got, err)
+	}
+	if got.Username != "root@pam" || got.Secret != "s3cret-Aa1" {
+		t.Fatalf("round-trip mismatch: %+v", got)
+	}
+	if has, _ := s.HasHostRecoveryCredential("h1"); !has {
+		t.Fatal("HasHostRecoveryCredential must be true after vault")
+	}
+
+	// upsert (rotate) → overwrites last-write-wins
+	if err := s.SaveHostRecoveryCredential("h1", "root@pam", "rotated-Bb2"); err != nil {
+		t.Fatalf("re-vault: %v", err)
+	}
+	got, _ = s.GetHostRecoveryCredential("h1")
+	if got.Secret != "rotated-Bb2" {
+		t.Fatalf("rotate did not overwrite: %+v", got)
+	}
+}
+
+func TestGetHostMgmtPlaneStates_ParsesHealMarker(t *testing.T) {
+	s := newTestStore(t)
+	if err := s.UpsertHost(&Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
+		t.Fatalf("UpsertHost: %v", err)
+	}
+	// a report WITH a heal marker
+	report := `{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true,"healed_recently":true,"privsep_healed_at":"2026-07-05T16:42:17Z"}}`
+	if err := s.SaveHostReport("h1", "c1", []byte(report), HostReportDenorm{}); err != nil {
+		t.Fatalf("SaveHostReport: %v", err)
+	}
+	rows, err := s.GetHostMgmtPlaneStates()
+	if err != nil {
+		t.Fatalf("GetHostMgmtPlaneStates: %v", err)
+	}
+	var found bool
+	for _, r := range rows {
+		if r.HostID == "h1" {
+			found = true
+			if !r.PrivsepDirOK || r.PrivsepHealedAt != "2026-07-05T16:42:17Z" {
+				t.Fatalf("parsed row wrong: %+v", r)
+			}
+		}
+	}
+	if !found {
+		t.Fatal("h1 not in mgmt-plane states")
+	}
+
+	// a report WITHOUT the stanza (old agent) → zero values, no crash
+	if err := s.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), HostReportDenorm{}); err != nil {
+		t.Fatalf("SaveHostReport2: %v", err)
+	}
+	rows, _ = s.GetHostMgmtPlaneStates()
+	for _, r := range rows {
+		if r.HostID == "h1" && r.PrivsepHealedAt != "" {
+			t.Fatalf("old-agent report should yield empty healed_at, got %+v", r)
+		}
+	}
+}

hub/internal/store/store.go

@@ -398,6 +398,24 @@ func (s *Store) migrate() error {
 		return err
 	}
 
+	// host_recovery (TASK G1): the break-glass root@pam console credential, vaulted at rest and
+	// operator-retrievable. UNLIKE host_escrow (opaque, hub-can't-open), this IS a hub-held secret the
+	// operator retrieves to reach the PVE web console (pveproxy — a failure domain distinct from sshd)
+	// when both the sshd path AND the agent-independent auto-heal have failed. One row per host,
+	// last-write-wins (day-0 sets it; --rotate re-sets). Never in desired-state, never logged.
+	_, err = s.db.Exec(`
+		CREATE TABLE IF NOT EXISTS host_recovery (
+			host_id     TEXT PRIMARY KEY,
+			username    TEXT NOT NULL,
+			secret      TEXT NOT NULL,
+			set_at      DATETIME NOT NULL DEFAULT (datetime('now')),
+			updated_at  DATETIME NOT NULL DEFAULT (datetime('now'))
+		);
+	`)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 

scripts/felhom-host-install.sh

@@ -153,6 +153,7 @@ UNINSTALL=false          # --uninstall: local host teardown (destroy guest + rem
 REMOVE_GOLDEN=false      # --remove-golden: also delete the golden vzdump during --uninstall
 ADOPT_POOL=false         # --adopt-pool: retrofit an EXISTING Felhom guest into the felhom pool (non-destructive)
 RESCOPE_ACL=false        # --rescope-acl: migrate an existing install from the broad-/ token to the scoped ACL
+ROTATE_RECOVERY=false    # --rotate-recovery: regenerate + re-vault the break-glass root@pam password (TASK G1)
 
 # --- Gitea (artifact source) + agent install model (BUNDLE slice) ---
 GITEA_BASE="https://gitea.dooplex.hu"
@@ -582,6 +583,23 @@ run_uninstall() {
     if [[ -f "$agent_cfg" ]]; then run rm -f "$agent_cfg"; else log_skip "  $agent_cfg already absent"; fi
     run rmdir "$(dirname "$agent_cfg")" 2>/dev/null || true
 
+    # 4b2. Management-plane break-glass (TASK G1): timer+oneshot+script+tmpfiles. Stop/disable the
+    #      timer, remove all four artifacts + the runtime heal-marker. We do NOT `rmdir /run/sshd` —
+    #      the stock sshd needs it; leaving the (now unit-less) dir in place is correct (a bare kernel
+    #      /run tmpfs recreates it empty on next boot anyway). Tolerate-absent throughout.
+    if systemctl list-unit-files felhom-mgmt-watchdog.timer >/dev/null 2>&1; then
+        systemctl is-active  --quiet felhom-mgmt-watchdog.timer 2>/dev/null && run systemctl stop    felhom-mgmt-watchdog.timer
+        systemctl is-enabled --quiet felhom-mgmt-watchdog.timer 2>/dev/null && run systemctl disable felhom-mgmt-watchdog.timer
+    else
+        log_skip "  felhom-mgmt-watchdog.timer not loaded — skip stop/disable"
+    fi
+    run systemctl reset-failed felhom-mgmt-watchdog.service 2>/dev/null || true
+    local wda
+    for wda in /etc/systemd/system/felhom-mgmt-watchdog.service /etc/systemd/system/felhom-mgmt-watchdog.timer \
+               /usr/local/sbin/felhom-mgmt-watchdog /etc/tmpfiles.d/felhom-privsep.conf /run/felhom-mgmt-watchdog.healed; do
+        if [[ -e "$wda" ]]; then run rm -f "$wda"; fi
+    done
+
     # 4c. Shared-parent unit + wrapper + /mnt/felhom-drives (agent-installed at runtime; drill R2).
     #     Stop/disable, remove unit + script, unbind + remove the (empty) parent dir. Tolerate-absent.
     if systemctl list-unit-files felhom-shared-parent.service 2>/dev/null | grep -q felhom-shared-parent; then
@@ -782,6 +800,7 @@ while [[ $# -gt 0 ]]; do
         --remove-golden)   REMOVE_GOLDEN=true; shift ;;
         --adopt-pool)      ADOPT_POOL=true; shift ;;
         --rescope-acl)     RESCOPE_ACL=true; shift ;;
+        --rotate-recovery) ROTATE_RECOVERY=true; shift ;;
         --acl-storages)    read -ra PVE_STORAGES <<< "$2"; shift 2 ;;
         --dry-run)         DRY_RUN=true; shift ;;
         --resume)          RESUME=true; shift ;;
@@ -1141,6 +1160,54 @@ step_enroll() {
     _state_mark enroll
 }
 
+#-------------------------------------------------------------------------------
+# STEP 4b — break-glass credential (TASK G1): generate + set + vault the root@pam console password
+#-------------------------------------------------------------------------------
+# The human fallback for when BOTH the sshd path AND the agent-independent auto-heal (layers 1+2) have
+# failed: a strong root@pam password lets the operator reach the PVE WEB CONSOLE (pveproxy :8006 — a
+# failure domain distinct from sshd) and run the one-line /run/sshd fix. Generated with strong entropy,
+# set via chpasswd, and vaulted to the hub over the enroll-authenticated channel (host api_key). The
+# password is NEVER logged, printed, or written to any file — it goes stdin→chpasswd and stdin→curl
+# only (SPIKE-felhom-sshd finding #9 / TASK G1 trap 3). Idempotent: skipped if already vaulted unless
+# --rotate-recovery (a re-set would strand the operator's saved copy).
+step_break_glass() {
+    log_step "4b/8 break-glass credential (root@pam console password → hub vault)"
+    if [[ -z "${HOST_ID:-}" || -z "${HOST_API_KEY:-}" ]]; then
+        log_warn "  no host_id/api_key (enroll skipped?) — cannot vault a recovery credential; skipping"
+        return 0
+    fi
+    if $DRY_RUN; then
+        log_dry "openssl rand → strong root@pam password (never logged) ; chpasswd ; PUT $HUB_URL/api/v1/hosts/$HOST_ID/recovery-credential (Bearer host key)"
+        _state_mark break_glass; return 0
+    fi
+    if _state_has break_glass && ! $ROTATE_RECOVERY; then
+        log_skip "  recovery credential already vaulted (use --rotate-recovery to regenerate)"
+        return 0
+    fi
+    # Strong password: 24 url-safe bytes (~144 bits). Kept ONLY in a local shell var, never on disk.
+    local newpw
+    newpw=$(openssl rand -base64 24 2>/dev/null | tr -d '\n' | tr '+/' '-_')
+    [[ ${#newpw} -ge 24 ]] || die "failed to generate a strong recovery password"
+    # Set root@pam (= the Linux root user on PVE) via chpasswd on STDIN — no argv, no log.
+    if ! printf 'root:%s\n' "$newpw" | chpasswd 2>/dev/null; then
+        newpw=""  # scrub
+        die "chpasswd failed to set the root@pam recovery password"
+    fi
+    # Vault to the hub over the host-key-authenticated channel; password only on stdin (-d @-).
+    local code
+    code=$(printf '{"username":"root@pam","password":"%s"}' "$newpw" \
+        | curl -sS -o /dev/null -w '%{http_code}' -X PUT \
+            "$HUB_URL/api/v1/hosts/$HOST_ID/recovery-credential" \
+            -H "Authorization: Bearer $HOST_API_KEY" -H 'Content-Type: application/json' -d @- 2>/dev/null)
+    newpw=""  # scrub the plaintext from the shell var the moment it is vaulted
+    case "$code" in
+        200) log_success "  root@pam password set + vaulted to the hub (retrieve via the operator /admin path; never logged here)" ;;
+        401|403) die "recovery-credential vault rejected ($code) — host key/authorization problem" ;;
+        *)   die "recovery-credential vault failed (HTTP $code)" ;;
+    esac
+    _state_mark break_glass
+}
+
 #-------------------------------------------------------------------------------
 # STEP 5 — agent install: fetch+verify the binary, ensure the service user, sudoers, unit
 #-------------------------------------------------------------------------------
@@ -1331,9 +1398,63 @@ step_agent_install() {
         fi
         rm -f "$rbtmp"
     fi
+
+    # Management-plane break-glass layers 1+2 (TASK G1). Three artifacts that keep the host reachable
+    # even if a second sshd (H1) removes the SHARED /run/sshd privsep dir (SPIKE-felhom-sshd §8):
+    #   • felhom-privsep.conf   (tmpfiles) — layer 1: /run/sshd is boot-persistent, owned by no unit.
+    #   • felhom-mgmt-watchdog  (script)   — layer 2 heal action (recreate dir + reset-failed sshd).
+    #   • .service + .timer                — run it every ~60s, AGENT-INDEPENDENTLY (heals with the
+    #                                        agent down — the whole point; trap 1).
+    # Non-fatal if the agent repo predates them (raw fetch 404s → break-glass just stays manual).
+    # HARD GUARD: refuse ANY fetched unit that declares RuntimeDirectory= — that directive is the very
+    # incident G1 closes (a second sshd's `RuntimeDirectory=sshd` removed the shared /run/sshd).
+    install_mgmt_watchdog
+
     _state_mark agent_install
 }
 
+# install_mgmt_watchdog fetches + installs the G1 break-glass host artifacts (idempotent; enables the
+# timer). Split out for readability; called from step_agent_install. Every unit is RuntimeDirectory-
+# guarded (trap 2). Non-fatal on a repo that predates the artifacts.
+install_mgmt_watchdog() {
+    if $DRY_RUN; then
+        log_dry "fetch configs/felhom-privsep.tmpfiles -> /etc/tmpfiles.d/felhom-privsep.conf ; systemd-tmpfiles --create"
+        log_dry "fetch configs/felhom-mgmt-watchdog.sh -> /usr/local/sbin/felhom-mgmt-watchdog (0755)"
+        log_dry "fetch configs/felhom-mgmt-watchdog.{service,timer} -> /etc/systemd/system/ ; enable --now felhom-mgmt-watchdog.timer"
+        return 0
+    fi
+    local wdtmp; wdtmp=$(mktemp -t felhom-mgmt-wd.XXXXXX)
+    if ! fetch_raw "configs/felhom-mgmt-watchdog.sh" "$wdtmp" 2>/dev/null; then
+        log_skip "  mgmt-watchdog artifacts not in the agent repo yet — break-glass auto-heal stays manual"
+        rm -f "$wdtmp"; return 0
+    fi
+    sh -n "$wdtmp" || { rm -f "$wdtmp"; die "fetched felhom-mgmt-watchdog.sh failed sh -n — refusing to install"; }
+    install -m 0755 -o root -g root "$wdtmp" /usr/local/sbin/felhom-mgmt-watchdog
+    rm -f "$wdtmp"
+
+    # tmpfiles (layer 1) — install + create now so /run/sshd is guaranteed present immediately.
+    local tftmp; tftmp=$(mktemp -t felhom-privsep.XXXXXX)
+    fetch_raw "configs/felhom-privsep.tmpfiles" "$tftmp"
+    install -m 0644 -o root -g root "$tftmp" /etc/tmpfiles.d/felhom-privsep.conf
+    rm -f "$tftmp"
+    systemd-tmpfiles --create /etc/tmpfiles.d/felhom-privsep.conf 2>/dev/null || true
+
+    # units (layer 2) — RuntimeDirectory guard on BOTH before install (trap 2 / the incident cause).
+    local svctmp tmrtmp; svctmp=$(mktemp -t felhom-wd-svc.XXXXXX); tmrtmp=$(mktemp -t felhom-wd-tmr.XXXXXX)
+    fetch_raw "configs/felhom-mgmt-watchdog.service" "$svctmp"
+    fetch_raw "configs/felhom-mgmt-watchdog.timer" "$tmrtmp"
+    if grep -qiE '^[[:space:]]*RuntimeDirectory[[:space:]]*=' "$svctmp" "$tmrtmp"; then
+        rm -f "$svctmp" "$tmrtmp"
+        die "mgmt-watchdog unit declares RuntimeDirectory= — that is the incident G1 fixes; refusing to install"
+    fi
+    install -m 0644 -o root -g root "$svctmp" /etc/systemd/system/felhom-mgmt-watchdog.service
+    install -m 0644 -o root -g root "$tmrtmp" /etc/systemd/system/felhom-mgmt-watchdog.timer
+    rm -f "$svctmp" "$tmrtmp"
+    systemctl daemon-reload
+    systemctl enable --now felhom-mgmt-watchdog.timer >/dev/null 2>&1 || true
+    log_success "  installed break-glass layers 1+2 (tmpfiles /run/sshd + agent-independent watchdog timer)"
+}
+
 #-------------------------------------------------------------------------------
 # STEP 6 — write agent config + ensure service healthy
 #-------------------------------------------------------------------------------
@@ -1611,6 +1732,7 @@ fi
 should_skip token         || step_token
 should_skip grows         || step_grows
 should_skip enroll        || step_enroll
+should_skip break_glass   || step_break_glass
 should_skip agent_install || step_agent_install
 should_skip agent_config  || step_agent_config
 should_skip golden        || step_golden