feat(hub,install): break-glass recovery vault + mgmt_plane surfacing (TASK G1)

Hub half of the management-plane break-glass (prereq for felhom-sshd/H1; agent
half = felhom-agent v0.71.0). Closes SPIKE-felhom-sshd §8/#9.

- store.host_recovery + methods: per-host root@pam console password, at-rest,
  operator-retrievable (the PVE-web-console fallback when sshd + auto-heal both fail).
- API: PUT /hosts/{id}/recovery-credential (self-scoped, day-0 vaults) + GET
  /admin/hosts/{id}/recovery-credential (global key only). Secret never logged
  (red-proofed).
- monitor/host_mgmtplane: parses the agent mgmt_plane stanza, raises
  mgmt_plane_healed WARNING on a new privsep_healed_at (recurring clobber surfaces
  before lockout; complements host_staleness).
- host-install: step_break_glass generates a strong root@pam password (openssl
  rand, never logged/filed — stdin to chpasswd + curl), vaults via host key;
  idempotent unless --rotate-recovery. Installs the G1 host artifacts (tmpfiles +
  agent-independent watchdog timer), RuntimeDirectory-guarded; uninstall removes them.

Hub v0.34.0. Non-hollow tests + red-proofs; full suite green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
2026-07-05 19:03:18 +02:00
parent 2f97ce31dd
commit 05d81810d4
10 changed files with 739 additions and 0 deletions
+97
View File
@@ -155,6 +155,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/escrow"):
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/escrow")
h.handleHostEscrowPut(w, r, hostID)
// G1 break-glass: day-0 vaults the root@pam console credential (self-scoped host key); the
// operator retrieves it via the /admin/ path (global key only).
case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/recovery-credential"):
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/recovery-credential")
h.handleHostRecoveryCredentialPut(w, r, hostID)
case r.Method == http.MethodGet && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-credential"):
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/admin/hosts/"), "/recovery-credential")
h.handleAdminGetRecoveryCredential(w, r, hostID)
// DR capstone (slice 10D). Recovery-mode toggle (global key); re-enroll + restore-directive
// (gated on recovery mode — no old key needed, the box is lost).
case r.Method == http.MethodPut && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-mode"):
@@ -883,6 +891,95 @@ func (h *Handler) handleHostEscrowPut(w http.ResponseWriter, r *http.Request, pa
w.Write([]byte(`{"status":"ok"}`))
}
// handleHostRecoveryCredentialPut vaults a host's break-glass root@pam console credential (TASK G1).
// SELF-SCOPED (a host key writes only its own; global may write any) — day-0 posts it with the
// host api_key. The secret is stored at rest and NEVER logged (only the username + a length are
// logged). This is the human fallback for when both the sshd path AND the agent-independent
// auto-heal have failed: the operator retrieves it to reach the PVE web console (pveproxy — a
// failure domain distinct from sshd).
func (h *Handler) handleHostRecoveryCredentialPut(w http.ResponseWriter, r *http.Request, pathHostID string) {
authHostID, _, isGlobal, ok := h.checkAuthHost(r)
if !ok {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
if pathHostID == "" {
http.Error(w, "Missing host_id", http.StatusBadRequest)
return
}
if !isGlobal && authHostID != pathHostID {
http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
return
}
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<16)) // 64 KiB cap; a username+password is tiny
if err != nil {
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
var req struct {
Username string `json:"username"`
Password string `json:"password"`
}
if err := json.Unmarshal(body, &req); err != nil || req.Username == "" || req.Password == "" {
http.Error(w, "Invalid payload: username + password required", http.StatusBadRequest)
return
}
// The host must exist (mint-first) — a per-host key already proves it; the global path re-checks.
if isGlobal {
host, herr := h.store.GetHost(pathHostID)
if herr != nil || host == nil {
http.Error(w, "Unknown host_id", http.StatusBadRequest)
return
}
}
if err := h.store.SaveHostRecoveryCredential(pathHostID, req.Username, req.Password); err != nil {
h.logger.Printf("[ERROR] Failed to vault recovery credential for host %s: %v", pathHostID, err)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
// SECRET DISCIPLINE: log the username + a length only — NEVER the password.
h.logger.Printf("[INFO] vaulted break-glass recovery credential for host %s (user=%s, secret %d chars)",
pathHostID, req.Username, len(req.Password))
w.WriteHeader(http.StatusOK)
w.Write([]byte(`{"status":"ok"}`))
}
// handleAdminGetRecoveryCredential returns a host's vaulted break-glass credential to the OPERATOR
// (global key only — a per-host key must NOT read its own console password back out). This is the
// authenticated retrieval path the break-glass runbook uses. The response body carries the secret by
// necessity; it is never written to the hub log.
func (h *Handler) handleAdminGetRecoveryCredential(w http.ResponseWriter, r *http.Request, pathHostID string) {
_, _, isGlobal, ok := h.checkAuthHost(r)
if !ok || !isGlobal {
http.Error(w, "Unauthorized", http.StatusUnauthorized) // operator/global key ONLY
return
}
if pathHostID == "" {
http.Error(w, "Missing host_id", http.StatusBadRequest)
return
}
cred, err := h.store.GetHostRecoveryCredential(pathHostID)
if err != nil {
h.logger.Printf("[ERROR] Failed to read recovery credential for host %s: %v", pathHostID, err)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
if cred == nil {
http.Error(w, "No recovery credential vaulted for this host", http.StatusNotFound)
return
}
h.logger.Printf("[INFO] operator retrieved break-glass recovery credential for host %s (user=%s)", pathHostID, cred.Username)
resp, _ := json.Marshal(map[string]string{
"host_id": cred.HostID,
"username": cred.Username,
"password": cred.Secret,
"set_at": cred.SetAt.UTC().Format(time.RFC3339),
})
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(resp)
}
// handleGetDesiredState serves a host its authoritative desired-state (slice 10A). Per-host key,
// SELF-SCOPED: a host reads ONLY its own (the global operator key may read any). The agent fetches
// this when the heartbeat envelope's desired_generation has advanced past its cached one. The