feat(hub,install): break-glass recovery vault + mgmt_plane surfacing (TASK G1)

Hub half of the management-plane break-glass (prereq for felhom-sshd/H1; agent half = felhom-agent v0.71.0). Closes SPIKE-felhom-sshd §8/#9. - store.host_recovery + methods: per-host root@pam console password, at-rest, operator-retrievable (the PVE-web-console fallback when sshd + auto-heal both fail). - API: PUT /hosts/{id}/recovery-credential (self-scoped, day-0 vaults) + GET /admin/hosts/{id}/recovery-credential (global key only). Secret never logged (red-proofed). - monitor/host_mgmtplane: parses the agent mgmt_plane stanza, raises mgmt_plane_healed WARNING on a new privsep_healed_at (recurring clobber surfaces before lockout; complements host_staleness). - host-install: step_break_glass generates a strong root@pam password (openssl rand, never logged/filed — stdin to chpasswd + curl), vaults via host key; idempotent unless --rotate-recovery. Installs the G1 host artifacts (tmpfiles + agent-independent watchdog timer), RuntimeDirectory-guarded; uninstall removes them. Hub v0.34.0. Non-hollow tests + red-proofs; full suite green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
2026-07-05 19:03:18 +02:00
parent 2f97ce31dd
commit 05d81810d4
10 changed files with 739 additions and 0 deletions
@@ -155,6 +155,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/escrow"):
 		hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/escrow")
 		h.handleHostEscrowPut(w, r, hostID)
+	// G1 break-glass: day-0 vaults the root@pam console credential (self-scoped host key); the
+	// operator retrieves it via the /admin/ path (global key only).
+	case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/recovery-credential"):
+		hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/recovery-credential")
+		h.handleHostRecoveryCredentialPut(w, r, hostID)
+	case r.Method == http.MethodGet && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-credential"):
+		hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/admin/hosts/"), "/recovery-credential")
+		h.handleAdminGetRecoveryCredential(w, r, hostID)
 	// DR capstone (slice 10D). Recovery-mode toggle (global key); re-enroll + restore-directive
 	// (gated on recovery mode — no old key needed, the box is lost).
 	case r.Method == http.MethodPut && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-mode"):
@@ -883,6 +891,95 @@ func (h *Handler) handleHostEscrowPut(w http.ResponseWriter, r *http.Request, pa
 	w.Write([]byte(`{"status":"ok"}`))
 }

+// handleHostRecoveryCredentialPut vaults a host's break-glass root@pam console credential (TASK G1).
+// SELF-SCOPED (a host key writes only its own; global may write any) — day-0 posts it with the
+// host api_key. The secret is stored at rest and NEVER logged (only the username + a length are
+// logged). This is the human fallback for when both the sshd path AND the agent-independent
+// auto-heal have failed: the operator retrieves it to reach the PVE web console (pveproxy — a
+// failure domain distinct from sshd).
+func (h *Handler) handleHostRecoveryCredentialPut(w http.ResponseWriter, r *http.Request, pathHostID string) {
+	authHostID, _, isGlobal, ok := h.checkAuthHost(r)
+	if !ok {
+		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		return
+	}
+	if pathHostID == "" {
+		http.Error(w, "Missing host_id", http.StatusBadRequest)
+		return
+	}
+	if !isGlobal && authHostID != pathHostID {
+		http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
+		return
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, 1<<16)) // 64 KiB cap; a username+password is tiny
+	if err != nil {
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+	var req struct {
+		Username string `json:"username"`
+		Password string `json:"password"`
+	}
+	if err := json.Unmarshal(body, &req); err != nil || req.Username == "" || req.Password == "" {
+		http.Error(w, "Invalid payload: username + password required", http.StatusBadRequest)
+		return
+	}
+	// The host must exist (mint-first) — a per-host key already proves it; the global path re-checks.
+	if isGlobal {
+		host, herr := h.store.GetHost(pathHostID)
+		if herr != nil || host == nil {
+			http.Error(w, "Unknown host_id", http.StatusBadRequest)
+			return
+		}
+	}
+	if err := h.store.SaveHostRecoveryCredential(pathHostID, req.Username, req.Password); err != nil {
+		h.logger.Printf("[ERROR] Failed to vault recovery credential for host %s: %v", pathHostID, err)
+		http.Error(w, "Internal error", http.StatusInternalServerError)
+		return
+	}
+	// SECRET DISCIPLINE: log the username + a length only — NEVER the password.
+	h.logger.Printf("[INFO] vaulted break-glass recovery credential for host %s (user=%s, secret %d chars)",
+		pathHostID, req.Username, len(req.Password))
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte(`{"status":"ok"}`))
+}
+
+// handleAdminGetRecoveryCredential returns a host's vaulted break-glass credential to the OPERATOR
+// (global key only — a per-host key must NOT read its own console password back out). This is the
+// authenticated retrieval path the break-glass runbook uses. The response body carries the secret by
+// necessity; it is never written to the hub log.
+func (h *Handler) handleAdminGetRecoveryCredential(w http.ResponseWriter, r *http.Request, pathHostID string) {
+	_, _, isGlobal, ok := h.checkAuthHost(r)
+	if !ok || !isGlobal {
+		http.Error(w, "Unauthorized", http.StatusUnauthorized) // operator/global key ONLY
+		return
+	}
+	if pathHostID == "" {
+		http.Error(w, "Missing host_id", http.StatusBadRequest)
+		return
+	}
+	cred, err := h.store.GetHostRecoveryCredential(pathHostID)
+	if err != nil {
+		h.logger.Printf("[ERROR] Failed to read recovery credential for host %s: %v", pathHostID, err)
+		http.Error(w, "Internal error", http.StatusInternalServerError)
+		return
+	}
+	if cred == nil {
+		http.Error(w, "No recovery credential vaulted for this host", http.StatusNotFound)
+		return
+	}
+	h.logger.Printf("[INFO] operator retrieved break-glass recovery credential for host %s (user=%s)", pathHostID, cred.Username)
+	resp, _ := json.Marshal(map[string]string{
+		"host_id":  cred.HostID,
+		"username": cred.Username,
+		"password": cred.Secret,
+		"set_at":   cred.SetAt.UTC().Format(time.RFC3339),
+	})
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	w.Write(resp)
+}
+
 // handleGetDesiredState serves a host its authoritative desired-state (slice 10A). Per-host key,
 // SELF-SCOPED: a host reads ONLY its own (the global operator key may read any). The agent fetches
 // this when the heartbeat envelope's desired_generation has advanced past its cached one. The
@@ -0,0 +1,74 @@
+package api
+
+import (
+	"bytes"
+	"log"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+	_ "modernc.org/sqlite"
+)
+
+func TestRecoveryCredential_VaultSelfScopedAndOperatorRetrieval(t *testing.T) {
+	h, st, _ := newTestHandler(t)
+	st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"})
+	st.UpsertHost(&store.Host{HostID: "h2", CustomerID: "c1", APIKey: "HKEY2"})
+
+	body := `{"username":"root@pam","password":"Str0ng-Break-Glass"}`
+
+	// self-scoped write: h1's key vaults h1 → 200
+	if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "HKEY", body); rr.Code != 200 {
+		t.Fatalf("self vault = %d: %s", rr.Code, rr.Body.String())
+	}
+	// a host key CANNOT vault ANOTHER host → 403
+	if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "HKEY2", body); rr.Code != 403 {
+		t.Fatalf("cross-host vault must be 403, got %d", rr.Code)
+	}
+	// unauthenticated → 401
+	if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "", body); rr.Code != 401 {
+		t.Fatalf("unauth vault must be 401, got %d", rr.Code)
+	}
+
+	// operator retrieval requires the GLOBAL key; a host key is refused (401 — can't read its own back)
+	if rr := do(h, "GET", "/admin/hosts/h1/recovery-credential", "HKEY", ""); rr.Code != 401 {
+		t.Fatalf("host key reading recovery credential must be 401, got %d", rr.Code)
+	}
+	rr := do(h, "GET", "/admin/hosts/h1/recovery-credential", globalKey, "")
+	if rr.Code != 200 {
+		t.Fatalf("operator retrieval = %d: %s", rr.Code, rr.Body.String())
+	}
+	if !strings.Contains(rr.Body.String(), "Str0ng-Break-Glass") || !strings.Contains(rr.Body.String(), "root@pam") {
+		t.Fatalf("retrieval must return the vaulted credential, got %s", rr.Body.String())
+	}
+	// a host with no credential → 404
+	if rr := do(h, "GET", "/admin/hosts/h2/recovery-credential", globalKey, ""); rr.Code != 404 {
+		t.Fatalf("no-credential host must be 404, got %d", rr.Code)
+	}
+}
+
+// SECRET DISCIPLINE (red-proof for trap 3): the password must NEVER appear in the hub log — on the
+// vault write NOR the operator retrieval. Companion: if a handler logged the password, this fails.
+func TestRecoveryCredential_PasswordNeverLogged(t *testing.T) {
+	var buf bytes.Buffer
+	path := filepath.Join(t.TempDir(), "test.db")
+	st, err := store.New(path, log.New(&buf, "", 0))
+	if err != nil {
+		t.Fatalf("store.New: %v", err)
+	}
+	defer st.Close()
+	h := New(st, globalKey, "", "", nil, log.New(&buf, "", 0))
+	st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"})
+
+	const secret = "SuperSecret-DoNotLog-42"
+	if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "HKEY", `{"username":"root@pam","password":"`+secret+`"}`); rr.Code != 200 {
+		t.Fatalf("vault = %d", rr.Code)
+	}
+	if rr := do(h, "GET", "/admin/hosts/h1/recovery-credential", globalKey, ""); rr.Code != 200 {
+		t.Fatalf("retrieve = %d", rr.Code)
+	}
+	if strings.Contains(buf.String(), secret) {
+		t.Fatalf("the recovery password LEAKED into the hub log:\n%s", buf.String())
+	}
+}