feat(hub,install): break-glass recovery vault + mgmt_plane surfacing (TASK G1)
Hub half of the management-plane break-glass (prereq for felhom-sshd/H1; agent
half = felhom-agent v0.71.0). Closes SPIKE-felhom-sshd §8/#9.
- store.host_recovery + methods: per-host root@pam console password, at-rest,
operator-retrievable (the PVE-web-console fallback when sshd + auto-heal both fail).
- API: PUT /hosts/{id}/recovery-credential (self-scoped, day-0 vaults) + GET
/admin/hosts/{id}/recovery-credential (global key only). Secret never logged
(red-proofed).
- monitor/host_mgmtplane: parses the agent mgmt_plane stanza, raises
mgmt_plane_healed WARNING on a new privsep_healed_at (recurring clobber surfaces
before lockout; complements host_staleness).
- host-install: step_break_glass generates a strong root@pam password (openssl
rand, never logged/filed — stdin to chpasswd + curl), vaults via host key;
idempotent unless --rotate-recovery. Installs the G1 host artifacts (tmpfiles +
agent-independent watchdog timer), RuntimeDirectory-guarded; uninstall removes them.
Hub v0.34.0. Non-hollow tests + red-proofs; full suite green.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -1,5 +1,24 @@
|
||||
# Felhom Hub — Changelog
|
||||
|
||||
## v0.34.0 — break-glass recovery vault + mgmt_plane surfacing (TASK G1) (2026-07-05)
|
||||
|
||||
The hub half of the management-plane break-glass system (prerequisite for felhom-sshd / H1; agent half
|
||||
= felhom-agent v0.71.0). Closes the recovery gap from
|
||||
`documentation/audits/SPIKE-felhom-sshd-2026-07-05.md` §8/#9.
|
||||
|
||||
- **Break-glass credential vault** (`store.host_recovery` + `internal/store/host_recovery.go`): a
|
||||
per-host root@pam console password, stored at rest, operator-retrievable — the human fallback for
|
||||
reaching the PVE web console (pveproxy, a failure domain distinct from sshd) when both the sshd path
|
||||
and the agent-independent auto-heal have failed. `PUT /hosts/{id}/recovery-credential` (SELF-scoped
|
||||
host key — day-0 vaults it) + `GET /admin/hosts/{id}/recovery-credential` (GLOBAL key only — a host
|
||||
key cannot read its own console password back). Secret discipline: never logged (username + length
|
||||
only); red-proofed that the password never reaches the hub log.
|
||||
- **mgmt_plane surfacing** (`internal/monitor/host_mgmtplane.go`, on the 60s sweep): parses the agent's
|
||||
additive `mgmt_plane` heartbeat stanza and raises a `mgmt_plane_healed` WARNING when the watchdog
|
||||
auto-healed a missing `/run/sshd` (new `privsep_healed_at`) — a recurring clobber surfaces BEFORE it
|
||||
becomes a lockout, complementing host_staleness. Trust-on-first-report (seed, then alert on change),
|
||||
mirroring HostLeafChecker.
|
||||
|
||||
## v0.33.0 — S2 offsite connectivity: box-facing WG registration + wireguard desired-state block + /offsite UI (2026-07-04)
|
||||
|
||||
Doc 06 roadmap row S2 (commits `fcf84a0`/`ba52005`/`13203c2`); the S2 architectural decision:
|
||||
|
||||
@@ -378,6 +378,9 @@ func main() {
|
||||
// target (dump/backup volume, data drive, lvmthin pool, PBS datastore). Born/persistent; excludes the
|
||||
// root-backed builtin (hostDiskChecker owns root, no double-alert); emits natural `critical`. Same sweep.
|
||||
storageFillChecker := monitor.NewStorageFillChecker(dataStore, cfg.Alerting.StorageFillWarnPercent, cfg.Alerting.StorageFillCritPercent, dispatcher.ProcessEvent, logger)
|
||||
// TASK G1: warn when a host's agent-independent watchdog auto-healed a missing /run/sshd privsep
|
||||
// dir — a recurring clobber that can lead to an SSH lockout (complements host_staleness). Same sweep.
|
||||
hostMgmtPlaneChecker := monitor.NewHostMgmtPlaneChecker(dataStore, dispatcher.ProcessEvent, logger)
|
||||
go func() {
|
||||
ticker := time.NewTicker(60 * time.Second)
|
||||
defer ticker.Stop()
|
||||
@@ -392,6 +395,7 @@ func main() {
|
||||
hostLeafChecker.Check()
|
||||
hostDiskChecker.Check()
|
||||
storageFillChecker.Check()
|
||||
hostMgmtPlaneChecker.Check()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -155,6 +155,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/escrow"):
|
||||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/escrow")
|
||||
h.handleHostEscrowPut(w, r, hostID)
|
||||
// G1 break-glass: day-0 vaults the root@pam console credential (self-scoped host key); the
|
||||
// operator retrieves it via the /admin/ path (global key only).
|
||||
case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/recovery-credential"):
|
||||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/recovery-credential")
|
||||
h.handleHostRecoveryCredentialPut(w, r, hostID)
|
||||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-credential"):
|
||||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/admin/hosts/"), "/recovery-credential")
|
||||
h.handleAdminGetRecoveryCredential(w, r, hostID)
|
||||
// DR capstone (slice 10D). Recovery-mode toggle (global key); re-enroll + restore-directive
|
||||
// (gated on recovery mode — no old key needed, the box is lost).
|
||||
case r.Method == http.MethodPut && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-mode"):
|
||||
@@ -883,6 +891,95 @@ func (h *Handler) handleHostEscrowPut(w http.ResponseWriter, r *http.Request, pa
|
||||
w.Write([]byte(`{"status":"ok"}`))
|
||||
}
|
||||
|
||||
// handleHostRecoveryCredentialPut vaults a host's break-glass root@pam console credential (TASK G1).
|
||||
// SELF-SCOPED (a host key writes only its own; global may write any) — day-0 posts it with the
|
||||
// host api_key. The secret is stored at rest and NEVER logged (only the username + a length are
|
||||
// logged). This is the human fallback for when both the sshd path AND the agent-independent
|
||||
// auto-heal have failed: the operator retrieves it to reach the PVE web console (pveproxy — a
|
||||
// failure domain distinct from sshd).
|
||||
func (h *Handler) handleHostRecoveryCredentialPut(w http.ResponseWriter, r *http.Request, pathHostID string) {
|
||||
authHostID, _, isGlobal, ok := h.checkAuthHost(r)
|
||||
if !ok {
|
||||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
if pathHostID == "" {
|
||||
http.Error(w, "Missing host_id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !isGlobal && authHostID != pathHostID {
|
||||
http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<16)) // 64 KiB cap; a username+password is tiny
|
||||
if err != nil {
|
||||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &req); err != nil || req.Username == "" || req.Password == "" {
|
||||
http.Error(w, "Invalid payload: username + password required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// The host must exist (mint-first) — a per-host key already proves it; the global path re-checks.
|
||||
if isGlobal {
|
||||
host, herr := h.store.GetHost(pathHostID)
|
||||
if herr != nil || host == nil {
|
||||
http.Error(w, "Unknown host_id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := h.store.SaveHostRecoveryCredential(pathHostID, req.Username, req.Password); err != nil {
|
||||
h.logger.Printf("[ERROR] Failed to vault recovery credential for host %s: %v", pathHostID, err)
|
||||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// SECRET DISCIPLINE: log the username + a length only — NEVER the password.
|
||||
h.logger.Printf("[INFO] vaulted break-glass recovery credential for host %s (user=%s, secret %d chars)",
|
||||
pathHostID, req.Username, len(req.Password))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(`{"status":"ok"}`))
|
||||
}
|
||||
|
||||
// handleAdminGetRecoveryCredential returns a host's vaulted break-glass credential to the OPERATOR
|
||||
// (global key only — a per-host key must NOT read its own console password back out). This is the
|
||||
// authenticated retrieval path the break-glass runbook uses. The response body carries the secret by
|
||||
// necessity; it is never written to the hub log.
|
||||
func (h *Handler) handleAdminGetRecoveryCredential(w http.ResponseWriter, r *http.Request, pathHostID string) {
|
||||
_, _, isGlobal, ok := h.checkAuthHost(r)
|
||||
if !ok || !isGlobal {
|
||||
http.Error(w, "Unauthorized", http.StatusUnauthorized) // operator/global key ONLY
|
||||
return
|
||||
}
|
||||
if pathHostID == "" {
|
||||
http.Error(w, "Missing host_id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
cred, err := h.store.GetHostRecoveryCredential(pathHostID)
|
||||
if err != nil {
|
||||
h.logger.Printf("[ERROR] Failed to read recovery credential for host %s: %v", pathHostID, err)
|
||||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if cred == nil {
|
||||
http.Error(w, "No recovery credential vaulted for this host", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
h.logger.Printf("[INFO] operator retrieved break-glass recovery credential for host %s (user=%s)", pathHostID, cred.Username)
|
||||
resp, _ := json.Marshal(map[string]string{
|
||||
"host_id": cred.HostID,
|
||||
"username": cred.Username,
|
||||
"password": cred.Secret,
|
||||
"set_at": cred.SetAt.UTC().Format(time.RFC3339),
|
||||
})
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(resp)
|
||||
}
|
||||
|
||||
// handleGetDesiredState serves a host its authoritative desired-state (slice 10A). Per-host key,
|
||||
// SELF-SCOPED: a host reads ONLY its own (the global operator key may read any). The agent fetches
|
||||
// this when the heartbeat envelope's desired_generation has advanced past its cached one. The
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func TestRecoveryCredential_VaultSelfScopedAndOperatorRetrieval(t *testing.T) {
|
||||
h, st, _ := newTestHandler(t)
|
||||
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"})
|
||||
st.UpsertHost(&store.Host{HostID: "h2", CustomerID: "c1", APIKey: "HKEY2"})
|
||||
|
||||
body := `{"username":"root@pam","password":"Str0ng-Break-Glass"}`
|
||||
|
||||
// self-scoped write: h1's key vaults h1 → 200
|
||||
if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "HKEY", body); rr.Code != 200 {
|
||||
t.Fatalf("self vault = %d: %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
// a host key CANNOT vault ANOTHER host → 403
|
||||
if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "HKEY2", body); rr.Code != 403 {
|
||||
t.Fatalf("cross-host vault must be 403, got %d", rr.Code)
|
||||
}
|
||||
// unauthenticated → 401
|
||||
if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "", body); rr.Code != 401 {
|
||||
t.Fatalf("unauth vault must be 401, got %d", rr.Code)
|
||||
}
|
||||
|
||||
// operator retrieval requires the GLOBAL key; a host key is refused (401 — can't read its own back)
|
||||
if rr := do(h, "GET", "/admin/hosts/h1/recovery-credential", "HKEY", ""); rr.Code != 401 {
|
||||
t.Fatalf("host key reading recovery credential must be 401, got %d", rr.Code)
|
||||
}
|
||||
rr := do(h, "GET", "/admin/hosts/h1/recovery-credential", globalKey, "")
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("operator retrieval = %d: %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "Str0ng-Break-Glass") || !strings.Contains(rr.Body.String(), "root@pam") {
|
||||
t.Fatalf("retrieval must return the vaulted credential, got %s", rr.Body.String())
|
||||
}
|
||||
// a host with no credential → 404
|
||||
if rr := do(h, "GET", "/admin/hosts/h2/recovery-credential", globalKey, ""); rr.Code != 404 {
|
||||
t.Fatalf("no-credential host must be 404, got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// SECRET DISCIPLINE (red-proof for trap 3): the password must NEVER appear in the hub log — on the
|
||||
// vault write NOR the operator retrieval. Companion: if a handler logged the password, this fails.
|
||||
func TestRecoveryCredential_PasswordNeverLogged(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
path := filepath.Join(t.TempDir(), "test.db")
|
||||
st, err := store.New(path, log.New(&buf, "", 0))
|
||||
if err != nil {
|
||||
t.Fatalf("store.New: %v", err)
|
||||
}
|
||||
defer st.Close()
|
||||
h := New(st, globalKey, "", "", nil, log.New(&buf, "", 0))
|
||||
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"})
|
||||
|
||||
const secret = "SuperSecret-DoNotLog-42"
|
||||
if rr := do(h, "PUT", "/hosts/h1/recovery-credential", "HKEY", `{"username":"root@pam","password":"`+secret+`"}`); rr.Code != 200 {
|
||||
t.Fatalf("vault = %d", rr.Code)
|
||||
}
|
||||
if rr := do(h, "GET", "/admin/hosts/h1/recovery-credential", globalKey, ""); rr.Code != 200 {
|
||||
t.Fatalf("retrieve = %d", rr.Code)
|
||||
}
|
||||
if strings.Contains(buf.String(), secret) {
|
||||
t.Fatalf("the recovery password LEAKED into the hub log:\n%s", buf.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
)
|
||||
|
||||
// HostMgmtPlaneChecker raises an operator WARNING when a host's agent-independent break-glass watchdog
|
||||
// AUTO-HEALED a missing /run/sshd privsep dir (TASK G1). The heal itself is silent and login-free (the
|
||||
// point of the watchdog); this surfaces a RECURRING clobber so the operator can find the cause BEFORE
|
||||
// it becomes a full management lockout — complementing HostStalenessChecker (which only catches a box
|
||||
// gone silent). Sibling of HostLeafChecker; runs on the same 60s sweep.
|
||||
//
|
||||
// Design (mirrors HostLeafChecker's trust-on-first-report): the state is the host's last-seen
|
||||
// privsep_healed_at marker timestamp. The watchdog rewrites the marker on EACH heal, so a new, different
|
||||
// timestamp = a new heal event → one warning. The first observation of a non-empty timestamp seeds the
|
||||
// baseline WITHOUT alerting (it may be a heal from before the hub was watching — avoid a false alarm on
|
||||
// startup; a genuinely recurring cause re-heals and re-alerts on the next occurrence). An empty
|
||||
// timestamp (healthy host / old agent) never alerts and never overwrites a baseline.
|
||||
type HostMgmtPlaneChecker struct {
|
||||
store *store.Store
|
||||
logger *log.Logger
|
||||
onEvent EventNotifyFunc
|
||||
|
||||
mu sync.Mutex
|
||||
states map[string]string // hostID → last-seen privsep_healed_at
|
||||
customerOf map[string]string
|
||||
}
|
||||
|
||||
// NewHostMgmtPlaneChecker seeds per-host baselines from the latest reports. No events on init.
|
||||
func NewHostMgmtPlaneChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *HostMgmtPlaneChecker {
|
||||
mc := &HostMgmtPlaneChecker{
|
||||
store: s,
|
||||
logger: logger,
|
||||
onEvent: onEvent,
|
||||
states: make(map[string]string),
|
||||
customerOf: make(map[string]string),
|
||||
}
|
||||
rows, err := s.GetHostMgmtPlaneStates()
|
||||
if err != nil {
|
||||
logger.Printf("[WARN] Host mgmt-plane checker: failed to seed: %v", err)
|
||||
return mc
|
||||
}
|
||||
seeded := 0
|
||||
for _, row := range rows {
|
||||
if s.IsCustomerBlocked(row.CustomerID) || row.PrivsepHealedAt == "" {
|
||||
continue
|
||||
}
|
||||
mc.customerOf[row.HostID] = row.CustomerID
|
||||
mc.states[row.HostID] = row.PrivsepHealedAt
|
||||
seeded++
|
||||
}
|
||||
logger.Printf("[INFO] Host mgmt-plane checker initialized: %d host heal-state(s) seeded", seeded)
|
||||
return mc
|
||||
}
|
||||
|
||||
// Check evaluates all hosts and emits mgmt_plane_healed on a NEW heal timestamp.
|
||||
func (mc *HostMgmtPlaneChecker) Check() {
|
||||
rows, err := mc.store.GetHostMgmtPlaneStates()
|
||||
if err != nil {
|
||||
mc.logger.Printf("[WARN] Host mgmt-plane check failed: %v", err)
|
||||
return
|
||||
}
|
||||
mc.mu.Lock()
|
||||
defer mc.mu.Unlock()
|
||||
|
||||
seen := make(map[string]bool, len(rows))
|
||||
for _, row := range rows {
|
||||
if mc.store.IsCustomerBlocked(row.CustomerID) {
|
||||
delete(mc.states, row.HostID)
|
||||
continue
|
||||
}
|
||||
seen[row.HostID] = true
|
||||
if row.PrivsepHealedAt == "" {
|
||||
continue // no heal marker → healthy / old agent → no alert, no baseline change
|
||||
}
|
||||
mc.customerOf[row.HostID] = row.CustomerID
|
||||
old := mc.states[row.HostID]
|
||||
if old == "" {
|
||||
mc.states[row.HostID] = row.PrivsepHealedAt // first observation → seed, no event
|
||||
continue
|
||||
}
|
||||
if old == row.PrivsepHealedAt {
|
||||
continue // same heal already alerted
|
||||
}
|
||||
mc.states[row.HostID] = row.PrivsepHealedAt
|
||||
mc.emit(row.HostID, row.CustomerID, row.PrivsepHealedAt)
|
||||
}
|
||||
|
||||
for id := range mc.states {
|
||||
if !seen[id] {
|
||||
delete(mc.states, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetState returns the last-seen heal timestamp for a host ("" if none).
|
||||
func (mc *HostMgmtPlaneChecker) GetState(hostID string) string {
|
||||
mc.mu.Lock()
|
||||
defer mc.mu.Unlock()
|
||||
return mc.states[hostID]
|
||||
}
|
||||
|
||||
func (mc *HostMgmtPlaneChecker) emit(hostID, customerID, healedAt string) {
|
||||
msg := "Host " + hostID + ": the management-plane privsep dir (/run/sshd) was missing and was AUTO-HEALED by the watchdog at " + healedAt +
|
||||
" — a recurring cause can lead to an SSH lockout; investigate (e.g. a unit declaring RuntimeDirectory=sshd)."
|
||||
details, _ := json.Marshal(map[string]string{
|
||||
"host_id": hostID,
|
||||
"privsep_healed_at": healedAt,
|
||||
})
|
||||
mc.logger.Printf("[WARN] Host mgmt-plane: %s privsep dir auto-healed at %s (mgmt_plane_healed)", hostID, healedAt)
|
||||
if _, err := mc.store.SaveEvent(customerID, "mgmt_plane_healed", "warning", msg, string(details), "hub"); err != nil {
|
||||
mc.logger.Printf("[WARN] save mgmt_plane_healed for %s: %v", hostID, err)
|
||||
return
|
||||
}
|
||||
if mc.onEvent != nil {
|
||||
mc.onEvent(customerID, "mgmt_plane_healed", "warning", msg, string(details), "hub")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func reportWithHeal(healedAt string) []byte {
|
||||
if healedAt == "" {
|
||||
return []byte(`{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true}}`)
|
||||
}
|
||||
return []byte(`{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true,"healed_recently":true,"privsep_healed_at":"` + healedAt + `"}}`)
|
||||
}
|
||||
|
||||
func newMgmtStore(t *testing.T) *store.Store {
|
||||
t.Helper()
|
||||
st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"})
|
||||
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"})
|
||||
return st
|
||||
}
|
||||
|
||||
// A NEW heal timestamp fires exactly one mgmt_plane_healed; the first observation only seeds; a repeat
|
||||
// of the same timestamp does not re-fire. Companion TestHostMgmtPlaneChecker_NoHealNoEvent proves it's
|
||||
// the heal, not the sweep, that fires it (drop the emit → this test fails).
|
||||
func TestHostMgmtPlaneChecker_NewHealAlertsOnce(t *testing.T) {
|
||||
st := newMgmtStore(t)
|
||||
// first report already carries a heal marker → seed baseline, NO event on construction.
|
||||
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T16:42:17Z"), store.HostReportDenorm{})
|
||||
var events []string
|
||||
mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
||||
if mc.GetState("h1") != "2026-07-05T16:42:17Z" {
|
||||
t.Fatalf("seed = %q", mc.GetState("h1"))
|
||||
}
|
||||
if len(events) != 0 {
|
||||
t.Fatalf("construction must not emit, got %v", events)
|
||||
}
|
||||
|
||||
// a NEW heal (different timestamp) → one warning.
|
||||
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T18:00:00Z"), store.HostReportDenorm{})
|
||||
mc.Check()
|
||||
if len(events) != 1 || events[0] != "mgmt_plane_healed" {
|
||||
t.Fatalf("new heal → one mgmt_plane_healed, got %v", events)
|
||||
}
|
||||
// same timestamp again → no duplicate.
|
||||
mc.Check()
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("same heal must not re-emit, got %v", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostMgmtPlaneChecker_NoHealNoEvent(t *testing.T) {
|
||||
st := newMgmtStore(t)
|
||||
st.SaveHostReport("h1", "c1", reportWithHeal(""), store.HostReportDenorm{}) // healthy, no marker
|
||||
var events []string
|
||||
mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
||||
mc.Check()
|
||||
mc.Check()
|
||||
if len(events) != 0 {
|
||||
t.Fatalf("a healthy host (no heal marker) must never alert, got %v", events)
|
||||
}
|
||||
if mc.GetState("h1") != "" {
|
||||
t.Fatalf("no marker → no baseline, got %q", mc.GetState("h1"))
|
||||
}
|
||||
}
|
||||
|
||||
// A recurring clobber: heal at T1 (seed), heal again at T2 (alert), heal again at T3 (alert) — each
|
||||
// distinct heal surfaces, which is the whole point (find the recurring cause before a lockout).
|
||||
func TestHostMgmtPlaneChecker_RecurringHealsEachAlert(t *testing.T) {
|
||||
st := newMgmtStore(t)
|
||||
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T10:00:00Z"), store.HostReportDenorm{})
|
||||
var events []string
|
||||
mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
||||
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T11:00:00Z"), store.HostReportDenorm{})
|
||||
mc.Check()
|
||||
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T12:00:00Z"), store.HostReportDenorm{})
|
||||
mc.Check()
|
||||
if len(events) != 2 {
|
||||
t.Fatalf("two distinct new heals after seed → two alerts, got %d (%v)", len(events), events)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// HostRecoveryCredential is the break-glass PVE console credential for a host (TASK G1). Secret is
|
||||
// the root@pam password — a hub-held secret, operator-retrievable (NOT zero-knowledge like escrow).
|
||||
type HostRecoveryCredential struct {
|
||||
HostID string
|
||||
Username string
|
||||
Secret string
|
||||
SetAt time.Time
|
||||
}
|
||||
|
||||
// SaveHostRecoveryCredential upserts a host's break-glass credential (last-write-wins: day-0 sets it,
|
||||
// --rotate re-sets). The secret is stored as-is at rest; the hub NEVER logs it and only ever returns
|
||||
// it over the operator-authenticated retrieval path.
|
||||
func (s *Store) SaveHostRecoveryCredential(hostID, username, secret string) error {
|
||||
_, err := s.db.Exec(`
|
||||
INSERT INTO host_recovery (host_id, username, secret, set_at, updated_at)
|
||||
VALUES (?, ?, ?, datetime('now'), datetime('now'))
|
||||
ON CONFLICT(host_id) DO UPDATE SET
|
||||
username = excluded.username,
|
||||
secret = excluded.secret,
|
||||
updated_at = datetime('now')`,
|
||||
hostID, username, secret)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetHostRecoveryCredential returns a host's break-glass credential, or (nil, nil) if none is vaulted.
|
||||
func (s *Store) GetHostRecoveryCredential(hostID string) (*HostRecoveryCredential, error) {
|
||||
var c HostRecoveryCredential
|
||||
var setAt string
|
||||
err := s.db.QueryRow(
|
||||
`SELECT host_id, username, secret, set_at FROM host_recovery WHERE host_id = ?`, hostID).
|
||||
Scan(&c.HostID, &c.Username, &c.Secret, &setAt)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.SetAt = parseSQLiteTime(setAt)
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// HasHostRecoveryCredential reports whether a host already has a vaulted credential (day-0 idempotency:
|
||||
// don't regenerate/re-set on a re-run unless --rotate).
|
||||
func (s *Store) HasHostRecoveryCredential(hostID string) (bool, error) {
|
||||
var one int
|
||||
err := s.db.QueryRow(`SELECT 1 FROM host_recovery WHERE host_id = ?`, hostID).Scan(&one)
|
||||
if err == sql.ErrNoRows {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// HostMgmtPlaneRow is the latest management-plane state per host (TASK G1), parsed from the newest
|
||||
// host_report. PrivsepHealedAt is the watchdog heal-marker timestamp ("" when never healed / old agent).
|
||||
type HostMgmtPlaneRow struct {
|
||||
HostID string
|
||||
CustomerID string
|
||||
PrivsepDirOK bool
|
||||
SshdReachable bool
|
||||
PrivsepHealedAt string
|
||||
}
|
||||
|
||||
// GetHostMgmtPlaneStates returns the latest mgmt_plane stanza per host (mirrors
|
||||
// GetHostLeafFingerprints). A report without the stanza (old agent, feature off) yields zero values →
|
||||
// no alert. Malformed JSON degrades to zero values, never an error for that host.
|
||||
func (s *Store) GetHostMgmtPlaneStates() ([]HostMgmtPlaneRow, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT hr.host_id, hr.customer_id, hr.report_json
|
||||
FROM host_reports hr
|
||||
JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest
|
||||
ON hr.id = latest.mx`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []HostMgmtPlaneRow
|
||||
for rows.Next() {
|
||||
var r HostMgmtPlaneRow
|
||||
var reportJSON string
|
||||
if err := rows.Scan(&r.HostID, &r.CustomerID, &reportJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var body struct {
|
||||
MgmtPlane *struct {
|
||||
PrivsepDirOK bool `json:"privsep_dir_ok"`
|
||||
SshdReachable bool `json:"sshd_reachable"`
|
||||
PrivsepHealedAt string `json:"privsep_healed_at"`
|
||||
} `json:"mgmt_plane"`
|
||||
}
|
||||
_ = json.Unmarshal([]byte(reportJSON), &body) // malformed/old → nil mgmt_plane → zero values
|
||||
if body.MgmtPlane != nil {
|
||||
r.PrivsepDirOK = body.MgmtPlane.PrivsepDirOK
|
||||
r.SshdReachable = body.MgmtPlane.SshdReachable
|
||||
r.PrivsepHealedAt = body.MgmtPlane.PrivsepHealedAt
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package store
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestHostRecoveryCredential_RoundTripUpsertAndAbsent(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
if err := s.UpsertHost(&Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
|
||||
t.Fatalf("UpsertHost: %v", err)
|
||||
}
|
||||
|
||||
// absent → (nil, nil) + Has=false
|
||||
got, err := s.GetHostRecoveryCredential("h1")
|
||||
if err != nil || got != nil {
|
||||
t.Fatalf("absent cred: got %+v / %v (want nil,nil)", got, err)
|
||||
}
|
||||
has, _ := s.HasHostRecoveryCredential("h1")
|
||||
if has {
|
||||
t.Fatal("HasHostRecoveryCredential must be false before any vault")
|
||||
}
|
||||
|
||||
// vault → round-trips
|
||||
if err := s.SaveHostRecoveryCredential("h1", "root@pam", "s3cret-Aa1"); err != nil {
|
||||
t.Fatalf("SaveHostRecoveryCredential: %v", err)
|
||||
}
|
||||
got, err = s.GetHostRecoveryCredential("h1")
|
||||
if err != nil || got == nil {
|
||||
t.Fatalf("GetHostRecoveryCredential: %+v / %v", got, err)
|
||||
}
|
||||
if got.Username != "root@pam" || got.Secret != "s3cret-Aa1" {
|
||||
t.Fatalf("round-trip mismatch: %+v", got)
|
||||
}
|
||||
if has, _ := s.HasHostRecoveryCredential("h1"); !has {
|
||||
t.Fatal("HasHostRecoveryCredential must be true after vault")
|
||||
}
|
||||
|
||||
// upsert (rotate) → overwrites last-write-wins
|
||||
if err := s.SaveHostRecoveryCredential("h1", "root@pam", "rotated-Bb2"); err != nil {
|
||||
t.Fatalf("re-vault: %v", err)
|
||||
}
|
||||
got, _ = s.GetHostRecoveryCredential("h1")
|
||||
if got.Secret != "rotated-Bb2" {
|
||||
t.Fatalf("rotate did not overwrite: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetHostMgmtPlaneStates_ParsesHealMarker(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
if err := s.UpsertHost(&Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
|
||||
t.Fatalf("UpsertHost: %v", err)
|
||||
}
|
||||
// a report WITH a heal marker
|
||||
report := `{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true,"healed_recently":true,"privsep_healed_at":"2026-07-05T16:42:17Z"}}`
|
||||
if err := s.SaveHostReport("h1", "c1", []byte(report), HostReportDenorm{}); err != nil {
|
||||
t.Fatalf("SaveHostReport: %v", err)
|
||||
}
|
||||
rows, err := s.GetHostMgmtPlaneStates()
|
||||
if err != nil {
|
||||
t.Fatalf("GetHostMgmtPlaneStates: %v", err)
|
||||
}
|
||||
var found bool
|
||||
for _, r := range rows {
|
||||
if r.HostID == "h1" {
|
||||
found = true
|
||||
if !r.PrivsepDirOK || r.PrivsepHealedAt != "2026-07-05T16:42:17Z" {
|
||||
t.Fatalf("parsed row wrong: %+v", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("h1 not in mgmt-plane states")
|
||||
}
|
||||
|
||||
// a report WITHOUT the stanza (old agent) → zero values, no crash
|
||||
if err := s.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), HostReportDenorm{}); err != nil {
|
||||
t.Fatalf("SaveHostReport2: %v", err)
|
||||
}
|
||||
rows, _ = s.GetHostMgmtPlaneStates()
|
||||
for _, r := range rows {
|
||||
if r.HostID == "h1" && r.PrivsepHealedAt != "" {
|
||||
t.Fatalf("old-agent report should yield empty healed_at, got %+v", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -398,6 +398,24 @@ func (s *Store) migrate() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// host_recovery (TASK G1): the break-glass root@pam console credential, vaulted at rest and
|
||||
// operator-retrievable. UNLIKE host_escrow (opaque, hub-can't-open), this IS a hub-held secret the
|
||||
// operator retrieves to reach the PVE web console (pveproxy — a failure domain distinct from sshd)
|
||||
// when both the sshd path AND the agent-independent auto-heal have failed. One row per host,
|
||||
// last-write-wins (day-0 sets it; --rotate re-sets). Never in desired-state, never logged.
|
||||
_, err = s.db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS host_recovery (
|
||||
host_id TEXT PRIMARY KEY,
|
||||
username TEXT NOT NULL,
|
||||
secret TEXT NOT NULL,
|
||||
set_at DATETIME NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at DATETIME NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user