admin
05d81810d4
Hub half of the management-plane break-glass (prereq for felhom-sshd/H1; agent
half = felhom-agent v0.71.0). Closes SPIKE-felhom-sshd §8/#9.
- store.host_recovery + methods: per-host root@pam console password, at-rest,
operator-retrievable (the PVE-web-console fallback when sshd + auto-heal both fail).
- API: PUT /hosts/{id}/recovery-credential (self-scoped, day-0 vaults) + GET
/admin/hosts/{id}/recovery-credential (global key only). Secret never logged
(red-proofed).
- monitor/host_mgmtplane: parses the agent mgmt_plane stanza, raises
mgmt_plane_healed WARNING on a new privsep_healed_at (recurring clobber surfaces
before lockout; complements host_staleness).
- host-install: step_break_glass generates a strong root@pam password (openssl
rand, never logged/filed — stdin to chpasswd + curl), vaults via host key;
idempotent unless --rotate-recovery. Installs the G1 host artifacts (tmpfiles +
agent-independent watchdog timer), RuntimeDirectory-guarded; uninstall removes them.
Hub v0.34.0. Non-hollow tests + red-proofs; full suite green.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
91 lines
3.6 KiB
Go
91 lines
3.6 KiB
Go
package monitor
|
|
|
|
import (
|
|
"io"
|
|
"log"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
func reportWithHeal(healedAt string) []byte {
|
|
if healedAt == "" {
|
|
return []byte(`{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true}}`)
|
|
}
|
|
return []byte(`{"host_id":"h1","mgmt_plane":{"privsep_dir_ok":true,"sshd_reachable":true,"healed_recently":true,"privsep_healed_at":"` + healedAt + `"}}`)
|
|
}
|
|
|
|
func newMgmtStore(t *testing.T) *store.Store {
|
|
t.Helper()
|
|
st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { st.Close() })
|
|
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"})
|
|
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"})
|
|
return st
|
|
}
|
|
|
|
// A NEW heal timestamp fires exactly one mgmt_plane_healed; the first observation only seeds; a repeat
|
|
// of the same timestamp does not re-fire. Companion TestHostMgmtPlaneChecker_NoHealNoEvent proves it's
|
|
// the heal, not the sweep, that fires it (drop the emit → this test fails).
|
|
func TestHostMgmtPlaneChecker_NewHealAlertsOnce(t *testing.T) {
|
|
st := newMgmtStore(t)
|
|
// first report already carries a heal marker → seed baseline, NO event on construction.
|
|
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T16:42:17Z"), store.HostReportDenorm{})
|
|
var events []string
|
|
mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
|
if mc.GetState("h1") != "2026-07-05T16:42:17Z" {
|
|
t.Fatalf("seed = %q", mc.GetState("h1"))
|
|
}
|
|
if len(events) != 0 {
|
|
t.Fatalf("construction must not emit, got %v", events)
|
|
}
|
|
|
|
// a NEW heal (different timestamp) → one warning.
|
|
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T18:00:00Z"), store.HostReportDenorm{})
|
|
mc.Check()
|
|
if len(events) != 1 || events[0] != "mgmt_plane_healed" {
|
|
t.Fatalf("new heal → one mgmt_plane_healed, got %v", events)
|
|
}
|
|
// same timestamp again → no duplicate.
|
|
mc.Check()
|
|
if len(events) != 1 {
|
|
t.Fatalf("same heal must not re-emit, got %v", events)
|
|
}
|
|
}
|
|
|
|
func TestHostMgmtPlaneChecker_NoHealNoEvent(t *testing.T) {
|
|
st := newMgmtStore(t)
|
|
st.SaveHostReport("h1", "c1", reportWithHeal(""), store.HostReportDenorm{}) // healthy, no marker
|
|
var events []string
|
|
mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
|
mc.Check()
|
|
mc.Check()
|
|
if len(events) != 0 {
|
|
t.Fatalf("a healthy host (no heal marker) must never alert, got %v", events)
|
|
}
|
|
if mc.GetState("h1") != "" {
|
|
t.Fatalf("no marker → no baseline, got %q", mc.GetState("h1"))
|
|
}
|
|
}
|
|
|
|
// A recurring clobber: heal at T1 (seed), heal again at T2 (alert), heal again at T3 (alert) — each
|
|
// distinct heal surfaces, which is the whole point (find the recurring cause before a lockout).
|
|
func TestHostMgmtPlaneChecker_RecurringHealsEachAlert(t *testing.T) {
|
|
st := newMgmtStore(t)
|
|
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T10:00:00Z"), store.HostReportDenorm{})
|
|
var events []string
|
|
mc := NewHostMgmtPlaneChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
|
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T11:00:00Z"), store.HostReportDenorm{})
|
|
mc.Check()
|
|
st.SaveHostReport("h1", "c1", reportWithHeal("2026-07-05T12:00:00Z"), store.HostReportDenorm{})
|
|
mc.Check()
|
|
if len(events) != 2 {
|
|
t.Fatalf("two distinct new heals after seed → two alerts, got %d (%v)", len(events), events)
|
|
}
|
|
}
|