0635640848
Phase-1 of SPIKE-infra-backup-2026-06-15. The infra-backup mechanism was dead since slice 8C yet stored plaintext customer secrets at rest (app-secret key, restic password, Cloudflare tokens) — a zero-knowledge violation — and its absence made the daily expected_backup_missed email fire for healthy customers. - Repoint monitor.CheckBackupDeadlines backup half to the agent host-report's PBS snapshots (+vzdump): alarm only on no-backup / >26h stale / verify failed. Keep the db_dump half. No host-report → no backup alarm (liveness owns that). New store.GetLatestHostReportJSON. Tests incl. a companion that fails pre-fix. - Remove the infra-backup endpoints, store methods/types, and operator panel; /recovery now returns config_yaml only. - migrate(): DROP infra_backup_versions/infra_backups + VACUUM (+wal_checkpoint) to physically reclaim the plaintext pages, gated on table existence. Flagged out-of-scope: exposed creds need operator rotation; legacy reports table holds historical plaintext restic_password rows (separate leak, not purged here). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
202 lines
7.6 KiB
Go
202 lines
7.6 KiB
Go
package monitor
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"log"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// newDeadlineStore creates an isolated store with one active customer + a host row.
|
|
func newDeadlineStore(t *testing.T) *store.Store {
|
|
t.Helper()
|
|
st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatalf("store.New: %v", err)
|
|
}
|
|
t.Cleanup(func() { st.Close() })
|
|
if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"}); err != nil {
|
|
t.Fatalf("SaveCustomerConfig: %v", err)
|
|
}
|
|
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
|
|
t.Fatalf("UpsertHost: %v", err)
|
|
}
|
|
return st
|
|
}
|
|
|
|
// hostReportJSON builds a host-report payload with the given PBS snapshots and vzdump
|
|
// backups. Each snapshot is {backup_time, verify_state}; each backup is {started_at, success}.
|
|
func hostReportJSON(t *testing.T, pbs [][2]string, backups []struct {
|
|
at string
|
|
ok bool
|
|
}) string {
|
|
t.Helper()
|
|
type snap struct {
|
|
BackupTime string `json:"backup_time"`
|
|
VerifyState string `json:"verify_state"`
|
|
}
|
|
type bk struct {
|
|
StartedAt string `json:"started_at"`
|
|
Success bool `json:"success"`
|
|
}
|
|
payload := struct {
|
|
HostID string `json:"host_id"`
|
|
PBSSnapshots []snap `json:"pbs_snapshots"`
|
|
Backups []bk `json:"backups"`
|
|
}{HostID: "h1"}
|
|
for _, p := range pbs {
|
|
payload.PBSSnapshots = append(payload.PBSSnapshots, snap{BackupTime: p[0], VerifyState: p[1]})
|
|
}
|
|
for _, b := range backups {
|
|
payload.Backups = append(payload.Backups, bk{StartedAt: b.at, Success: b.ok})
|
|
}
|
|
out, err := json.Marshal(payload)
|
|
if err != nil {
|
|
t.Fatalf("marshal report: %v", err)
|
|
}
|
|
return string(out)
|
|
}
|
|
|
|
// runDeadline runs CheckBackupDeadlines and returns the list of emitted event types for c1.
|
|
func runDeadline(t *testing.T, st *store.Store) []string {
|
|
t.Helper()
|
|
var got []string
|
|
onEvent := func(customerID, eventType, severity, message, detailsJSON, source string) {
|
|
if customerID == "c1" {
|
|
got = append(got, eventType)
|
|
}
|
|
}
|
|
// nil staleness → no "down" skip; the check evaluates c1.
|
|
CheckBackupDeadlines(st, nil, onEvent, log.New(io.Discard, "", 0))
|
|
return got
|
|
}
|
|
|
|
func has(events []string, t string) bool {
|
|
for _, e := range events {
|
|
if e == t {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func rfc(d time.Duration) string {
|
|
return time.Now().UTC().Add(d).Format(time.RFC3339)
|
|
}
|
|
|
|
// TestCheckBackupDeadlines_FreshVerifiedPBS_NoBackupAlarm is the COMPANION test.
|
|
// It FAILS against the pre-fix (event-based) check: with a fresh, verified PBS snapshot
|
|
// but no backup_completed event, the old code emitted expected_backup_missed anyway.
|
|
// The repoint reads the host-report instead, so a healthy customer raises no alarm.
|
|
func TestCheckBackupDeadlines_FreshVerifiedPBS_NoBackupAlarm(t *testing.T) {
|
|
st := newDeadlineStore(t)
|
|
// Make the db-dump half pass too, so the only thing under test is the backup half.
|
|
if _, err := st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
report := hostReportJSON(t, [][2]string{{rfc(-3 * time.Hour), "ok"}}, nil)
|
|
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
got := runDeadline(t, st)
|
|
if has(got, "expected_backup_missed") {
|
|
t.Fatalf("fresh+verified PBS must NOT raise expected_backup_missed; got %v", got)
|
|
}
|
|
if has(got, "expected_dbdump_missed") {
|
|
t.Fatalf("db_dump_completed present → no dbdump alarm expected; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestCheckBackupDeadlines_StalePBS_Alarms: newest backup older than 26h → alarm.
|
|
func TestCheckBackupDeadlines_StalePBS_Alarms(t *testing.T) {
|
|
st := newDeadlineStore(t)
|
|
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
|
|
report := hostReportJSON(t, [][2]string{{rfc(-30 * time.Hour), "ok"}}, nil)
|
|
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := runDeadline(t, st)
|
|
if !has(got, "expected_backup_missed") {
|
|
t.Fatalf("stale (>26h) PBS snapshot must raise expected_backup_missed; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestCheckBackupDeadlines_FailedVerify_Alarms: fresh snapshot but verify failed → alarm.
|
|
func TestCheckBackupDeadlines_FailedVerify_Alarms(t *testing.T) {
|
|
st := newDeadlineStore(t)
|
|
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
|
|
report := hostReportJSON(t, [][2]string{{rfc(-2 * time.Hour), "failed"}}, nil)
|
|
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := runDeadline(t, st)
|
|
if !has(got, "expected_backup_missed") {
|
|
t.Fatalf("failed PBS verify must raise expected_backup_missed; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestCheckBackupDeadlines_NoHostReport_NoBackupAlarm: a customer with no host-report
|
|
// (legacy/defunct controller-only) must NOT get a backup alarm from this check —
|
|
// liveness is the staleness checker's job.
|
|
func TestCheckBackupDeadlines_NoHostReport_NoBackupAlarm(t *testing.T) {
|
|
st := newDeadlineStore(t)
|
|
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
|
|
// No SaveHostReport call.
|
|
got := runDeadline(t, st)
|
|
if has(got, "expected_backup_missed") {
|
|
t.Fatalf("no host-report → no backup alarm; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestCheckBackupDeadlines_DbDumpHalfPreserved: fresh backup (no backup alarm) but a
|
|
// missing db_dump_completed event must still raise expected_dbdump_missed.
|
|
func TestCheckBackupDeadlines_DbDumpHalfPreserved(t *testing.T) {
|
|
st := newDeadlineStore(t)
|
|
report := hostReportJSON(t, [][2]string{{rfc(-2 * time.Hour), "ok"}}, nil)
|
|
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := runDeadline(t, st)
|
|
if has(got, "expected_backup_missed") {
|
|
t.Fatalf("fresh backup → no backup alarm; got %v", got)
|
|
}
|
|
if !has(got, "expected_dbdump_missed") {
|
|
t.Fatalf("missing db_dump_completed must still raise expected_dbdump_missed; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestAssessBackupFreshness exercises the pure freshness policy directly.
|
|
func TestAssessBackupFreshness(t *testing.T) {
|
|
now := time.Date(2026, 6, 16, 3, 0, 0, 0, time.UTC)
|
|
at := func(d time.Duration) string { return now.Add(d).Format(time.RFC3339) }
|
|
|
|
cases := []struct {
|
|
name string
|
|
report string
|
|
wantMissed bool
|
|
}{
|
|
{"fresh verified PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"ok"}]}`, false},
|
|
{"fresh unverified (none) is not a failure", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"none"}]}`, false},
|
|
{"stale verified", `{"pbs_snapshots":[{"backup_time":"` + at(-30*time.Hour) + `","verify_state":"ok"}]}`, true},
|
|
{"fresh but verify failed", `{"pbs_snapshots":[{"backup_time":"` + at(-2*time.Hour) + `","verify_state":"failed"}]}`, true},
|
|
{"no snapshots and no backups", `{"pbs_snapshots":[],"backups":[]}`, true},
|
|
{"vzdump fallback fresh success", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":true}]}`, false},
|
|
{"vzdump only, failed → counts as none", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":false}]}`, true},
|
|
{"newest vzdump fresh rescues stale PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-40*time.Hour) + `","verify_state":"ok"}],"backups":[{"started_at":"` + at(-2*time.Hour) + `","success":true}]}`, false},
|
|
{"unparseable report", `not json`, true},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
got := assessBackupFreshness(c.report, now)
|
|
if got.missed != c.wantMissed {
|
|
t.Fatalf("missed=%v want=%v (reason=%q)", got.missed, c.wantMissed, got.reason)
|
|
}
|
|
})
|
|
}
|
|
}
|