Files
felhom.eu/hub/internal/monitor/deadline_test.go
T
admin 0635640848 hub v0.12.0: retire Infra Backup, purge its plaintext secrets, fix backup-deadline email
Phase-1 of SPIKE-infra-backup-2026-06-15. The infra-backup mechanism was dead
since slice 8C yet stored plaintext customer secrets at rest (app-secret key,
restic password, Cloudflare tokens) — a zero-knowledge violation — and its
absence made the daily expected_backup_missed email fire for healthy customers.

- Repoint monitor.CheckBackupDeadlines backup half to the agent host-report's
  PBS snapshots (+vzdump): alarm only on no-backup / >26h stale / verify failed.
  Keep the db_dump half. No host-report → no backup alarm (liveness owns that).
  New store.GetLatestHostReportJSON. Tests incl. a companion that fails pre-fix.
- Remove the infra-backup endpoints, store methods/types, and operator panel;
  /recovery now returns config_yaml only.
- migrate(): DROP infra_backup_versions/infra_backups + VACUUM (+wal_checkpoint)
  to physically reclaim the plaintext pages, gated on table existence.

Flagged out-of-scope: exposed creds need operator rotation; legacy reports table
holds historical plaintext restic_password rows (separate leak, not purged here).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-16 11:08:06 +02:00

202 lines
7.6 KiB
Go

package monitor
import (
"encoding/json"
"io"
"log"
"path/filepath"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
// newDeadlineStore creates an isolated store with one active customer + a host row.
func newDeadlineStore(t *testing.T) *store.Store {
t.Helper()
st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
if err != nil {
t.Fatalf("store.New: %v", err)
}
t.Cleanup(func() { st.Close() })
if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"}); err != nil {
t.Fatalf("SaveCustomerConfig: %v", err)
}
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
t.Fatalf("UpsertHost: %v", err)
}
return st
}
// hostReportJSON builds a host-report payload with the given PBS snapshots and vzdump
// backups. Each snapshot is {backup_time, verify_state}; each backup is {started_at, success}.
func hostReportJSON(t *testing.T, pbs [][2]string, backups []struct {
at string
ok bool
}) string {
t.Helper()
type snap struct {
BackupTime string `json:"backup_time"`
VerifyState string `json:"verify_state"`
}
type bk struct {
StartedAt string `json:"started_at"`
Success bool `json:"success"`
}
payload := struct {
HostID string `json:"host_id"`
PBSSnapshots []snap `json:"pbs_snapshots"`
Backups []bk `json:"backups"`
}{HostID: "h1"}
for _, p := range pbs {
payload.PBSSnapshots = append(payload.PBSSnapshots, snap{BackupTime: p[0], VerifyState: p[1]})
}
for _, b := range backups {
payload.Backups = append(payload.Backups, bk{StartedAt: b.at, Success: b.ok})
}
out, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal report: %v", err)
}
return string(out)
}
// runDeadline runs CheckBackupDeadlines and returns the list of emitted event types for c1.
func runDeadline(t *testing.T, st *store.Store) []string {
t.Helper()
var got []string
onEvent := func(customerID, eventType, severity, message, detailsJSON, source string) {
if customerID == "c1" {
got = append(got, eventType)
}
}
// nil staleness → no "down" skip; the check evaluates c1.
CheckBackupDeadlines(st, nil, onEvent, log.New(io.Discard, "", 0))
return got
}
func has(events []string, t string) bool {
for _, e := range events {
if e == t {
return true
}
}
return false
}
func rfc(d time.Duration) string {
return time.Now().UTC().Add(d).Format(time.RFC3339)
}
// TestCheckBackupDeadlines_FreshVerifiedPBS_NoBackupAlarm is the COMPANION test.
// It FAILS against the pre-fix (event-based) check: with a fresh, verified PBS snapshot
// but no backup_completed event, the old code emitted expected_backup_missed anyway.
// The repoint reads the host-report instead, so a healthy customer raises no alarm.
func TestCheckBackupDeadlines_FreshVerifiedPBS_NoBackupAlarm(t *testing.T) {
st := newDeadlineStore(t)
// Make the db-dump half pass too, so the only thing under test is the backup half.
if _, err := st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller"); err != nil {
t.Fatal(err)
}
report := hostReportJSON(t, [][2]string{{rfc(-3 * time.Hour), "ok"}}, nil)
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
t.Fatal(err)
}
got := runDeadline(t, st)
if has(got, "expected_backup_missed") {
t.Fatalf("fresh+verified PBS must NOT raise expected_backup_missed; got %v", got)
}
if has(got, "expected_dbdump_missed") {
t.Fatalf("db_dump_completed present → no dbdump alarm expected; got %v", got)
}
}
// TestCheckBackupDeadlines_StalePBS_Alarms: newest backup older than 26h → alarm.
func TestCheckBackupDeadlines_StalePBS_Alarms(t *testing.T) {
st := newDeadlineStore(t)
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
report := hostReportJSON(t, [][2]string{{rfc(-30 * time.Hour), "ok"}}, nil)
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
t.Fatal(err)
}
got := runDeadline(t, st)
if !has(got, "expected_backup_missed") {
t.Fatalf("stale (>26h) PBS snapshot must raise expected_backup_missed; got %v", got)
}
}
// TestCheckBackupDeadlines_FailedVerify_Alarms: fresh snapshot but verify failed → alarm.
func TestCheckBackupDeadlines_FailedVerify_Alarms(t *testing.T) {
st := newDeadlineStore(t)
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
report := hostReportJSON(t, [][2]string{{rfc(-2 * time.Hour), "failed"}}, nil)
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
t.Fatal(err)
}
got := runDeadline(t, st)
if !has(got, "expected_backup_missed") {
t.Fatalf("failed PBS verify must raise expected_backup_missed; got %v", got)
}
}
// TestCheckBackupDeadlines_NoHostReport_NoBackupAlarm: a customer with no host-report
// (legacy/defunct controller-only) must NOT get a backup alarm from this check —
// liveness is the staleness checker's job.
func TestCheckBackupDeadlines_NoHostReport_NoBackupAlarm(t *testing.T) {
st := newDeadlineStore(t)
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
// No SaveHostReport call.
got := runDeadline(t, st)
if has(got, "expected_backup_missed") {
t.Fatalf("no host-report → no backup alarm; got %v", got)
}
}
// TestCheckBackupDeadlines_DbDumpHalfPreserved: fresh backup (no backup alarm) but a
// missing db_dump_completed event must still raise expected_dbdump_missed.
func TestCheckBackupDeadlines_DbDumpHalfPreserved(t *testing.T) {
st := newDeadlineStore(t)
report := hostReportJSON(t, [][2]string{{rfc(-2 * time.Hour), "ok"}}, nil)
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
t.Fatal(err)
}
got := runDeadline(t, st)
if has(got, "expected_backup_missed") {
t.Fatalf("fresh backup → no backup alarm; got %v", got)
}
if !has(got, "expected_dbdump_missed") {
t.Fatalf("missing db_dump_completed must still raise expected_dbdump_missed; got %v", got)
}
}
// TestAssessBackupFreshness exercises the pure freshness policy directly.
func TestAssessBackupFreshness(t *testing.T) {
now := time.Date(2026, 6, 16, 3, 0, 0, 0, time.UTC)
at := func(d time.Duration) string { return now.Add(d).Format(time.RFC3339) }
cases := []struct {
name string
report string
wantMissed bool
}{
{"fresh verified PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"ok"}]}`, false},
{"fresh unverified (none) is not a failure", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"none"}]}`, false},
{"stale verified", `{"pbs_snapshots":[{"backup_time":"` + at(-30*time.Hour) + `","verify_state":"ok"}]}`, true},
{"fresh but verify failed", `{"pbs_snapshots":[{"backup_time":"` + at(-2*time.Hour) + `","verify_state":"failed"}]}`, true},
{"no snapshots and no backups", `{"pbs_snapshots":[],"backups":[]}`, true},
{"vzdump fallback fresh success", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":true}]}`, false},
{"vzdump only, failed → counts as none", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":false}]}`, true},
{"newest vzdump fresh rescues stale PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-40*time.Hour) + `","verify_state":"ok"}],"backups":[{"started_at":"` + at(-2*time.Hour) + `","success":true}]}`, false},
{"unparseable report", `not json`, true},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := assessBackupFreshness(c.report, now)
if got.missed != c.wantMissed {
t.Fatalf("missed=%v want=%v (reason=%q)", got.missed, c.wantMissed, got.reason)
}
})
}
}