hub v0.12.0: retire Infra Backup, purge its plaintext secrets, fix backup-deadline email

Phase-1 of SPIKE-infra-backup-2026-06-15. The infra-backup mechanism was dead since slice 8C yet stored plaintext customer secrets at rest (app-secret key, restic password, Cloudflare tokens) — a zero-knowledge violation — and its absence made the daily expected_backup_missed email fire for healthy customers. - Repoint monitor.CheckBackupDeadlines backup half to the agent host-report's PBS snapshots (+vzdump): alarm only on no-backup / >26h stale / verify failed. Keep the db_dump half. No host-report → no backup alarm (liveness owns that). New store.GetLatestHostReportJSON. Tests incl. a companion that fails pre-fix. - Remove the infra-backup endpoints, store methods/types, and operator panel; /recovery now returns config_yaml only. - migrate(): DROP infra_backup_versions/infra_backups + VACUUM (+wal_checkpoint) to physically reclaim the plaintext pages, gated on table existence. Flagged out-of-scope: exposed creds need operator rotation; legacy reports table holds historical plaintext restic_password rows (separate leak, not purged here). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-16 11:08:06 +02:00
parent 2f7acb7d07
commit 0635640848
10 changed files with 466 additions and 595 deletions
@@ -1,12 +1,120 @@
 package monitor

 import (
+	"encoding/json"
+	"fmt"
 	"log"
+	"strings"
 	"time"

 	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
 )

+// backupStaleAfter is the maximum age of the newest offsite backup before the daily
+// deadline check raises expected_backup_missed. 26h covers an evening backup schedule
+// (e.g. ~18:00–22:00) plus headroom, so a healthy once-daily cadence never trips the
+// early-morning check.
+const backupStaleAfter = 26 * time.Hour
+
+// hostReportBackups is the minimal slice of an agent host-report the deadline check
+// reads to judge backup freshness (pbs_snapshots is the offsite-DR signal; backups is
+// the local vzdump fallback). Mirrors the agent's hub.PBSSnapshot / hub.Backup wire
+// contract for just the fields we need.
+type hostReportBackups struct {
+	PBSSnapshots []struct {
+		BackupTime  string `json:"backup_time"`
+		VerifyState string `json:"verify_state"`
+	} `json:"pbs_snapshots"`
+	Backups []struct {
+		StartedAt string `json:"started_at"`
+		Success   bool   `json:"success"`
+	} `json:"backups"`
+}
+
+// backupAssessment is the verdict for one customer's offsite backup health.
+type backupAssessment struct {
+	missed bool   // raise expected_backup_missed
+	reason string // human-readable cause (event message + logs)
+}
+
+// assessBackupFreshness decides whether a customer's latest host-report shows a healthy,
+// recent backup. Pure (now is injected) so the policy is unit-tested. Only POSITIVE
+// evidence of a problem fires an alarm:
+//   - no PBS snapshot AND no successful vzdump in the report  → missed ("no backup recorded")
+//   - newest backup older than backupStaleAfter               → missed ("stale")
+//   - the newest PBS snapshot's verify_state is "failed"      → missed ("verify failed")
+//
+// A fresh-but-not-yet-verified snapshot (verify_state "none"/"") is NOT treated as a
+// failure: PBS verification runs on its own cadence, so a snapshot taken hours before the
+// 03:00 check may legitimately be unverified. Alarming on that would re-introduce exactly
+// the daily false alarm this repoint removes (hence "failed" only, not "≠ ok").
+func assessBackupFreshness(reportJSON string, now time.Time) backupAssessment {
+	var hr hostReportBackups
+	if err := json.Unmarshal([]byte(reportJSON), &hr); err != nil {
+		// Unparseable report → can't confirm a backup. Surface it rather than swallow it.
+		return backupAssessment{missed: true, reason: "latest host-report could not be parsed"}
+	}
+
+	var newestPBS time.Time
+	var newestPBSVerify string
+	havePBS := false
+	for _, ps := range hr.PBSSnapshots {
+		t, ok := parseBackupTime(ps.BackupTime)
+		if !ok {
+			continue
+		}
+		if !havePBS || t.After(newestPBS) {
+			havePBS = true
+			newestPBS = t
+			newestPBSVerify = strings.ToLower(strings.TrimSpace(ps.VerifyState))
+		}
+	}
+
+	var newestVzdump time.Time
+	haveVzdump := false
+	for _, b := range hr.Backups {
+		if !b.Success {
+			continue
+		}
+		t, ok := parseBackupTime(b.StartedAt)
+		if !ok {
+			continue
+		}
+		if !haveVzdump || t.After(newestVzdump) {
+			haveVzdump = true
+			newestVzdump = t
+		}
+	}
+
+	if !havePBS && !haveVzdump {
+		return backupAssessment{missed: true, reason: "no PBS snapshot or successful backup in the latest host-report"}
+	}
+
+	newest := newestPBS
+	if haveVzdump && (!havePBS || newestVzdump.After(newest)) {
+		newest = newestVzdump
+	}
+	if age := now.Sub(newest); age > backupStaleAfter {
+		return backupAssessment{missed: true, reason: fmt.Sprintf("newest backup is %s old (limit %s)", age.Round(time.Hour), backupStaleAfter)}
+	}
+	if havePBS && newestPBSVerify == "failed" {
+		return backupAssessment{missed: true, reason: "newest PBS snapshot failed verification"}
+	}
+	return backupAssessment{missed: false}
+}
+
+// parseBackupTime parses an RFC3339 timestamp from a host-report and normalizes to UTC.
+func parseBackupTime(s string) (time.Time, bool) {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return time.Time{}, false
+	}
+	if t, err := time.Parse(time.RFC3339, s); err == nil {
+		return t.UTC(), true
+	}
+	return time.Time{}, false
+}
+
 // budapest returns the Europe/Budapest timezone (cached).
 var budapest *time.Location

@@ -20,11 +128,16 @@ func init() {
 }

 // CheckBackupDeadlines checks whether active customers had their expected
-// daily backups and DB dumps. Runs once daily at 05:00 Budapest time.
+// daily backups and DB dumps. Runs once daily (early morning, Budapest time).
 //
-// For each active customer, it checks for backup_completed and db_dump_completed
-// events since Budapest midnight. If neither success nor failure events exist,
-// it inserts expected_backup_missed / expected_dbdump_missed events.
+// Backup half: read the agent's latest host-report and raise expected_backup_missed
+// only when its PBS snapshots / vzdump backups show no fresh, verified backup (see
+// assessBackupFreshness). This replaced the old backup_completed-event check, which
+// fired daily for every healthy customer because no component emits that event anymore
+// (the controller's disk-tier backup moved to the agent in slice 8C).
+//
+// DB-dump half: unchanged — the in-guest controller still emits db_dump_completed, so
+// the event-based check there is correct.
 //
 // Customers whose nodes are "down" (no report in >1h) are skipped — they
 // already have staleness events.
@@ -54,17 +167,27 @@ func CheckBackupDeadlines(s *store.Store, staleness *StalenessChecker, onEvent E
 			continue
 		}

-		// Check backup_completed / backup_failed since midnight
-		backupOK, _ := s.GetEventsByType(id, "backup_completed", sinceUTC)
-		backupFailed, _ := s.GetEventsByType(id, "backup_failed", sinceUTC)
-		if len(backupOK) == 0 && len(backupFailed) == 0 {
-			msg := "No backup completed or failed since midnight"
-			if _, err := s.SaveEvent(id, "expected_backup_missed", "error", msg, "{}", "hub"); err != nil {
-				logger.Printf("[WARN] Failed to save expected_backup_missed for %s: %v", id, err)
-			} else if onEvent != nil {
-				onEvent(id, "expected_backup_missed", "error", msg, "{}", "hub")
+		// Backup freshness from the agent's host-report (PBS snapshots + vzdump),
+		// the authoritative offsite-backup signal post-slice-8C.
+		reportJSON, rerr := s.GetLatestHostReportJSON(id)
+		switch {
+		case rerr != nil:
+			logger.Printf("[WARN] Deadline check: failed to read host-report for %s: %v", id, rerr)
+		case reportJSON == "":
+			// No agent host-report at all (legacy/defunct controller-only customer).
+			// Liveness is owned by the host-staleness checker; the backup deadline check
+			// has no PBS data to judge here and must not emit a daily backup alarm of its
+			// own. (The DB-dump half below still applies.)
+		default:
+			if a := assessBackupFreshness(reportJSON, time.Now().UTC()); a.missed {
+				msg := "No fresh verified backup: " + a.reason
+				if _, err := s.SaveEvent(id, "expected_backup_missed", "error", msg, "{}", "hub"); err != nil {
+					logger.Printf("[WARN] Failed to save expected_backup_missed for %s: %v", id, err)
+				} else if onEvent != nil {
+					onEvent(id, "expected_backup_missed", "error", msg, "{}", "hub")
+				}
+				backupMissed++
 			}
-			backupMissed++
 		}

 		// Check db_dump_completed / db_dump_failed since midnight
@@ -0,0 +1,201 @@
+package monitor
+
+import (
+	"encoding/json"
+	"io"
+	"log"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// newDeadlineStore creates an isolated store with one active customer + a host row.
+func newDeadlineStore(t *testing.T) *store.Store {
+	t.Helper()
+	st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
+	if err != nil {
+		t.Fatalf("store.New: %v", err)
+	}
+	t.Cleanup(func() { st.Close() })
+	if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"}); err != nil {
+		t.Fatalf("SaveCustomerConfig: %v", err)
+	}
+	if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
+		t.Fatalf("UpsertHost: %v", err)
+	}
+	return st
+}
+
+// hostReportJSON builds a host-report payload with the given PBS snapshots and vzdump
+// backups. Each snapshot is {backup_time, verify_state}; each backup is {started_at, success}.
+func hostReportJSON(t *testing.T, pbs [][2]string, backups []struct {
+	at string
+	ok bool
+}) string {
+	t.Helper()
+	type snap struct {
+		BackupTime  string `json:"backup_time"`
+		VerifyState string `json:"verify_state"`
+	}
+	type bk struct {
+		StartedAt string `json:"started_at"`
+		Success   bool   `json:"success"`
+	}
+	payload := struct {
+		HostID       string `json:"host_id"`
+		PBSSnapshots []snap `json:"pbs_snapshots"`
+		Backups      []bk   `json:"backups"`
+	}{HostID: "h1"}
+	for _, p := range pbs {
+		payload.PBSSnapshots = append(payload.PBSSnapshots, snap{BackupTime: p[0], VerifyState: p[1]})
+	}
+	for _, b := range backups {
+		payload.Backups = append(payload.Backups, bk{StartedAt: b.at, Success: b.ok})
+	}
+	out, err := json.Marshal(payload)
+	if err != nil {
+		t.Fatalf("marshal report: %v", err)
+	}
+	return string(out)
+}
+
+// runDeadline runs CheckBackupDeadlines and returns the list of emitted event types for c1.
+func runDeadline(t *testing.T, st *store.Store) []string {
+	t.Helper()
+	var got []string
+	onEvent := func(customerID, eventType, severity, message, detailsJSON, source string) {
+		if customerID == "c1" {
+			got = append(got, eventType)
+		}
+	}
+	// nil staleness → no "down" skip; the check evaluates c1.
+	CheckBackupDeadlines(st, nil, onEvent, log.New(io.Discard, "", 0))
+	return got
+}
+
+func has(events []string, t string) bool {
+	for _, e := range events {
+		if e == t {
+			return true
+		}
+	}
+	return false
+}
+
+func rfc(d time.Duration) string {
+	return time.Now().UTC().Add(d).Format(time.RFC3339)
+}
+
+// TestCheckBackupDeadlines_FreshVerifiedPBS_NoBackupAlarm is the COMPANION test.
+// It FAILS against the pre-fix (event-based) check: with a fresh, verified PBS snapshot
+// but no backup_completed event, the old code emitted expected_backup_missed anyway.
+// The repoint reads the host-report instead, so a healthy customer raises no alarm.
+func TestCheckBackupDeadlines_FreshVerifiedPBS_NoBackupAlarm(t *testing.T) {
+	st := newDeadlineStore(t)
+	// Make the db-dump half pass too, so the only thing under test is the backup half.
+	if _, err := st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller"); err != nil {
+		t.Fatal(err)
+	}
+	report := hostReportJSON(t, [][2]string{{rfc(-3 * time.Hour), "ok"}}, nil)
+	if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
+		t.Fatal(err)
+	}
+
+	got := runDeadline(t, st)
+	if has(got, "expected_backup_missed") {
+		t.Fatalf("fresh+verified PBS must NOT raise expected_backup_missed; got %v", got)
+	}
+	if has(got, "expected_dbdump_missed") {
+		t.Fatalf("db_dump_completed present → no dbdump alarm expected; got %v", got)
+	}
+}
+
+// TestCheckBackupDeadlines_StalePBS_Alarms: newest backup older than 26h → alarm.
+func TestCheckBackupDeadlines_StalePBS_Alarms(t *testing.T) {
+	st := newDeadlineStore(t)
+	st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
+	report := hostReportJSON(t, [][2]string{{rfc(-30 * time.Hour), "ok"}}, nil)
+	if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
+		t.Fatal(err)
+	}
+	got := runDeadline(t, st)
+	if !has(got, "expected_backup_missed") {
+		t.Fatalf("stale (>26h) PBS snapshot must raise expected_backup_missed; got %v", got)
+	}
+}
+
+// TestCheckBackupDeadlines_FailedVerify_Alarms: fresh snapshot but verify failed → alarm.
+func TestCheckBackupDeadlines_FailedVerify_Alarms(t *testing.T) {
+	st := newDeadlineStore(t)
+	st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
+	report := hostReportJSON(t, [][2]string{{rfc(-2 * time.Hour), "failed"}}, nil)
+	if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
+		t.Fatal(err)
+	}
+	got := runDeadline(t, st)
+	if !has(got, "expected_backup_missed") {
+		t.Fatalf("failed PBS verify must raise expected_backup_missed; got %v", got)
+	}
+}
+
+// TestCheckBackupDeadlines_NoHostReport_NoBackupAlarm: a customer with no host-report
+// (legacy/defunct controller-only) must NOT get a backup alarm from this check —
+// liveness is the staleness checker's job.
+func TestCheckBackupDeadlines_NoHostReport_NoBackupAlarm(t *testing.T) {
+	st := newDeadlineStore(t)
+	st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
+	// No SaveHostReport call.
+	got := runDeadline(t, st)
+	if has(got, "expected_backup_missed") {
+		t.Fatalf("no host-report → no backup alarm; got %v", got)
+	}
+}
+
+// TestCheckBackupDeadlines_DbDumpHalfPreserved: fresh backup (no backup alarm) but a
+// missing db_dump_completed event must still raise expected_dbdump_missed.
+func TestCheckBackupDeadlines_DbDumpHalfPreserved(t *testing.T) {
+	st := newDeadlineStore(t)
+	report := hostReportJSON(t, [][2]string{{rfc(-2 * time.Hour), "ok"}}, nil)
+	if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
+		t.Fatal(err)
+	}
+	got := runDeadline(t, st)
+	if has(got, "expected_backup_missed") {
+		t.Fatalf("fresh backup → no backup alarm; got %v", got)
+	}
+	if !has(got, "expected_dbdump_missed") {
+		t.Fatalf("missing db_dump_completed must still raise expected_dbdump_missed; got %v", got)
+	}
+}
+
+// TestAssessBackupFreshness exercises the pure freshness policy directly.
+func TestAssessBackupFreshness(t *testing.T) {
+	now := time.Date(2026, 6, 16, 3, 0, 0, 0, time.UTC)
+	at := func(d time.Duration) string { return now.Add(d).Format(time.RFC3339) }
+
+	cases := []struct {
+		name       string
+		report     string
+		wantMissed bool
+	}{
+		{"fresh verified PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"ok"}]}`, false},
+		{"fresh unverified (none) is not a failure", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"none"}]}`, false},
+		{"stale verified", `{"pbs_snapshots":[{"backup_time":"` + at(-30*time.Hour) + `","verify_state":"ok"}]}`, true},
+		{"fresh but verify failed", `{"pbs_snapshots":[{"backup_time":"` + at(-2*time.Hour) + `","verify_state":"failed"}]}`, true},
+		{"no snapshots and no backups", `{"pbs_snapshots":[],"backups":[]}`, true},
+		{"vzdump fallback fresh success", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":true}]}`, false},
+		{"vzdump only, failed → counts as none", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":false}]}`, true},
+		{"newest vzdump fresh rescues stale PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-40*time.Hour) + `","verify_state":"ok"}],"backups":[{"started_at":"` + at(-2*time.Hour) + `","success":true}]}`, false},
+		{"unparseable report", `not json`, true},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got := assessBackupFreshness(c.report, now)
+			if got.missed != c.wantMissed {
+				t.Fatalf("missed=%v want=%v (reason=%q)", got.missed, c.wantMissed, got.reason)
+			}
+		})
+	}
+}