f5a5e2b911
Third instance of one class (hub v0.12.0, v0.73.0, this), fixed as a class. On 2026-07-26 03:00 UTC expected_backup_missed fired on demo-felhom, demo-hp and drill-r50 at once; the demo-felhom one reached the CUSTOMER channel claiming "newest backup is 176h0m0s old". Nothing was wrong — three vzdump archives were on disk. Cause: the agent backup store is in-memory, so the R-50 fleet restart emptied `backups` until the next run, and the hub read empty as "no backup exists". - assessBackupFreshness returns OK/UNKNOWN/MISSED instead of `missed bool`; absence is UNKNOWN until it outlives an anchored window. Still pure. - store.GetHostReportsSince + monitor.newestBackupEvidence read the hubs own retained history (bounded 7-day lookback, early-exit on fresh evidence) — "when did I last SEE evidence of a backup?" The anchor was free: the hub already retains 90 days. No agent change, no new persisted state. - store.GetFirstHostReportAt anchors absence at first contact, reusing the existing 26h threshold as the grace (no new knob, the v0.73.0 shape). - Deferrals logged + counted; reason strings kept distinct. - backupStaleAfter untouched; landmine recorded (a weekly PBS snapshot would alarm six days in seven) and owned by R-82. Tests 493->508. Red-proofs A/B/C observed and restored; A reproduces the live message verbatim. Replayed the real 03:00 reports (600/417/77 rows): all three now silent. Source: documentation/audits/DIAG-backup-missed-2026-07-26.md
208 lines
8.0 KiB
Go
208 lines
8.0 KiB
Go
package monitor
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"log"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// newDeadlineStore creates an isolated store with one active customer + a host row.
|
|
func newDeadlineStore(t *testing.T) *store.Store {
|
|
t.Helper()
|
|
st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatalf("store.New: %v", err)
|
|
}
|
|
t.Cleanup(func() { st.Close() })
|
|
if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"}); err != nil {
|
|
t.Fatalf("SaveCustomerConfig: %v", err)
|
|
}
|
|
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
|
|
t.Fatalf("UpsertHost: %v", err)
|
|
}
|
|
return st
|
|
}
|
|
|
|
// hostReportJSON builds a host-report payload with the given PBS snapshots and vzdump
|
|
// backups. Each snapshot is {backup_time, verify_state}; each backup is {started_at, success}.
|
|
func hostReportJSON(t *testing.T, pbs [][2]string, backups []struct {
|
|
at string
|
|
ok bool
|
|
}) string {
|
|
t.Helper()
|
|
type snap struct {
|
|
BackupTime string `json:"backup_time"`
|
|
VerifyState string `json:"verify_state"`
|
|
}
|
|
type bk struct {
|
|
StartedAt string `json:"started_at"`
|
|
Success bool `json:"success"`
|
|
}
|
|
payload := struct {
|
|
HostID string `json:"host_id"`
|
|
PBSSnapshots []snap `json:"pbs_snapshots"`
|
|
Backups []bk `json:"backups"`
|
|
}{HostID: "h1"}
|
|
for _, p := range pbs {
|
|
payload.PBSSnapshots = append(payload.PBSSnapshots, snap{BackupTime: p[0], VerifyState: p[1]})
|
|
}
|
|
for _, b := range backups {
|
|
payload.Backups = append(payload.Backups, bk{StartedAt: b.at, Success: b.ok})
|
|
}
|
|
out, err := json.Marshal(payload)
|
|
if err != nil {
|
|
t.Fatalf("marshal report: %v", err)
|
|
}
|
|
return string(out)
|
|
}
|
|
|
|
// runDeadline runs CheckBackupDeadlines and returns the list of emitted event types for c1.
|
|
func runDeadline(t *testing.T, st *store.Store) []string {
|
|
t.Helper()
|
|
var got []string
|
|
onEvent := func(customerID, eventType, severity, message, detailsJSON, source string) {
|
|
if customerID == "c1" {
|
|
got = append(got, eventType)
|
|
}
|
|
}
|
|
// nil staleness → no "down" skip; the check evaluates c1.
|
|
CheckBackupDeadlines(st, nil, onEvent, log.New(io.Discard, "", 0))
|
|
return got
|
|
}
|
|
|
|
func has(events []string, t string) bool {
|
|
for _, e := range events {
|
|
if e == t {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func rfc(d time.Duration) string {
|
|
return time.Now().UTC().Add(d).Format(time.RFC3339)
|
|
}
|
|
|
|
// TestCheckBackupDeadlines_FreshVerifiedPBS_NoBackupAlarm is the COMPANION test.
|
|
// It FAILS against the pre-fix (event-based) check: with a fresh, verified PBS snapshot
|
|
// but no backup_completed event, the old code emitted expected_backup_missed anyway.
|
|
// The repoint reads the host-report instead, so a healthy customer raises no alarm.
|
|
func TestCheckBackupDeadlines_FreshVerifiedPBS_NoBackupAlarm(t *testing.T) {
|
|
st := newDeadlineStore(t)
|
|
// Make the db-dump half pass too, so the only thing under test is the backup half.
|
|
if _, err := st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
report := hostReportJSON(t, [][2]string{{rfc(-3 * time.Hour), "ok"}}, nil)
|
|
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
got := runDeadline(t, st)
|
|
if has(got, "expected_backup_missed") {
|
|
t.Fatalf("fresh+verified PBS must NOT raise expected_backup_missed; got %v", got)
|
|
}
|
|
if has(got, "expected_dbdump_missed") {
|
|
t.Fatalf("db_dump_completed present → no dbdump alarm expected; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestCheckBackupDeadlines_StalePBS_Alarms: newest backup older than 26h → alarm.
|
|
func TestCheckBackupDeadlines_StalePBS_Alarms(t *testing.T) {
|
|
st := newDeadlineStore(t)
|
|
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
|
|
report := hostReportJSON(t, [][2]string{{rfc(-30 * time.Hour), "ok"}}, nil)
|
|
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := runDeadline(t, st)
|
|
if !has(got, "expected_backup_missed") {
|
|
t.Fatalf("stale (>26h) PBS snapshot must raise expected_backup_missed; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestCheckBackupDeadlines_FailedVerify_Alarms: fresh snapshot but verify failed → alarm.
|
|
func TestCheckBackupDeadlines_FailedVerify_Alarms(t *testing.T) {
|
|
st := newDeadlineStore(t)
|
|
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
|
|
report := hostReportJSON(t, [][2]string{{rfc(-2 * time.Hour), "failed"}}, nil)
|
|
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := runDeadline(t, st)
|
|
if !has(got, "expected_backup_missed") {
|
|
t.Fatalf("failed PBS verify must raise expected_backup_missed; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestCheckBackupDeadlines_NoHostReport_NoBackupAlarm: a customer with no host-report
|
|
// (legacy/defunct controller-only) must NOT get a backup alarm from this check —
|
|
// liveness is the staleness checker's job.
|
|
func TestCheckBackupDeadlines_NoHostReport_NoBackupAlarm(t *testing.T) {
|
|
st := newDeadlineStore(t)
|
|
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
|
|
// No SaveHostReport call.
|
|
got := runDeadline(t, st)
|
|
if has(got, "expected_backup_missed") {
|
|
t.Fatalf("no host-report → no backup alarm; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestCheckBackupDeadlines_DbDumpHalfPreserved: fresh backup (no backup alarm) but a
|
|
// missing db_dump_completed event must still raise expected_dbdump_missed.
|
|
func TestCheckBackupDeadlines_DbDumpHalfPreserved(t *testing.T) {
|
|
st := newDeadlineStore(t)
|
|
report := hostReportJSON(t, [][2]string{{rfc(-2 * time.Hour), "ok"}}, nil)
|
|
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := runDeadline(t, st)
|
|
if has(got, "expected_backup_missed") {
|
|
t.Fatalf("fresh backup → no backup alarm; got %v", got)
|
|
}
|
|
if !has(got, "expected_dbdump_missed") {
|
|
t.Fatalf("missing db_dump_completed must still raise expected_dbdump_missed; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestAssessBackupFreshness exercises the pure freshness policy directly.
|
|
//
|
|
// R-81: every case here passes a ZERO backupEvidence — no hub-history evidence and no
|
|
// first-contact anchor. That is deliberate: it pins the latest-report-only behaviour
|
|
// unchanged, and the "no snapshots and no backups" row exercises the UNANCHORED absence
|
|
// branch (zero anchor → fail toward visibility, the v0.73.0 legacy-shape precedent).
|
|
// The anchored branches have their own named tests below.
|
|
func TestAssessBackupFreshness(t *testing.T) {
|
|
now := time.Date(2026, 6, 16, 3, 0, 0, 0, time.UTC)
|
|
at := func(d time.Duration) string { return now.Add(d).Format(time.RFC3339) }
|
|
|
|
cases := []struct {
|
|
name string
|
|
report string
|
|
wantMissed bool
|
|
}{
|
|
{"fresh verified PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"ok"}]}`, false},
|
|
{"fresh unverified (none) is not a failure", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"none"}]}`, false},
|
|
{"stale verified", `{"pbs_snapshots":[{"backup_time":"` + at(-30*time.Hour) + `","verify_state":"ok"}]}`, true},
|
|
{"fresh but verify failed", `{"pbs_snapshots":[{"backup_time":"` + at(-2*time.Hour) + `","verify_state":"failed"}]}`, true},
|
|
{"no snapshots and no backups", `{"pbs_snapshots":[],"backups":[]}`, true},
|
|
{"vzdump fallback fresh success", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":true}]}`, false},
|
|
{"vzdump only, failed → counts as none", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":false}]}`, true},
|
|
{"newest vzdump fresh rescues stale PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-40*time.Hour) + `","verify_state":"ok"}],"backups":[{"started_at":"` + at(-2*time.Hour) + `","success":true}]}`, false},
|
|
{"unparseable report", `not json`, true},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
got := assessBackupFreshness(c.report, backupEvidence{}, now)
|
|
if got.missed() != c.wantMissed {
|
|
t.Fatalf("missed=%v want=%v (reason=%q)", got.missed(), c.wantMissed, got.reason)
|
|
}
|
|
})
|
|
}
|
|
}
|