Files
felhom.eu/hub/internal/monitor/deadline_test.go
T
Claude Code f5a5e2b911 hub v0.75.0: R-81 — "no signal" is not "bad signal" (anchor the backup deadline check)
Third instance of one class (hub v0.12.0, v0.73.0, this), fixed as a class.
On 2026-07-26 03:00 UTC expected_backup_missed fired on demo-felhom, demo-hp
and drill-r50 at once; the demo-felhom one reached the CUSTOMER channel
claiming "newest backup is 176h0m0s old". Nothing was wrong — three vzdump
archives were on disk. Cause: the agent backup store is in-memory, so the
R-50 fleet restart emptied `backups` until the next run, and the hub read
empty as "no backup exists".

- assessBackupFreshness returns OK/UNKNOWN/MISSED instead of `missed bool`;
  absence is UNKNOWN until it outlives an anchored window. Still pure.
- store.GetHostReportsSince + monitor.newestBackupEvidence read the hubs own
  retained history (bounded 7-day lookback, early-exit on fresh evidence) —
  "when did I last SEE evidence of a backup?" The anchor was free: the hub
  already retains 90 days. No agent change, no new persisted state.
- store.GetFirstHostReportAt anchors absence at first contact, reusing the
  existing 26h threshold as the grace (no new knob, the v0.73.0 shape).
- Deferrals logged + counted; reason strings kept distinct.
- backupStaleAfter untouched; landmine recorded (a weekly PBS snapshot would
  alarm six days in seven) and owned by R-82.

Tests 493->508. Red-proofs A/B/C observed and restored; A reproduces the live
message verbatim. Replayed the real 03:00 reports (600/417/77 rows): all
three now silent.

Source: documentation/audits/DIAG-backup-missed-2026-07-26.md
2026-07-26 11:44:15 +02:00

208 lines
8.0 KiB
Go

package monitor
import (
"encoding/json"
"io"
"log"
"path/filepath"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
// newDeadlineStore creates an isolated store with one active customer + a host row.
func newDeadlineStore(t *testing.T) *store.Store {
t.Helper()
st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
if err != nil {
t.Fatalf("store.New: %v", err)
}
t.Cleanup(func() { st.Close() })
if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"}); err != nil {
t.Fatalf("SaveCustomerConfig: %v", err)
}
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
t.Fatalf("UpsertHost: %v", err)
}
return st
}
// hostReportJSON builds a host-report payload with the given PBS snapshots and vzdump
// backups. Each snapshot is {backup_time, verify_state}; each backup is {started_at, success}.
func hostReportJSON(t *testing.T, pbs [][2]string, backups []struct {
at string
ok bool
}) string {
t.Helper()
type snap struct {
BackupTime string `json:"backup_time"`
VerifyState string `json:"verify_state"`
}
type bk struct {
StartedAt string `json:"started_at"`
Success bool `json:"success"`
}
payload := struct {
HostID string `json:"host_id"`
PBSSnapshots []snap `json:"pbs_snapshots"`
Backups []bk `json:"backups"`
}{HostID: "h1"}
for _, p := range pbs {
payload.PBSSnapshots = append(payload.PBSSnapshots, snap{BackupTime: p[0], VerifyState: p[1]})
}
for _, b := range backups {
payload.Backups = append(payload.Backups, bk{StartedAt: b.at, Success: b.ok})
}
out, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal report: %v", err)
}
return string(out)
}
// runDeadline runs CheckBackupDeadlines and returns the list of emitted event types for c1.
func runDeadline(t *testing.T, st *store.Store) []string {
t.Helper()
var got []string
onEvent := func(customerID, eventType, severity, message, detailsJSON, source string) {
if customerID == "c1" {
got = append(got, eventType)
}
}
// nil staleness → no "down" skip; the check evaluates c1.
CheckBackupDeadlines(st, nil, onEvent, log.New(io.Discard, "", 0))
return got
}
func has(events []string, t string) bool {
for _, e := range events {
if e == t {
return true
}
}
return false
}
func rfc(d time.Duration) string {
return time.Now().UTC().Add(d).Format(time.RFC3339)
}
// TestCheckBackupDeadlines_FreshVerifiedPBS_NoBackupAlarm is the COMPANION test.
// It FAILS against the pre-fix (event-based) check: with a fresh, verified PBS snapshot
// but no backup_completed event, the old code emitted expected_backup_missed anyway.
// The repoint reads the host-report instead, so a healthy customer raises no alarm.
func TestCheckBackupDeadlines_FreshVerifiedPBS_NoBackupAlarm(t *testing.T) {
st := newDeadlineStore(t)
// Make the db-dump half pass too, so the only thing under test is the backup half.
if _, err := st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller"); err != nil {
t.Fatal(err)
}
report := hostReportJSON(t, [][2]string{{rfc(-3 * time.Hour), "ok"}}, nil)
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
t.Fatal(err)
}
got := runDeadline(t, st)
if has(got, "expected_backup_missed") {
t.Fatalf("fresh+verified PBS must NOT raise expected_backup_missed; got %v", got)
}
if has(got, "expected_dbdump_missed") {
t.Fatalf("db_dump_completed present → no dbdump alarm expected; got %v", got)
}
}
// TestCheckBackupDeadlines_StalePBS_Alarms: newest backup older than 26h → alarm.
func TestCheckBackupDeadlines_StalePBS_Alarms(t *testing.T) {
st := newDeadlineStore(t)
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
report := hostReportJSON(t, [][2]string{{rfc(-30 * time.Hour), "ok"}}, nil)
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
t.Fatal(err)
}
got := runDeadline(t, st)
if !has(got, "expected_backup_missed") {
t.Fatalf("stale (>26h) PBS snapshot must raise expected_backup_missed; got %v", got)
}
}
// TestCheckBackupDeadlines_FailedVerify_Alarms: fresh snapshot but verify failed → alarm.
func TestCheckBackupDeadlines_FailedVerify_Alarms(t *testing.T) {
st := newDeadlineStore(t)
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
report := hostReportJSON(t, [][2]string{{rfc(-2 * time.Hour), "failed"}}, nil)
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
t.Fatal(err)
}
got := runDeadline(t, st)
if !has(got, "expected_backup_missed") {
t.Fatalf("failed PBS verify must raise expected_backup_missed; got %v", got)
}
}
// TestCheckBackupDeadlines_NoHostReport_NoBackupAlarm: a customer with no host-report
// (legacy/defunct controller-only) must NOT get a backup alarm from this check —
// liveness is the staleness checker's job.
func TestCheckBackupDeadlines_NoHostReport_NoBackupAlarm(t *testing.T) {
st := newDeadlineStore(t)
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
// No SaveHostReport call.
got := runDeadline(t, st)
if has(got, "expected_backup_missed") {
t.Fatalf("no host-report → no backup alarm; got %v", got)
}
}
// TestCheckBackupDeadlines_DbDumpHalfPreserved: fresh backup (no backup alarm) but a
// missing db_dump_completed event must still raise expected_dbdump_missed.
func TestCheckBackupDeadlines_DbDumpHalfPreserved(t *testing.T) {
st := newDeadlineStore(t)
report := hostReportJSON(t, [][2]string{{rfc(-2 * time.Hour), "ok"}}, nil)
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
t.Fatal(err)
}
got := runDeadline(t, st)
if has(got, "expected_backup_missed") {
t.Fatalf("fresh backup → no backup alarm; got %v", got)
}
if !has(got, "expected_dbdump_missed") {
t.Fatalf("missing db_dump_completed must still raise expected_dbdump_missed; got %v", got)
}
}
// TestAssessBackupFreshness exercises the pure freshness policy directly.
//
// R-81: every case here passes a ZERO backupEvidence — no hub-history evidence and no
// first-contact anchor. That is deliberate: it pins the latest-report-only behaviour
// unchanged, and the "no snapshots and no backups" row exercises the UNANCHORED absence
// branch (zero anchor → fail toward visibility, the v0.73.0 legacy-shape precedent).
// The anchored branches have their own named tests below.
func TestAssessBackupFreshness(t *testing.T) {
now := time.Date(2026, 6, 16, 3, 0, 0, 0, time.UTC)
at := func(d time.Duration) string { return now.Add(d).Format(time.RFC3339) }
cases := []struct {
name string
report string
wantMissed bool
}{
{"fresh verified PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"ok"}]}`, false},
{"fresh unverified (none) is not a failure", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"none"}]}`, false},
{"stale verified", `{"pbs_snapshots":[{"backup_time":"` + at(-30*time.Hour) + `","verify_state":"ok"}]}`, true},
{"fresh but verify failed", `{"pbs_snapshots":[{"backup_time":"` + at(-2*time.Hour) + `","verify_state":"failed"}]}`, true},
{"no snapshots and no backups", `{"pbs_snapshots":[],"backups":[]}`, true},
{"vzdump fallback fresh success", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":true}]}`, false},
{"vzdump only, failed → counts as none", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":false}]}`, true},
{"newest vzdump fresh rescues stale PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-40*time.Hour) + `","verify_state":"ok"}],"backups":[{"started_at":"` + at(-2*time.Hour) + `","success":true}]}`, false},
{"unparseable report", `not json`, true},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := assessBackupFreshness(c.report, backupEvidence{}, now)
if got.missed() != c.wantMissed {
t.Fatalf("missed=%v want=%v (reason=%q)", got.missed(), c.wantMissed, got.reason)
}
})
}
}