package monitor import ( "encoding/json" "strings" "testing" "time" ) // R-82 Slice C — tier-aware thresholds. // // The whole point: a healthy WEEKLY offsite snapshot is >26h old six days in seven, so the merged // threshold would cry wolf on it — exactly what `backupStaleAfter`'s comment predicted. Each tier is // now judged against its own limit, and R-81's structure (three verdicts, anchored absence, one // distinct reason per failure mode) is preserved intact. func tieredReport(pbsAge time.Duration, verify string, hostAge time.Duration, now time.Time) string { at := func(d time.Duration) string { return now.Add(-d).Format(time.RFC3339) } return `{ "storage_targets":[ {"name":"local","type":"local","content":"backup,iso"}, {"name":"felhom-pbs","type":"pbs","content":"backup"} ], "pbs_snapshots":[{"backup_time":"` + at(pbsAge) + `","verify_state":"` + verify + `"}], "backups":[{"target_id":"local","started_at":"` + at(hostAge) + `","success":true}] }` } func freshEv(now time.Time, hostAge, offAge time.Duration) backupEvidence { return backupEvidence{ newestHost: now.Add(-hostAge), haveHost: true, newestOffsite: now.Add(-offAge), haveOffsite: true, firstReportAt: now.Add(-60 * 24 * time.Hour), } } // ── THE SCENARIO ───────────────────────────────────────────────────────────────────────────── // // A 6-day-old offsite snapshot with a fresh host backup is SILENT; the same snapshot at 9 days // ALARMS. // // COMPANION RED-PROOF (observed): give tierOffsite the host threshold // (`tierOffsite = backupTier{name:"offsite", staleWhen: backupStaleAfter}`) — i.e. restore the // single merged threshold — and the 6-day case fails with // // 6-day-old offsite snapshot is HEALTHY under a weekly cadence and must be SILENT; // got verdict=2 reason="offsite tier: newest backup is 144h0m0s old (limit 26h0m0s)" // // which is verbatim the cry-wolf this slice exists to remove. Restored. func TestTierAware_SixDayOffsiteSilent_NineDayAlarms(t *testing.T) { now := time.Date(2026, 7, 26, 3, 0, 0, 0, time.UTC) t.Run("6 days offsite + fresh host → SILENT", func(t *testing.T) { got := assessBackupFreshness(tieredReport(6*24*time.Hour, "ok", 2*time.Hour, now), freshEv(now, 2*time.Hour, 6*24*time.Hour), now) if got.missed() { t.Fatalf("6-day-old offsite snapshot is HEALTHY under a weekly cadence and must be SILENT; got verdict=%d reason=%q", got.verdict, got.reason) } if got.verdict != verdictOK { t.Fatalf("both tiers have evidence inside their limits → OK, not a deferred UNKNOWN; got verdict=%d reason=%q", got.verdict, got.reason) } }) t.Run("9 days offsite + fresh host → ALARMS, naming the offsite tier", func(t *testing.T) { got := assessBackupFreshness(tieredReport(9*24*time.Hour, "ok", 2*time.Hour, now), freshEv(now, 2*time.Hour, 9*24*time.Hour), now) if !got.missed() { t.Fatalf("9-day-old offsite snapshot is past the 8-day limit and MUST alarm; got verdict=%d reason=%q", got.verdict, got.reason) } if !strings.HasPrefix(got.reason, "offsite tier:") { t.Fatalf("the reason must name the OFFSITE tier — a merged message would have made the R-80 diagnosis impossible; got %q", got.reason) } if strings.Contains(got.reason, "host tier") { t.Fatalf("the host tier is fresh and must not appear in the reason; got %q", got.reason) } }) } // Neither tier rescues the other. This is the merged-threshold bug in both directions. func TestTierAware_TiersDoNotRescueEachOther(t *testing.T) { now := time.Date(2026, 7, 26, 3, 0, 0, 0, time.UTC) t.Run("fresh host does not rescue a stale offsite tier", func(t *testing.T) { got := assessBackupFreshness(tieredReport(9*24*time.Hour, "ok", time.Hour, now), freshEv(now, time.Hour, 9*24*time.Hour), now) if !got.missed() { t.Fatalf("a fresh DAILY backup must not satisfy the WEEKLY tier; got %q", got.reason) } }) t.Run("fresh offsite does not rescue a stale host tier", func(t *testing.T) { got := assessBackupFreshness(tieredReport(time.Hour, "ok", 30*time.Hour, now), freshEv(now, 30*time.Hour, time.Hour), now) if !got.missed() { t.Fatalf("a fresh WEEKLY snapshot must not satisfy the DAILY tier; got %q", got.reason) } if !strings.Contains(got.reason, "host tier") { t.Fatalf("the reason must name the HOST tier; got %q", got.reason) } }) t.Run("both stale → BOTH named", func(t *testing.T) { got := assessBackupFreshness(tieredReport(9*24*time.Hour, "ok", 30*time.Hour, now), freshEv(now, 30*time.Hour, 9*24*time.Hour), now) if !got.missed() { t.Fatalf("both tiers stale must alarm; got %q", got.reason) } if !strings.Contains(got.reason, "host tier") || !strings.Contains(got.reason, "offsite tier") { t.Fatalf("a box with TWO broken tiers must not report only one; got %q", got.reason) } }) } // ── The A.4 rule: classify by TARGET TYPE, never by array membership ───────────────────────── // // A PBS-targeted vzdump appears in BOTH arrays. Treating every `backups[]` entry as the host tier // would let a PBS backup make a STALE HOST TIER LOOK FRESH — silently losing the daily tier's alarm. func TestTierAware_PBSTargetedVzdumpIsNotHostEvidence(t *testing.T) { now := time.Date(2026, 7, 26, 3, 0, 0, 0, time.UTC) at := func(d time.Duration) string { return now.Add(-d).Format(time.RFC3339) } // The host tier's own backup is 40h old (stale). A FRESH PBS-targeted vzdump sits in the same // array — exactly the double-appearance Slice A.4 recorded. report := `{ "storage_targets":[ {"name":"local","type":"local","content":"backup"}, {"name":"felhom-pbs","type":"pbs","content":"backup"} ], "pbs_snapshots":[{"backup_time":"` + at(time.Hour) + `","verify_state":"ok"}], "backups":[ {"target_id":"local","started_at":"` + at(40*time.Hour) + `","success":true}, {"target_id":"felhom-pbs","started_at":"` + at(time.Hour) + `","success":true} ] }` ev := backupEvidence{ newestHost: now.Add(-40 * time.Hour), haveHost: true, newestOffsite: now.Add(-time.Hour), haveOffsite: true, firstReportAt: now.Add(-60 * 24 * time.Hour), } got := assessBackupFreshness(report, ev, now) if !got.missed() { t.Fatalf("a PBS-targeted vzdump must NOT count as host-tier evidence — the host tier is 40h stale; got verdict=%d reason=%q", got.verdict, got.reason) } if !strings.Contains(got.reason, "host tier") { t.Fatalf("the stale HOST tier must be named; got %q", got.reason) } } // splitTiers is the classifier the whole slice rests on. func TestSplitTiers_Classification(t *testing.T) { now := time.Date(2026, 7, 26, 3, 0, 0, 0, time.UTC) at := func(d time.Duration) string { return now.Add(-d).Format(time.RFC3339) } t.Run("no storage_targets and no evidence → neither tier expected (old-agent fallback)", func(t *testing.T) { var hr hostReportBackups mustUnmarshal(t, `{"pbs_snapshots":[],"backups":[]}`, &hr) h, o := splitTiers(hr) if h.expected || o.expected { t.Fatalf("nothing identifiable must leave BOTH tiers unexpected so the pre-Slice-C path runs; got host=%v offsite=%v", h.expected, o.expected) } }) t.Run("a PBS storage alone makes the offsite tier expected", func(t *testing.T) { var hr hostReportBackups mustUnmarshal(t, `{"storage_targets":[{"name":"felhom-pbs","type":"pbs","content":"backup"}]}`, &hr) _, o := splitTiers(hr) if !o.expected || o.have { t.Fatalf("a configured-but-empty offsite tier is EXPECTED with no evidence — that is the R-82 fault; got %+v", o) } }) t.Run("verify_state failed is carried on the newest offsite snapshot only", func(t *testing.T) { var hr hostReportBackups mustUnmarshal(t, `{"pbs_snapshots":[ {"backup_time":"`+at(48*time.Hour)+`","verify_state":"failed"}, {"backup_time":"`+at(time.Hour)+`","verify_state":"ok"}]}`, &hr) _, o := splitTiers(hr) if o.failed { t.Fatalf("an OLDER failed snapshot must not condemn a newer good one; got %+v", o) } }) } func mustUnmarshal(t *testing.T, s string, v any) { t.Helper() if err := json.Unmarshal([]byte(s), v); err != nil { t.Fatal(err) } }