hub v0.76.0 — R-82 Slice C: tier-aware backup thresholds
R-81 merged every backup signal into one 'newest' against a single 26h limit. backupStaleAfter's own comment recorded why that stops being right under a weekly offsite tier. Each tier is now judged against its own threshold; R-81's structure (three verdicts, anchored absence, distinct reasons) and its boundary test are preserved intact. - offsiteBackupStaleAfter = 8d (7d cadence + headroom); backupStaleAfter keeps 26h and now names the HOST tier only - splitTiers / assessTier / newestBackupEvidenceByTier Slice-A.4 rule implemented: a PBS-targeted vzdump appears in BOTH arrays, so classification is by TARGET TYPE (target_id -> storage_targets[].name -> type), never by array membership — otherwise a PBS backup makes a stale host tier look fresh. storage_targets is used rather than pbs_dr.storage_id because the latter is null on a box with a PBS storage but no DR descriptor. A tier is only judged when the box HAS it, else every box without an offsite tier would alarm once the anchor elapsed — R-81's mistake one level down. With neither tier identifiable (old agent) the pre-Slice-C path runs unchanged. Intended behaviour change: a 30h offsite snapshot no longer alarms. Three fixtures asserted the merged threshold; each still asserts an alarm at the correct limit. No assertion was weakened. RECORDED LIMITATION: the hub infers 'PBS => weekly' from storage type. defaultBackupTarget is felhom-pbs, so a box that never sets local_backup_target would run PBS as its DAILY tier and be judged against 8 days — 7 days of blindness. No box is in that shape today; the real fix is the agent reporting per-tier cadences. Own task. Red-proof observed. Replayed live: demo-felhom OK, demo-hp UNKNOWN (defers correctly), drill-r50 MISSED (true positive). No customer email would be sent.
This commit is contained in:
@@ -112,11 +112,16 @@ func TestCheckBackupDeadlines_FreshVerifiedPBS_NoBackupAlarm(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckBackupDeadlines_StalePBS_Alarms: newest backup older than 26h → alarm.
|
||||
// TestCheckBackupDeadlines_StalePBS_Alarms: an offsite snapshot past the OFFSITE limit → alarm.
|
||||
//
|
||||
// R-82 Slice C changed the threshold this fixture must cross, NOT the behaviour it asserts. 30h was
|
||||
// "stale" while one 26h limit covered every tier; under a weekly offsite tier a 30h snapshot is
|
||||
// healthy, and alarming on it is exactly the cry-wolf `backupStaleAfter`'s comment predicted. The
|
||||
// test still proves a stale offsite tier alarms — now at 9 days, past the 8-day offsite limit.
|
||||
func TestCheckBackupDeadlines_StalePBS_Alarms(t *testing.T) {
|
||||
st := newDeadlineStore(t)
|
||||
st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller")
|
||||
report := hostReportJSON(t, [][2]string{{rfc(-30 * time.Hour), "ok"}}, nil)
|
||||
report := hostReportJSON(t, [][2]string{{rfc(-9 * 24 * time.Hour), "ok"}}, nil)
|
||||
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -188,12 +193,19 @@ func TestAssessBackupFreshness(t *testing.T) {
|
||||
}{
|
||||
{"fresh verified PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"ok"}]}`, false},
|
||||
{"fresh unverified (none) is not a failure", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"none"}]}`, false},
|
||||
{"stale verified", `{"pbs_snapshots":[{"backup_time":"` + at(-30*time.Hour) + `","verify_state":"ok"}]}`, true},
|
||||
// Slice C: a 30h offsite snapshot is HEALTHY under the weekly (8d) offsite limit.
|
||||
{"offsite 30h is fresh under the weekly limit", `{"pbs_snapshots":[{"backup_time":"` + at(-30*time.Hour) + `","verify_state":"ok"}]}`, false},
|
||||
{"offsite 6d is fresh under the weekly limit", `{"pbs_snapshots":[{"backup_time":"` + at(-6*24*time.Hour) + `","verify_state":"ok"}]}`, false},
|
||||
{"offsite 9d is STALE past the weekly limit", `{"pbs_snapshots":[{"backup_time":"` + at(-9*24*time.Hour) + `","verify_state":"ok"}]}`, true},
|
||||
{"fresh but verify failed", `{"pbs_snapshots":[{"backup_time":"` + at(-2*time.Hour) + `","verify_state":"failed"}]}`, true},
|
||||
{"no snapshots and no backups", `{"pbs_snapshots":[],"backups":[]}`, true},
|
||||
{"vzdump fallback fresh success", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":true}]}`, false},
|
||||
{"vzdump only, failed → counts as none", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":false}]}`, true},
|
||||
{"newest vzdump fresh rescues stale PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-40*time.Hour) + `","verify_state":"ok"}],"backups":[{"started_at":"` + at(-2*time.Hour) + `","success":true}]}`, false},
|
||||
{"host vzdump fresh success", `{"backups":[{"target_id":"local","started_at":"` + at(-4*time.Hour) + `","success":true}]}`, false},
|
||||
{"host vzdump only, failed → counts as none", `{"backups":[{"target_id":"local","started_at":"` + at(-4*time.Hour) + `","success":false}]}`, true},
|
||||
{"host fresh + offsite within weekly → both healthy", `{"pbs_snapshots":[{"backup_time":"` + at(-40*time.Hour) + `","verify_state":"ok"}],"backups":[{"target_id":"local","started_at":"` + at(-2*time.Hour) + `","success":true}]}`, false},
|
||||
// Slice C's core: the tiers are judged SEPARATELY. A fresh host backup must NOT rescue an
|
||||
// offsite tier that is genuinely past its own (weekly) limit — the merged threshold did.
|
||||
{"host fresh does NOT rescue a 9d offsite tier", `{"pbs_snapshots":[{"backup_time":"` + at(-9*24*time.Hour) + `","verify_state":"ok"}],"backups":[{"target_id":"local","started_at":"` + at(-2*time.Hour) + `","success":true}]}`, true},
|
||||
{"offsite fresh does NOT rescue a 30h host tier", `{"pbs_snapshots":[{"backup_time":"` + at(-2*time.Hour) + `","verify_state":"ok"}],"backups":[{"target_id":"local","started_at":"` + at(-30*time.Hour) + `","success":true}]}`, true},
|
||||
{"unparseable report", `not json`, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
|
||||
Reference in New Issue
Block a user