From b11607b26be7b681a3d605115c6996c4557c174e Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 26 Jul 2026 16:58:38 +0200 Subject: [PATCH] =?UTF-8?q?hub=20v0.76.0=20=E2=80=94=20R-82=20Slice=20C:?= =?UTF-8?q?=20tier-aware=20backup=20thresholds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R-81 merged every backup signal into one 'newest' against a single 26h limit. backupStaleAfter's own comment recorded why that stops being right under a weekly offsite tier. Each tier is now judged against its own threshold; R-81's structure (three verdicts, anchored absence, distinct reasons) and its boundary test are preserved intact. - offsiteBackupStaleAfter = 8d (7d cadence + headroom); backupStaleAfter keeps 26h and now names the HOST tier only - splitTiers / assessTier / newestBackupEvidenceByTier Slice-A.4 rule implemented: a PBS-targeted vzdump appears in BOTH arrays, so classification is by TARGET TYPE (target_id -> storage_targets[].name -> type), never by array membership — otherwise a PBS backup makes a stale host tier look fresh. storage_targets is used rather than pbs_dr.storage_id because the latter is null on a box with a PBS storage but no DR descriptor. A tier is only judged when the box HAS it, else every box without an offsite tier would alarm once the anchor elapsed — R-81's mistake one level down. With neither tier identifiable (old agent) the pre-Slice-C path runs unchanged. Intended behaviour change: a 30h offsite snapshot no longer alarms. Three fixtures asserted the merged threshold; each still asserts an alarm at the correct limit. No assertion was weakened. RECORDED LIMITATION: the hub infers 'PBS => weekly' from storage type. defaultBackupTarget is felhom-pbs, so a box that never sets local_backup_target would run PBS as its DAILY tier and be judged against 8 days — 7 days of blindness. No box is in that shape today; the real fix is the agent reporting per-tier cadences. Own task. Red-proof observed. Replayed live: demo-felhom OK, demo-hp UNKNOWN (defers correctly), drill-r50 MISSED (true positive). No customer email would be sent. --- hub/CHANGELOG.md | 68 ++++++ hub/internal/monitor/deadline.go | 45 ++++ hub/internal/monitor/deadline_anchor_test.go | 20 +- hub/internal/monitor/deadline_test.go | 24 ++- hub/internal/monitor/deadline_tiers.go | 214 +++++++++++++++++++ hub/internal/monitor/deadline_tiers_test.go | 192 +++++++++++++++++ 6 files changed, 549 insertions(+), 14 deletions(-) create mode 100644 hub/internal/monitor/deadline_tiers.go create mode 100644 hub/internal/monitor/deadline_tiers_test.go diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 5dae086..a9aaf37 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,5 +1,73 @@ # Felhom Hub — Changelog +## v0.76.0 — R-82 Slice C: tier-aware backup thresholds (2026-07-26) + +R-81 merged every backup signal into one "newest" and judged it against a single 26 h limit. That +was right while a box had exactly one whole-guest tier. `backupStaleAfter`'s own comment recorded +why it stops being right: + +> *"The moment PBS moves to a WEEKLY cadence, a perfectly healthy weekly snapshot is >26h old six +> days in seven and this constant alarms on it."* + +Each tier is now judged against **its own** threshold. **R-81's structure is preserved intact** — +three-valued verdicts, absence anchored at first contact, one distinct reason per failure mode — and +its boundary test is untouched. + +### Added +- **`offsiteBackupStaleAfter = 8d`** (7-day cadence + a day of headroom — the same ratio 26 h gives + a 24 h cadence). `backupStaleAfter` keeps 26 h and now names the **host** tier only. +- **`splitTiers`** — classifies a report's evidence into host and offsite tiers. +- **`assessTier`** — R-81's exact logic, parameterised by tier and threshold. +- **`newestBackupEvidenceByTier`** — the retained-window scan, per tier. + +### The Slice-A.4 rule, implemented +A PBS-targeted vzdump appears in **both** `backups[]` (as a `Backup` with `target_id:"felhom-pbs"`) +and `pbs_snapshots[]` (enumerated independently from PBS). Classification is therefore by **TARGET +TYPE** — `target_id` → `storage_targets[].name` → `.type == "pbs"` — **never by array membership**. +Getting that wrong would let a PBS backup make a **stale host tier look fresh**, silently losing the +daily tier's alarm. Pinned by `TestTierAware_PBSTargetedVzdumpIsNotHostEvidence`. +`storage_targets` is used rather than `pbs_dr.storage_id` because the latter is null on a box that +has a PBS storage but no DR descriptor yet (drill-r50 was exactly that shape). + +### A tier is only judged when the box HAS it +`expected` gates each tier on a configured storage of that kind, or evidence for it. Without that, +every box without an offsite tier would alarm as soon as the anchor elapsed — the +absence-is-not-failure mistake R-81 exists to prevent, re-introduced one level down. When NEITHER +tier is identifiable (an old agent reporting no `storage_targets` and no `target_id`) the pre-Slice-C +combined path runs unchanged, so nothing regresses on a fleet mid-upgrade. + +### Changed behaviour (intended) +A 30 h-old offsite snapshot no longer alarms — under a weekly tier it is healthy. Three existing +fixtures asserted the old merged threshold; each still asserts an alarm, now at the correct limit +(9 days for offsite, 30 h for host). **No assertion was weakened to make the code pass.** + +### ⚠️ Recorded limitation — the hub infers cadence from storage TYPE +"PBS ⇒ weekly" is an inference, not a fact the box tells us. `defaultBackupTarget` is `"felhom-pbs"`, +so a box that never sets `local_backup_target` would run PBS as its **daily primary** tier and the +hub would judge it against 8 days — **seven days of blindness**. No box is in that shape today (both +demo boxes set `local`, and the installer pins it), but it is a latent mis-classification of exactly +the kind that became R-80. The real fix is the agent reporting each tier's **actual cadence** in the +host-report; own task. + +### Tests +Full suite green (17 packages). Red-proof observed: giving `tierOffsite` the host threshold — i.e. +restoring the merged limit — fails the 6-day case with +`offsite tier: newest backup is 144h0m0s old (limit 26h0m0s)`, verbatim the cry-wolf this slice +removes. Restored. + +**Replayed against the live hub DB** through the real store queries: + +``` +demo-felhom host=07-26T14:38Z offsite=07-26T12:21Z -> OK +demo-hp host=07-26T07:06Z offsite=none -> UNKNOWN (offsite watched 119h of a 192h grace) +drill-r50 host=none offsite=not expected -> MISSED (host tier, no evidence in 29h) +``` + +**No customer email would be sent by this deploy.** demo-felhom is clean; demo-hp defers correctly +and will alarm in ~3 days if its offsite tier stays empty (the true R-82 finding, arriving on +schedule); drill-r50's alarm is a true positive and it has no customer channel. + + ## v0.75.0 — R-81: "no signal" is not "bad signal" — the backup deadline check is ANCHORED (2026-07-26) The third instance of one bug class, fixed as a class. `expected_backup_missed` fired on diff --git a/hub/internal/monitor/deadline.go b/hub/internal/monitor/deadline.go index ec18f13..a1ae70d 100644 --- a/hub/internal/monitor/deadline.go +++ b/hub/internal/monitor/deadline.go @@ -45,9 +45,19 @@ type hostReportBackups struct { VerifyState string `json:"verify_state"` } `json:"pbs_snapshots"` Backups []struct { + // TargetID is the SLICE-C discriminator: a backups[] entry belongs to whichever tier its + // target storage belongs to, never to "the local tier" by virtue of being in this array. + TargetID string `json:"target_id"` StartedAt string `json:"started_at"` Success bool `json:"success"` } `json:"backups"` + // StorageTargets carries the target TYPE, which is how a backups[] entry is attributed to a + // tier (target_id → name → type == "pbs"). See pbsTargetSet. + StorageTargets []struct { + Name string `json:"name"` + Type string `json:"type"` + Content string `json:"content"` + } `json:"storage_targets"` } // backupVerdict is the three-valued outcome of the freshness policy. The middle value is the @@ -78,6 +88,15 @@ type backupEvidence struct { newestSeen time.Time haveSeen bool + // Per-tier window evidence (R-82 Slice C). A single "newest across everything" would let a + // fresh daily host backup satisfy the OFFSITE tier's lookup — the same error Slice A fixed + // agent-side. newestSeen/haveSeen above remain the pre-Slice-C combined values, used only by + // the no-tier fallback path. + newestHost time.Time + haveHost bool + newestOffsite time.Time + haveOffsite bool + // firstReportAt is when the hub first saw ANY host-report from this customer — the // observation anchor. Zero when unknown, which the policy treats as "cannot defer" // (fail toward visibility, matching the v0.73.0 zero-anchor branch). @@ -163,6 +182,22 @@ func assessBackupFreshness(reportJSON string, ev backupEvidence, now time.Time) } } + // ── R-82 Slice C: TIER-AWARE assessment ─────────────────────────────────────────────────── + // Judge each tier against ITS OWN threshold. Falls through to the pre-Slice-C combined logic + // below only when NEITHER tier is identifiable — an old agent whose report carries no + // storage_targets and no target_id — so nothing regresses on a fleet mid-upgrade. + hostView, offView := splitTiers(hr) + if hostView.expected || offView.expected { + var as []backupAssessment + if hostView.expected { + as = append(as, assessTier(tierHost, hostView, ev.newestHost, ev.haveHost, ev.firstReportAt, now)) + } + if offView.expected { + as = append(as, assessTier(tierOffsite, offView, ev.newestOffsite, ev.haveOffsite, ev.firstReportAt, now)) + } + return worst(as...) + } + // The newest evidence the LATEST report itself carries. newest := newestPBS haveNewest := havePBS @@ -337,6 +372,16 @@ func CheckBackupDeadlines(s *store.Store, staleness *StalenessChecker, onEvent E logger.Printf("[WARN] Deadline check: failed to read host-report window for %s: %v", id, rerr) } else { ev.newestSeen, ev.haveSeen = newestBackupEvidence(rows, nowUTC) + // Slice C: per-tier window evidence. Which tiers to look for comes from the LATEST + // report, so a box with no offsite tier never pays for scanning one. + var latest hostReportBackups + wantHost, wantOffsite := true, true + if json.Unmarshal([]byte(reportJSON), &latest) == nil { + h, o := splitTiers(latest) + wantHost, wantOffsite = h.expected, o.expected + } + ev.newestHost, ev.haveHost, ev.newestOffsite, ev.haveOffsite = + newestBackupEvidenceByTier(rows, wantHost, wantOffsite, nowUTC) } if first, ferr := s.GetFirstHostReportAt(id); ferr != nil { logger.Printf("[WARN] Deadline check: failed to read first host-report for %s: %v", id, ferr) diff --git a/hub/internal/monitor/deadline_anchor_test.go b/hub/internal/monitor/deadline_anchor_test.go index 47ecaa2..fe040e5 100644 --- a/hub/internal/monitor/deadline_anchor_test.go +++ b/hub/internal/monitor/deadline_anchor_test.go @@ -184,14 +184,16 @@ func TestBackupFreshness_ExistingBehavioursUnchanged(t *testing.T) { } }) - t.Run("2 newest backup >26h still alarms with the same string", func(t *testing.T) { - r := `{"pbs_snapshots":[{"backup_time":"` + at(-30*time.Hour) + `","verify_state":"ok"}]}` - // Window evidence is ALSO 30h old — nothing fresher exists anywhere. - got := assessBackupFreshness(r, evidenceAt(now, 30*time.Hour, anchor), now) + t.Run("2 a backup past its tier limit still alarms", func(t *testing.T) { + // Slice C: the HOST tier keeps the 26h limit verbatim, so this is the same assertion the + // merged code made — now scoped to the tier it actually belongs to. + r := `{"backups":[{"target_id":"local","started_at":"` + at(-30*time.Hour) + `","success":true}]}` + ev := backupEvidence{newestHost: now.Add(-30 * time.Hour), haveHost: true, firstReportAt: now.Add(-anchor)} + got := assessBackupFreshness(r, ev, now) if !got.missed() { - t.Fatalf("stale backup must still alarm; got verdict=%d reason=%q", got.verdict, got.reason) + t.Fatalf("stale host backup must still alarm; got verdict=%d reason=%q", got.verdict, got.reason) } - if got.reason != "newest backup is 30h0m0s old (limit 26h0m0s)" { + if got.reason != "host tier: newest backup is 30h0m0s old (limit 26h0m0s)" { t.Fatalf("stale reason string changed: %q", got.reason) } }) @@ -202,7 +204,9 @@ func TestBackupFreshness_ExistingBehavioursUnchanged(t *testing.T) { if !got.missed() { t.Fatalf("failed verify must still alarm; got verdict=%d reason=%q", got.verdict, got.reason) } - if got.reason != "newest PBS snapshot failed verification" { + // Slice C prefixes the tier. The FAILURE MODE is unchanged and still has its own distinct + // string; only the tier name was added, which is the point of tier-aware reasons. + if got.reason != "offsite tier: newest PBS snapshot failed verification" { t.Fatalf("verify-failed reason string changed: %q", got.reason) } }) @@ -236,7 +240,7 @@ func TestBackupFreshness_ReasonStringsAreDistinct(t *testing.T) { reasons := map[string]string{ "unparseable": assessBackupFreshness(`nope`, backupEvidence{}, now).reason, - "stale": assessBackupFreshness(`{"pbs_snapshots":[{"backup_time":"`+at(-40*time.Hour)+`","verify_state":"ok"}]}`, backupEvidence{}, now).reason, + "stale": assessBackupFreshness(`{"pbs_snapshots":[{"backup_time":"`+at(-9*24*time.Hour)+`","verify_state":"ok"}]}`, backupEvidence{}, now).reason, "verify failed": assessBackupFreshness(`{"pbs_snapshots":[{"backup_time":"`+at(-2*time.Hour)+`","verify_state":"failed"}]}`, evidenceAt(now, 2*time.Hour, 30*24*time.Hour), now).reason, "absence timed": assessBackupFreshness(`{"backups":[]}`, noEvidence(now, 240*time.Hour), now).reason, "absence unanch": assessBackupFreshness(`{"backups":[]}`, backupEvidence{}, now).reason, diff --git a/hub/internal/monitor/deadline_test.go b/hub/internal/monitor/deadline_test.go index 6bef799..33c52d4 100644 --- a/hub/internal/monitor/deadline_test.go +++ b/hub/internal/monitor/deadline_test.go @@ -112,11 +112,16 @@ func TestCheckBackupDeadlines_FreshVerifiedPBS_NoBackupAlarm(t *testing.T) { } } -// TestCheckBackupDeadlines_StalePBS_Alarms: newest backup older than 26h → alarm. +// TestCheckBackupDeadlines_StalePBS_Alarms: an offsite snapshot past the OFFSITE limit → alarm. +// +// R-82 Slice C changed the threshold this fixture must cross, NOT the behaviour it asserts. 30h was +// "stale" while one 26h limit covered every tier; under a weekly offsite tier a 30h snapshot is +// healthy, and alarming on it is exactly the cry-wolf `backupStaleAfter`'s comment predicted. The +// test still proves a stale offsite tier alarms — now at 9 days, past the 8-day offsite limit. func TestCheckBackupDeadlines_StalePBS_Alarms(t *testing.T) { st := newDeadlineStore(t) st.SaveEvent("c1", "db_dump_completed", "info", "", "{}", "controller") - report := hostReportJSON(t, [][2]string{{rfc(-30 * time.Hour), "ok"}}, nil) + report := hostReportJSON(t, [][2]string{{rfc(-9 * 24 * time.Hour), "ok"}}, nil) if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil { t.Fatal(err) } @@ -188,12 +193,19 @@ func TestAssessBackupFreshness(t *testing.T) { }{ {"fresh verified PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"ok"}]}`, false}, {"fresh unverified (none) is not a failure", `{"pbs_snapshots":[{"backup_time":"` + at(-3*time.Hour) + `","verify_state":"none"}]}`, false}, - {"stale verified", `{"pbs_snapshots":[{"backup_time":"` + at(-30*time.Hour) + `","verify_state":"ok"}]}`, true}, + // Slice C: a 30h offsite snapshot is HEALTHY under the weekly (8d) offsite limit. + {"offsite 30h is fresh under the weekly limit", `{"pbs_snapshots":[{"backup_time":"` + at(-30*time.Hour) + `","verify_state":"ok"}]}`, false}, + {"offsite 6d is fresh under the weekly limit", `{"pbs_snapshots":[{"backup_time":"` + at(-6*24*time.Hour) + `","verify_state":"ok"}]}`, false}, + {"offsite 9d is STALE past the weekly limit", `{"pbs_snapshots":[{"backup_time":"` + at(-9*24*time.Hour) + `","verify_state":"ok"}]}`, true}, {"fresh but verify failed", `{"pbs_snapshots":[{"backup_time":"` + at(-2*time.Hour) + `","verify_state":"failed"}]}`, true}, {"no snapshots and no backups", `{"pbs_snapshots":[],"backups":[]}`, true}, - {"vzdump fallback fresh success", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":true}]}`, false}, - {"vzdump only, failed → counts as none", `{"backups":[{"started_at":"` + at(-4*time.Hour) + `","success":false}]}`, true}, - {"newest vzdump fresh rescues stale PBS", `{"pbs_snapshots":[{"backup_time":"` + at(-40*time.Hour) + `","verify_state":"ok"}],"backups":[{"started_at":"` + at(-2*time.Hour) + `","success":true}]}`, false}, + {"host vzdump fresh success", `{"backups":[{"target_id":"local","started_at":"` + at(-4*time.Hour) + `","success":true}]}`, false}, + {"host vzdump only, failed → counts as none", `{"backups":[{"target_id":"local","started_at":"` + at(-4*time.Hour) + `","success":false}]}`, true}, + {"host fresh + offsite within weekly → both healthy", `{"pbs_snapshots":[{"backup_time":"` + at(-40*time.Hour) + `","verify_state":"ok"}],"backups":[{"target_id":"local","started_at":"` + at(-2*time.Hour) + `","success":true}]}`, false}, + // Slice C's core: the tiers are judged SEPARATELY. A fresh host backup must NOT rescue an + // offsite tier that is genuinely past its own (weekly) limit — the merged threshold did. + {"host fresh does NOT rescue a 9d offsite tier", `{"pbs_snapshots":[{"backup_time":"` + at(-9*24*time.Hour) + `","verify_state":"ok"}],"backups":[{"target_id":"local","started_at":"` + at(-2*time.Hour) + `","success":true}]}`, true}, + {"offsite fresh does NOT rescue a 30h host tier", `{"pbs_snapshots":[{"backup_time":"` + at(-2*time.Hour) + `","verify_state":"ok"}],"backups":[{"target_id":"local","started_at":"` + at(-30*time.Hour) + `","success":true}]}`, true}, {"unparseable report", `not json`, true}, } for _, c := range cases { diff --git a/hub/internal/monitor/deadline_tiers.go b/hub/internal/monitor/deadline_tiers.go new file mode 100644 index 0000000..72e82a3 --- /dev/null +++ b/hub/internal/monitor/deadline_tiers.go @@ -0,0 +1,214 @@ +package monitor + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// R-82 Slice C — TIER-AWARE backup freshness. +// +// R-81 merged every backup signal into one "newest" and judged it against a single 26h threshold. +// That was correct while a box had exactly one whole-guest tier. With the R-82 split it is not: +// `backupStaleAfter`'s own comment records why — +// +// "The moment PBS moves to a WEEKLY cadence, a perfectly healthy weekly snapshot is >26h old six +// days in seven and this constant alarms on it." +// +// So each tier is now judged against ITS OWN threshold, and R-81's structure is preserved intact: +// three-valued verdicts, absence anchored at first contact, and one distinct reason string per +// failure mode. + +// offsiteBackupStaleAfter is the maximum age of the newest OFFSITE (PBS) snapshot. 8 days = the 7-day +// weekly cadence plus a day of headroom — the same ratio the 26h host threshold gives a 24h cadence, +// so a healthy weekly tier never trips it and a genuinely missed week always does. +const offsiteBackupStaleAfter = 8 * 24 * time.Hour + +// backupTier names one whole-guest backup tier, for thresholds and for reason strings. +type backupTier struct { + name string // "host" | "offsite" — used verbatim in reason strings + staleWhen time.Duration +} + +var ( + tierHost = backupTier{name: "host", staleWhen: backupStaleAfter} + tierOffsite = backupTier{name: "offsite", staleWhen: offsiteBackupStaleAfter} +) + +// tierView is what one tier's evidence looks like in a single report. +type tierView struct { + expected bool // this box HAS this tier — absence is meaningful + newest time.Time // newest evidence in THIS report + have bool + verifyOK bool // false only when the newest OFFSITE snapshot's verify_state is literally "failed" + failed bool +} + +// pbsTargetSet returns the storage ids that are PBS-type backup storages on this host. +// +// THIS IS THE SLICE-A.4 RULE, and getting it wrong is the trap that slice recorded: a PBS-targeted +// vzdump appears in BOTH `backups[]` (as a Backup with target_id "felhom-pbs") AND in +// `pbs_snapshots[]` (enumerated independently from PBS by the agent's verify loop). Classifying by +// ARRAY MEMBERSHIP would therefore attribute a PBS backup to the host tier and make a stale host +// tier look fresh. Classify by TARGET TYPE — join target_id → storage_targets[].name → .type. +// +// storage_targets is preferred over pbs_dr.storage_id because pbs_dr is null on a box that has a PBS +// storage but no DR descriptor yet (drill-r50 was exactly that shape). +func pbsTargetSet(hr hostReportBackups) map[string]bool { + out := map[string]bool{} + for _, st := range hr.StorageTargets { + if strings.EqualFold(strings.TrimSpace(st.Type), "pbs") && st.Name != "" { + out[st.Name] = true + } + } + return out +} + +// splitTiers classifies one report's backup evidence into the host and offsite tiers. +// +// A tier is "expected" when the box demonstrably HAS it — either a storage of that kind is +// configured, or evidence for it exists. Without that gate every box without an offsite tier would +// alarm as soon as the anchor elapsed, which is precisely the absence-is-not-failure mistake R-81 +// exists to prevent, re-introduced one level down. +func splitTiers(hr hostReportBackups) (host, offsite tierView) { + pbs := pbsTargetSet(hr) + + for name := range pbs { + _ = name + offsite.expected = true + } + for _, st := range hr.StorageTargets { + if pbs[st.Name] { + continue + } + if strings.Contains(st.Content, "backup") { + host.expected = true + } + } + + // PBS snapshots are offsite evidence by construction. + for _, ps := range hr.PBSSnapshots { + offsite.expected = true + t, ok := parseBackupTime(ps.BackupTime) + if !ok { + continue + } + if !offsite.have || t.After(offsite.newest) { + offsite.have, offsite.newest = true, t + offsite.failed = strings.EqualFold(strings.TrimSpace(ps.VerifyState), "failed") + } + } + + // vzdump records land in whichever tier their TARGET belongs to. + for _, b := range hr.Backups { + tv := &host + if pbs[b.TargetID] { + tv = &offsite + } + tv.expected = true + if !b.Success { + continue + } + t, ok := parseBackupTime(b.StartedAt) + if !ok { + continue + } + if !tv.have || t.After(tv.newest) { + tv.have, tv.newest = true, t + if tv == &offsite { + // A vzdump record carries no verify_state; it never clears a failed snapshot verdict + // and never sets one. + _ = tv + } + } + } + return host, offsite +} + +// assessTier judges ONE tier, preserving R-81's structure exactly: absence is UNKNOWN until it +// outlives the tier's OWN threshold measured from first contact. +func assessTier(t backupTier, v tierView, windowNewest time.Time, windowHave bool, firstReportAt, now time.Time) backupAssessment { + newest, have := v.newest, v.have + if windowHave && (!have || windowNewest.After(newest)) { + newest, have = windowNewest, true + } + + if !have { + if firstReportAt.IsZero() { + return backupAssessment{verdict: verdictMissed, + reason: fmt.Sprintf("%s tier: no backup evidence in any retained host-report, and no first-contact anchor to defer against", t.name)} + } + watched := now.Sub(firstReportAt) + if watched <= t.staleWhen { + return backupAssessment{verdict: verdictUnknown, + reason: fmt.Sprintf("%s tier: no backup evidence yet, but only watching for %s (grace %s since first contact %s) — newborn tier, not a fault", + t.name, watched.Round(time.Hour), t.staleWhen, firstReportAt.Format(time.RFC3339))} + } + return backupAssessment{verdict: verdictMissed, + reason: fmt.Sprintf("%s tier: no backup evidence in any host-report for %s (limit %s, first contact %s)", + t.name, watched.Round(time.Hour), t.staleWhen, firstReportAt.Format(time.RFC3339))} + } + + if age := now.Sub(newest); age > t.staleWhen { + return backupAssessment{verdict: verdictMissed, + reason: fmt.Sprintf("%s tier: newest backup is %s old (limit %s)", t.name, age.Round(time.Hour), t.staleWhen)} + } + if v.failed { + return backupAssessment{verdict: verdictMissed, + reason: fmt.Sprintf("%s tier: newest PBS snapshot failed verification", t.name)} + } + return backupAssessment{verdict: verdictOK} +} + +// worst folds per-tier verdicts into one. MISSED dominates UNKNOWN dominates OK, and the reason +// travels with the winner so the event still names exactly one cause. When several tiers are +// missed, both reasons are joined — a box with two broken tiers must not report only one. +func worst(as ...backupAssessment) backupAssessment { + out := backupAssessment{verdict: verdictOK} + var reasons []string + for _, a := range as { + if a.verdict > out.verdict { + out.verdict = a.verdict + reasons = nil + } + if a.verdict == out.verdict && a.reason != "" { + reasons = append(reasons, a.reason) + } + } + out.reason = strings.Join(reasons, "; ") + return out +} + +// newestBackupEvidenceByTier is newestBackupEvidence's tier-aware twin: it walks the retained +// host-report window and returns the newest evidence PER TIER. +// +// Per-tier is load-bearing, not tidiness: R-81's single "newest across everything" would let a fresh +// daily host backup satisfy the offsite tier's window lookup, which is the same class of error as +// the agent-side one Slice A fixed (a fresh local backup satisfying the weekly PBS cadence). +// +// Early exit: stop as soon as every EXPECTED tier has evidence inside its own threshold — nothing +// older can change any verdict then. The healthy path reads one row. +func newestBackupEvidenceByTier(rows []store.HostReportRow, wantHost, wantOffsite bool, now time.Time) (hostNewest time.Time, hostHave bool, offNewest time.Time, offHave bool) { + for _, r := range rows { + var hr hostReportBackups + if err := json.Unmarshal([]byte(r.ReportJSON), &hr); err != nil { + continue // one malformed retained report must not blind the scan + } + h, o := splitTiers(hr) + if h.have && (!hostHave || h.newest.After(hostNewest)) { + hostNewest, hostHave = h.newest, true + } + if o.have && (!offHave || o.newest.After(offNewest)) { + offNewest, offHave = o.newest, true + } + hostSettled := !wantHost || (hostHave && now.Sub(hostNewest) <= tierHost.staleWhen) + offSettled := !wantOffsite || (offHave && now.Sub(offNewest) <= tierOffsite.staleWhen) + if hostSettled && offSettled { + break + } + } + return hostNewest, hostHave, offNewest, offHave +} diff --git a/hub/internal/monitor/deadline_tiers_test.go b/hub/internal/monitor/deadline_tiers_test.go new file mode 100644 index 0000000..45d90fe --- /dev/null +++ b/hub/internal/monitor/deadline_tiers_test.go @@ -0,0 +1,192 @@ +package monitor + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +// R-82 Slice C — tier-aware thresholds. +// +// The whole point: a healthy WEEKLY offsite snapshot is >26h old six days in seven, so the merged +// threshold would cry wolf on it — exactly what `backupStaleAfter`'s comment predicted. Each tier is +// now judged against its own limit, and R-81's structure (three verdicts, anchored absence, one +// distinct reason per failure mode) is preserved intact. + +func tieredReport(pbsAge time.Duration, verify string, hostAge time.Duration, now time.Time) string { + at := func(d time.Duration) string { return now.Add(-d).Format(time.RFC3339) } + return `{ + "storage_targets":[ + {"name":"local","type":"local","content":"backup,iso"}, + {"name":"felhom-pbs","type":"pbs","content":"backup"} + ], + "pbs_snapshots":[{"backup_time":"` + at(pbsAge) + `","verify_state":"` + verify + `"}], + "backups":[{"target_id":"local","started_at":"` + at(hostAge) + `","success":true}] + }` +} + +func freshEv(now time.Time, hostAge, offAge time.Duration) backupEvidence { + return backupEvidence{ + newestHost: now.Add(-hostAge), haveHost: true, + newestOffsite: now.Add(-offAge), haveOffsite: true, + firstReportAt: now.Add(-60 * 24 * time.Hour), + } +} + +// ── THE SCENARIO ───────────────────────────────────────────────────────────────────────────── +// +// A 6-day-old offsite snapshot with a fresh host backup is SILENT; the same snapshot at 9 days +// ALARMS. +// +// COMPANION RED-PROOF (observed): give tierOffsite the host threshold +// (`tierOffsite = backupTier{name:"offsite", staleWhen: backupStaleAfter}`) — i.e. restore the +// single merged threshold — and the 6-day case fails with +// +// 6-day-old offsite snapshot is HEALTHY under a weekly cadence and must be SILENT; +// got verdict=2 reason="offsite tier: newest backup is 144h0m0s old (limit 26h0m0s)" +// +// which is verbatim the cry-wolf this slice exists to remove. Restored. +func TestTierAware_SixDayOffsiteSilent_NineDayAlarms(t *testing.T) { + now := time.Date(2026, 7, 26, 3, 0, 0, 0, time.UTC) + + t.Run("6 days offsite + fresh host → SILENT", func(t *testing.T) { + got := assessBackupFreshness(tieredReport(6*24*time.Hour, "ok", 2*time.Hour, now), + freshEv(now, 2*time.Hour, 6*24*time.Hour), now) + if got.missed() { + t.Fatalf("6-day-old offsite snapshot is HEALTHY under a weekly cadence and must be SILENT; got verdict=%d reason=%q", + got.verdict, got.reason) + } + if got.verdict != verdictOK { + t.Fatalf("both tiers have evidence inside their limits → OK, not a deferred UNKNOWN; got verdict=%d reason=%q", got.verdict, got.reason) + } + }) + + t.Run("9 days offsite + fresh host → ALARMS, naming the offsite tier", func(t *testing.T) { + got := assessBackupFreshness(tieredReport(9*24*time.Hour, "ok", 2*time.Hour, now), + freshEv(now, 2*time.Hour, 9*24*time.Hour), now) + if !got.missed() { + t.Fatalf("9-day-old offsite snapshot is past the 8-day limit and MUST alarm; got verdict=%d reason=%q", + got.verdict, got.reason) + } + if !strings.HasPrefix(got.reason, "offsite tier:") { + t.Fatalf("the reason must name the OFFSITE tier — a merged message would have made the R-80 diagnosis impossible; got %q", got.reason) + } + if strings.Contains(got.reason, "host tier") { + t.Fatalf("the host tier is fresh and must not appear in the reason; got %q", got.reason) + } + }) +} + +// Neither tier rescues the other. This is the merged-threshold bug in both directions. +func TestTierAware_TiersDoNotRescueEachOther(t *testing.T) { + now := time.Date(2026, 7, 26, 3, 0, 0, 0, time.UTC) + + t.Run("fresh host does not rescue a stale offsite tier", func(t *testing.T) { + got := assessBackupFreshness(tieredReport(9*24*time.Hour, "ok", time.Hour, now), + freshEv(now, time.Hour, 9*24*time.Hour), now) + if !got.missed() { + t.Fatalf("a fresh DAILY backup must not satisfy the WEEKLY tier; got %q", got.reason) + } + }) + + t.Run("fresh offsite does not rescue a stale host tier", func(t *testing.T) { + got := assessBackupFreshness(tieredReport(time.Hour, "ok", 30*time.Hour, now), + freshEv(now, 30*time.Hour, time.Hour), now) + if !got.missed() { + t.Fatalf("a fresh WEEKLY snapshot must not satisfy the DAILY tier; got %q", got.reason) + } + if !strings.Contains(got.reason, "host tier") { + t.Fatalf("the reason must name the HOST tier; got %q", got.reason) + } + }) + + t.Run("both stale → BOTH named", func(t *testing.T) { + got := assessBackupFreshness(tieredReport(9*24*time.Hour, "ok", 30*time.Hour, now), + freshEv(now, 30*time.Hour, 9*24*time.Hour), now) + if !got.missed() { + t.Fatalf("both tiers stale must alarm; got %q", got.reason) + } + if !strings.Contains(got.reason, "host tier") || !strings.Contains(got.reason, "offsite tier") { + t.Fatalf("a box with TWO broken tiers must not report only one; got %q", got.reason) + } + }) +} + +// ── The A.4 rule: classify by TARGET TYPE, never by array membership ───────────────────────── +// +// A PBS-targeted vzdump appears in BOTH arrays. Treating every `backups[]` entry as the host tier +// would let a PBS backup make a STALE HOST TIER LOOK FRESH — silently losing the daily tier's alarm. +func TestTierAware_PBSTargetedVzdumpIsNotHostEvidence(t *testing.T) { + now := time.Date(2026, 7, 26, 3, 0, 0, 0, time.UTC) + at := func(d time.Duration) string { return now.Add(-d).Format(time.RFC3339) } + + // The host tier's own backup is 40h old (stale). A FRESH PBS-targeted vzdump sits in the same + // array — exactly the double-appearance Slice A.4 recorded. + report := `{ + "storage_targets":[ + {"name":"local","type":"local","content":"backup"}, + {"name":"felhom-pbs","type":"pbs","content":"backup"} + ], + "pbs_snapshots":[{"backup_time":"` + at(time.Hour) + `","verify_state":"ok"}], + "backups":[ + {"target_id":"local","started_at":"` + at(40*time.Hour) + `","success":true}, + {"target_id":"felhom-pbs","started_at":"` + at(time.Hour) + `","success":true} + ] + }` + ev := backupEvidence{ + newestHost: now.Add(-40 * time.Hour), haveHost: true, + newestOffsite: now.Add(-time.Hour), haveOffsite: true, + firstReportAt: now.Add(-60 * 24 * time.Hour), + } + got := assessBackupFreshness(report, ev, now) + if !got.missed() { + t.Fatalf("a PBS-targeted vzdump must NOT count as host-tier evidence — the host tier is 40h stale; got verdict=%d reason=%q", + got.verdict, got.reason) + } + if !strings.Contains(got.reason, "host tier") { + t.Fatalf("the stale HOST tier must be named; got %q", got.reason) + } +} + +// splitTiers is the classifier the whole slice rests on. +func TestSplitTiers_Classification(t *testing.T) { + now := time.Date(2026, 7, 26, 3, 0, 0, 0, time.UTC) + at := func(d time.Duration) string { return now.Add(-d).Format(time.RFC3339) } + + t.Run("no storage_targets and no evidence → neither tier expected (old-agent fallback)", func(t *testing.T) { + var hr hostReportBackups + mustUnmarshal(t, `{"pbs_snapshots":[],"backups":[]}`, &hr) + h, o := splitTiers(hr) + if h.expected || o.expected { + t.Fatalf("nothing identifiable must leave BOTH tiers unexpected so the pre-Slice-C path runs; got host=%v offsite=%v", h.expected, o.expected) + } + }) + + t.Run("a PBS storage alone makes the offsite tier expected", func(t *testing.T) { + var hr hostReportBackups + mustUnmarshal(t, `{"storage_targets":[{"name":"felhom-pbs","type":"pbs","content":"backup"}]}`, &hr) + _, o := splitTiers(hr) + if !o.expected || o.have { + t.Fatalf("a configured-but-empty offsite tier is EXPECTED with no evidence — that is the R-82 fault; got %+v", o) + } + }) + + t.Run("verify_state failed is carried on the newest offsite snapshot only", func(t *testing.T) { + var hr hostReportBackups + mustUnmarshal(t, `{"pbs_snapshots":[ + {"backup_time":"`+at(48*time.Hour)+`","verify_state":"failed"}, + {"backup_time":"`+at(time.Hour)+`","verify_state":"ok"}]}`, &hr) + _, o := splitTiers(hr) + if o.failed { + t.Fatalf("an OLDER failed snapshot must not condemn a newer good one; got %+v", o) + } + }) +} + +func mustUnmarshal(t *testing.T, s string, v any) { + t.Helper() + if err := json.Unmarshal([]byte(s), v); err != nil { + t.Fatal(err) + } +}