package monitor import ( "encoding/json" "fmt" "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // R-82 Slice C — TIER-AWARE backup freshness. // // R-81 merged every backup signal into one "newest" and judged it against a single 26h threshold. // That was correct while a box had exactly one whole-guest tier. With the R-82 split it is not: // `backupStaleAfter`'s own comment records why — // // "The moment PBS moves to a WEEKLY cadence, a perfectly healthy weekly snapshot is >26h old six // days in seven and this constant alarms on it." // // So each tier is now judged against ITS OWN threshold, and R-81's structure is preserved intact: // three-valued verdicts, absence anchored at first contact, and one distinct reason string per // failure mode. // offsiteBackupStaleAfter is the maximum age of the newest OFFSITE (PBS) snapshot. 8 days = the 7-day // weekly cadence plus a day of headroom — the same ratio the 26h host threshold gives a 24h cadence, // so a healthy weekly tier never trips it and a genuinely missed week always does. const offsiteBackupStaleAfter = 8 * 24 * time.Hour // backupTier names one whole-guest backup tier, for thresholds and for reason strings. type backupTier struct { name string // "host" | "offsite" — used verbatim in reason strings staleWhen time.Duration } var ( tierHost = backupTier{name: "host", staleWhen: backupStaleAfter} tierOffsite = backupTier{name: "offsite", staleWhen: offsiteBackupStaleAfter} ) // tierView is what one tier's evidence looks like in a single report. type tierView struct { expected bool // this box HAS this tier — absence is meaningful newest time.Time // newest evidence in THIS report have bool verifyOK bool // false only when the newest OFFSITE snapshot's verify_state is literally "failed" failed bool } // pbsTargetSet returns the storage ids that are PBS-type backup storages on this host. // // THIS IS THE SLICE-A.4 RULE, and getting it wrong is the trap that slice recorded: a PBS-targeted // vzdump appears in BOTH `backups[]` (as a Backup with target_id "felhom-pbs") AND in // `pbs_snapshots[]` (enumerated independently from PBS by the agent's verify loop). Classifying by // ARRAY MEMBERSHIP would therefore attribute a PBS backup to the host tier and make a stale host // tier look fresh. Classify by TARGET TYPE — join target_id → storage_targets[].name → .type. // // storage_targets is preferred over pbs_dr.storage_id because pbs_dr is null on a box that has a PBS // storage but no DR descriptor yet (drill-r50 was exactly that shape). func pbsTargetSet(hr hostReportBackups) map[string]bool { out := map[string]bool{} for _, st := range hr.StorageTargets { if strings.EqualFold(strings.TrimSpace(st.Type), "pbs") && st.Name != "" { out[st.Name] = true } } return out } // splitTiers classifies one report's backup evidence into the host and offsite tiers. // // A tier is "expected" when the box demonstrably HAS it — either a storage of that kind is // configured, or evidence for it exists. Without that gate every box without an offsite tier would // alarm as soon as the anchor elapsed, which is precisely the absence-is-not-failure mistake R-81 // exists to prevent, re-introduced one level down. func splitTiers(hr hostReportBackups) (host, offsite tierView) { pbs := pbsTargetSet(hr) for name := range pbs { _ = name offsite.expected = true } for _, st := range hr.StorageTargets { if pbs[st.Name] { continue } if strings.Contains(st.Content, "backup") { host.expected = true } } // PBS snapshots are offsite evidence by construction. for _, ps := range hr.PBSSnapshots { offsite.expected = true t, ok := parseBackupTime(ps.BackupTime) if !ok { continue } if !offsite.have || t.After(offsite.newest) { offsite.have, offsite.newest = true, t offsite.failed = strings.EqualFold(strings.TrimSpace(ps.VerifyState), "failed") } } // vzdump records land in whichever tier their TARGET belongs to. for _, b := range hr.Backups { tv := &host if pbs[b.TargetID] { tv = &offsite } tv.expected = true if !b.Success { continue } t, ok := parseBackupTime(b.StartedAt) if !ok { continue } if !tv.have || t.After(tv.newest) { tv.have, tv.newest = true, t if tv == &offsite { // A vzdump record carries no verify_state; it never clears a failed snapshot verdict // and never sets one. _ = tv } } } return host, offsite } // assessTier judges ONE tier, preserving R-81's structure exactly: absence is UNKNOWN until it // outlives the tier's OWN threshold measured from first contact. func assessTier(t backupTier, v tierView, windowNewest time.Time, windowHave bool, firstReportAt, now time.Time) backupAssessment { newest, have := v.newest, v.have if windowHave && (!have || windowNewest.After(newest)) { newest, have = windowNewest, true } if !have { if firstReportAt.IsZero() { return backupAssessment{verdict: verdictMissed, reason: fmt.Sprintf("%s tier: no backup evidence in any retained host-report, and no first-contact anchor to defer against", t.name)} } watched := now.Sub(firstReportAt) if watched <= t.staleWhen { return backupAssessment{verdict: verdictUnknown, reason: fmt.Sprintf("%s tier: no backup evidence yet, but only watching for %s (grace %s since first contact %s) — newborn tier, not a fault", t.name, watched.Round(time.Hour), t.staleWhen, firstReportAt.Format(time.RFC3339))} } return backupAssessment{verdict: verdictMissed, reason: fmt.Sprintf("%s tier: no backup evidence in any host-report for %s (limit %s, first contact %s)", t.name, watched.Round(time.Hour), t.staleWhen, firstReportAt.Format(time.RFC3339))} } if age := now.Sub(newest); age > t.staleWhen { return backupAssessment{verdict: verdictMissed, reason: fmt.Sprintf("%s tier: newest backup is %s old (limit %s)", t.name, age.Round(time.Hour), t.staleWhen)} } if v.failed { return backupAssessment{verdict: verdictMissed, reason: fmt.Sprintf("%s tier: newest PBS snapshot failed verification", t.name)} } return backupAssessment{verdict: verdictOK} } // worst folds per-tier verdicts into one. MISSED dominates UNKNOWN dominates OK, and the reason // travels with the winner so the event still names exactly one cause. When several tiers are // missed, both reasons are joined — a box with two broken tiers must not report only one. func worst(as ...backupAssessment) backupAssessment { out := backupAssessment{verdict: verdictOK} var reasons []string for _, a := range as { if a.verdict > out.verdict { out.verdict = a.verdict reasons = nil } if a.verdict == out.verdict && a.reason != "" { reasons = append(reasons, a.reason) } } out.reason = strings.Join(reasons, "; ") return out } // newestBackupEvidenceByTier is newestBackupEvidence's tier-aware twin: it walks the retained // host-report window and returns the newest evidence PER TIER. // // Per-tier is load-bearing, not tidiness: R-81's single "newest across everything" would let a fresh // daily host backup satisfy the offsite tier's window lookup, which is the same class of error as // the agent-side one Slice A fixed (a fresh local backup satisfying the weekly PBS cadence). // // Early exit: stop as soon as every EXPECTED tier has evidence inside its own threshold — nothing // older can change any verdict then. The healthy path reads one row. func newestBackupEvidenceByTier(rows []store.HostReportRow, wantHost, wantOffsite bool, now time.Time) (hostNewest time.Time, hostHave bool, offNewest time.Time, offHave bool) { for _, r := range rows { var hr hostReportBackups if err := json.Unmarshal([]byte(r.ReportJSON), &hr); err != nil { continue // one malformed retained report must not blind the scan } h, o := splitTiers(hr) if h.have && (!hostHave || h.newest.After(hostNewest)) { hostNewest, hostHave = h.newest, true } if o.have && (!offHave || o.newest.After(offNewest)) { offNewest, offHave = o.newest, true } hostSettled := !wantHost || (hostHave && now.Sub(hostNewest) <= tierHost.staleWhen) offSettled := !wantOffsite || (offHave && now.Sub(offNewest) <= tierOffsite.staleWhen) if hostSettled && offSettled { break } } return hostNewest, hostHave, offNewest, offHave }