package monitor import ( "encoding/json" "fmt" "log" "sync" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // R-85 Part 2 — the restore-test result becomes a SIGNAL. // // Until now a failed restore-test was a `[WARN]` log line in the ingest handler and nothing else: // no event, no notification, no staleness gauge. That was true for the LOCAL tier that was already // being tested — so the loudest DR signal this system produces was, in practice, inaudible. // // TWO SIGNALS, DELIBERATELY NOT MERGED. They mean different things and warrant different urgency: // // restore_test_failed — a run completed and did NOT pass. Something is broken NOW. // restore_test_stale — a tier has not been PROVEN within its expected interval. Nothing has // necessarily broken; we simply no longer know whether it works. // // Merging them would collapse "your DR is broken" into "your DR is unverified", and the second is // the one that quietly becomes the first. // // ── THE INVARIANT, inherited from R-81 ──────────────────────────────────────────────────────── // // Absence of a signal is UNKNOWN, and becomes a fault only once it has outlived an ANCHORED window. // This monitor family has made the opposite mistake three times (hub v0.12.0, v0.73.0, R-81); this // is a NEW monitor written straight after the third, so it copies R-81's verdict structure rather // than re-deriving it. A tier never proven on a newborn box is UNKNOWN, never FAILED. // restoreProvenStaleAfter is how long a tier may go unproven before it is called stale. // // Derivation, not a guess: the restore-test cadence is 24h and rotation is oldest-first across two // tiers, so each tier is proven roughly every 2 days. 7 days therefore tolerates ~3 consecutive // missed opportunities before alarming — loud enough to matter, quiet enough not to fire on one // skipped cycle (a deferral behind a long backup is normal, not a fault). It is also comfortably // inside the 2-week offsite retention (operator ruling 2026-07-26), so a tier is never reported // stale against an archive that is about to be pruned anyway. const restoreProvenStaleAfter = 7 * 24 * time.Hour // Event types. Operator-tier only — see the dispatcher note in RestoreTestChecker. const ( EventRestoreTestFailed = "restore_test_failed" EventRestoreTestStale = "restore_test_stale" ) // hostReportRestoreTests is the slice of a host-report this checker reads. type hostReportRestoreTests struct { RestoreTests []struct { SourceArchive string `json:"source_archive"` SourceTier string `json:"source_tier"` Pass bool `json:"pass"` Error string `json:"error"` TestedAt string `json:"tested_at"` } `json:"restore_tests"` StorageTargets []struct { Name string `json:"name"` Type string `json:"type"` Content string `json:"content"` } `json:"storage_targets"` } // RestoreTestChecker watches restore-test outcomes per customer. type RestoreTestChecker struct { store *store.Store logger *log.Logger onEvent EventNotifyFunc now func() time.Time mu sync.Mutex // failStates is edge-trigger state: customerID|archive → already reported. A failing tier is // re-reported only when the FAILING RUN changes, so a permanently broken tier does not emit // every 60s sweep — the flapping-spam rule every checker here follows. failStates map[string]bool // staleStates is customerID|tier → "ok"|"stale", so the stale signal is edge-triggered too. staleStates map[string]string } // NewRestoreTestChecker builds the checker. onEvent may be nil. func NewRestoreTestChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *RestoreTestChecker { return &RestoreTestChecker{ store: s, logger: logger, onEvent: onEvent, now: func() time.Time { return time.Now().UTC() }, failStates: map[string]bool{}, staleStates: map[string]string{}, } } // Check sweeps every active customer. Safe to call on the shared monitor ticker. func (c *RestoreTestChecker) Check() { ids, err := c.store.GetActiveCustomerIDs() if err != nil { c.logger.Printf("[WARN] restore-test check: failed to list customers: %v", err) return } now := c.now() for _, id := range ids { latest, rerr := c.store.GetLatestHostReportJSON(id) if rerr != nil || latest == "" { continue // no agent host-report → nothing to judge (the R-81 legacy-shape branch) } c.checkFailure(id, latest) c.checkStaleness(id, latest, now) } } // checkFailure raises restore_test_failed when the latest report carries a FAILED run. func (c *RestoreTestChecker) checkFailure(customerID, reportJSON string) { var hr hostReportRestoreTests if json.Unmarshal([]byte(reportJSON), &hr) != nil { return } for _, rt := range hr.RestoreTests { if rt.Pass { // A pass CLEARS the edge state for that archive so a later failure re-reports. c.mu.Lock() delete(c.failStates, customerID+"|"+rt.SourceArchive) c.mu.Unlock() continue } key := customerID + "|" + rt.SourceArchive c.mu.Lock() already := c.failStates[key] c.failStates[key] = true c.mu.Unlock() if already { continue } tier := rt.SourceTier if tier == "" { tier = "unknown" } msg := fmt.Sprintf("Restore-test FAILED on the %s tier: archive %s could not be restored+booted (%s)", tier, rt.SourceArchive, rt.Error) c.emit(customerID, EventRestoreTestFailed, "error", msg) } } // checkStaleness raises restore_test_stale for a tier not PROVEN within restoreProvenStaleAfter. // // "Proven" means a PASSING run for that tier, found across the hub's retained host-report window — // the R-81 mechanism. The window is load-bearing here: the agent reports only its LATEST // restore-test and its store is in-memory, so the latest report alone cannot answer "when was the // OTHER tier last proven?". The hub's own history can. func (c *RestoreTestChecker) checkStaleness(customerID, latestJSON string, now time.Time) { var latest hostReportRestoreTests if json.Unmarshal([]byte(latestJSON), &latest) != nil { return } tiers := expectedRestoreTiers(latest) if len(tiers) == 0 { return } rows, err := c.store.GetHostReportsSince(customerID, now.Add(-2*restoreProvenStaleAfter)) if err != nil { c.logger.Printf("[WARN] restore-test check: window read failed for %s: %v", customerID, err) return } proven := lastProvenPerTier(rows) first, ferr := c.store.GetFirstHostReportAt(customerID) if ferr != nil { c.logger.Printf("[WARN] restore-test check: first-contact read failed for %s: %v", customerID, ferr) return } for _, tier := range tiers { v := assessRestoreProven(tier, proven[tier], first, now) key := customerID + "|" + tier c.mu.Lock() prev := c.staleStates[key] state := "ok" if v.verdict == verdictMissed { state = "stale" } c.staleStates[key] = state c.mu.Unlock() switch { case state == "stale" && prev != "stale": c.emit(customerID, EventRestoreTestStale, "warning", v.reason) case v.verdict == verdictUnknown && prev == "": // Make the deferral VISIBLE exactly once, the v0.73.0 / R-81 precedent: a quiet check // must never be indistinguishable from a check that did not run. c.logger.Printf("[INFO] restore-test staleness: %s %s", customerID, v.reason) } } } // assessRestoreProven is the per-tier verdict. PURE (now injected) so the policy is unit-tested — // the property that made R-81 provable, kept deliberately. // // no proof, anchor NOT elapsed → UNKNOWN (newborn box; never an alarm) // no proof, anchor elapsed → MISSED // proof older than the window → MISSED // otherwise → OK func assessRestoreProven(tier string, provenAt, firstReportAt, now time.Time) backupAssessment { if provenAt.IsZero() { if firstReportAt.IsZero() { return backupAssessment{verdict: verdictMissed, reason: fmt.Sprintf("%s tier: never restore-proven, and no first-contact anchor to defer against", tier)} } watched := now.Sub(firstReportAt) if watched <= restoreProvenStaleAfter { return backupAssessment{verdict: verdictUnknown, reason: fmt.Sprintf("%s tier: not restore-proven yet, but only watching for %s (grace %s since first contact %s) — newborn, not a fault", tier, watched.Round(time.Hour), restoreProvenStaleAfter, firstReportAt.Format(time.RFC3339))} } return backupAssessment{verdict: verdictMissed, reason: fmt.Sprintf("%s tier: NEVER successfully restore-proven in %s of watching (limit %s) — the tier is unverified, not known-broken", tier, watched.Round(time.Hour), restoreProvenStaleAfter)} } if age := now.Sub(provenAt); age > restoreProvenStaleAfter { return backupAssessment{verdict: verdictMissed, reason: fmt.Sprintf("%s tier: last successful restore-test was %s ago (limit %s) — the tier is unverified, not known-broken", tier, age.Round(time.Hour), restoreProvenStaleAfter)} } return backupAssessment{verdict: verdictOK} } // expectedRestoreTiers names the tiers this box actually HAS, so a box without an offsite tier is // never reported stale for one. Same gate as Slice C's `expected`, and for the same reason: without // it every box lacking a tier would alarm once the anchor elapsed — absence-is-not-failure, // re-introduced one level down. func expectedRestoreTiers(hr hostReportRestoreTests) []string { var out []string seenHost, seenOffsite := false, false for _, st := range hr.StorageTargets { isPBS := st.Type == "pbs" if isPBS && !seenOffsite { out = append(out, "pbs") seenOffsite = true continue } if !isPBS && !seenHost && containsBackupContent(st.Content) { out = append(out, "local") seenHost = true } } return out } func containsBackupContent(content string) bool { for i := 0; i+6 <= len(content); i++ { if content[i:i+6] == "backup" { return true } } return false } // lastProvenPerTier walks the retained window for the newest PASSING run per tier. func lastProvenPerTier(rows []store.HostReportRow) map[string]time.Time { out := map[string]time.Time{} for _, r := range rows { var hr hostReportRestoreTests if json.Unmarshal([]byte(r.ReportJSON), &hr) != nil { continue // one malformed retained report must not blind the scan } for _, rt := range hr.RestoreTests { if !rt.Pass || rt.SourceTier == "" { continue } t, err := time.Parse(time.RFC3339, rt.TestedAt) if err != nil { continue } if cur, ok := out[rt.SourceTier]; !ok || t.After(cur) { out[rt.SourceTier] = t.UTC() } } } return out } // emit saves the event and notifies. OPERATOR-TIER ONLY: neither type has a `customerMessages` // entry, so the dispatcher cannot route it to a customer. That is deliberate — a customer can take // no action on a failed restore-test, and „a visszaállítási teszt nem sikerült" would frighten // without informing. A PERSISTENTLY unproven DR tier may eventually warrant a customer-visible // statement, but that needs copy review, not a side effect of this task. func (c *RestoreTestChecker) emit(customerID, eventType, severity, msg string) { if _, err := c.store.SaveEvent(customerID, eventType, severity, msg, "{}", "hub"); err != nil { c.logger.Printf("[WARN] restore-test check: failed to save %s for %s: %v", eventType, customerID, err) return } c.logger.Printf("[%s] restore-test: %s %s", severityLabel(severity), customerID, msg) if c.onEvent != nil { c.onEvent(customerID, eventType, severity, msg, "{}", "hub") } } func severityLabel(sev string) string { if sev == "error" { return "ERROR" } return "WARN" }