package monitor import ( "encoding/json" "fmt" "log" "sync" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // OffsiteChecker (SLICE 4) watches each customer's offsite backup health from the controller report's // `offsite` status object. Two independent signals, one checker (sibling of StorageFillChecker — same // born/persistent, escalation-only emit, recovery re-arm shape; NOT bolted onto the disk checkers — // different data source, different remedy text): // // - FILL: repo_size_bytes vs the shared-model soft quota (quota_gb>0) at warn 90% / crit 95% — the // operator's early warning before the controller's own 100% run-refusal bites the customer. // - STALENESS: enabled + escrowed but no run in >48h (or never) — the silently-STUCK detector. A // RECENTLY-failing offsite is NOT stale (backup_failed already alerts it); staleness is the // complement: nothing is even trying. Pending/disabled targets are normal onboarding, never stale. // // Reports without an `offsite` object (pre-v0.109 controllers, offbox not enabled) are skipped nil-safe. type OffsiteChecker struct { store *store.Store logger *log.Logger onEvent EventNotifyFunc staleAfter time.Duration // legacyWarned tracks the one-shot R-100 legacy-degrade log, per customer. legacyMu sync.Mutex legacyWarned map[string]bool now func() time.Time // injectable clock (tests) mu sync.Mutex fillStates map[string]string // customerID → fill band staleStates map[string]string // customerID → "ok" | "stale" } const defaultOffsiteStaleAfter = 48 * time.Hour // offsiteReport mirrors the controller report's `offsite` object (v0.109.0). type offsiteReport struct { Enabled bool `json:"enabled"` EscrowState string `json:"escrow_state"` LastRun string `json:"last_run"` LastStatus string `json:"last_status"` // LastSuccess (R-100) is the last run that actually SUCCEEDED — the staleness anchor. Absent on a // controller older than v0.181.0; isStale degrades explicitly in that case, see there. LastSuccess string `json:"last_success"` SnapshotCount int `json:"snapshot_count"` RepoSizeBytes int64 `json:"repo_size_bytes"` QuotaGB int `json:"quota_gb"` } // NewOffsiteChecker builds the checker. Same seeding philosophy as StorageFillChecker: already-breached // customers are left UNSEEDED so their first Check emits (born/persistent); the dispatcher's cooldown // dedups a hub restart. func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNotifyFunc, logger *log.Logger) *OffsiteChecker { if staleAfter <= 0 { staleAfter = defaultOffsiteStaleAfter } oc := &OffsiteChecker{ store: s, logger: logger, onEvent: onEvent, staleAfter: staleAfter, now: time.Now, fillStates: make(map[string]string), staleStates: make(map[string]string), } customers, err := s.GetCustomers() if err != nil { logger.Printf("[WARN] Offsite checker: failed to seed states: %v", err) return oc } var seeded int for _, c := range customers { off := parseOffsite(c.ReportJSON) if off == nil || s.IsCustomerBlocked(c.CustomerID) { continue } if band := oc.fillBand(off); band == bandOK { oc.fillStates[c.CustomerID] = bandOK seeded++ } if !oc.isStale(c.CustomerID, off) { oc.staleStates[c.CustomerID] = "ok" } } logger.Printf("[INFO] Offsite checker initialized: fill warn=90%% crit=95%%, stale after %s, %d ok-seeded", staleAfter, seeded) return oc } func parseOffsite(reportJSON string) *offsiteReport { var r struct { Offsite *offsiteReport `json:"offsite"` } if json.Unmarshal([]byte(reportJSON), &r) != nil { return nil } return r.Offsite // nil when absent (old controller / offbox not enabled) — the caller skips } // fillBand maps the quota usage to a band. quota<=0 (dedicated/unset) never alerts. func (oc *OffsiteChecker) fillBand(off *offsiteReport) string { if off.QuotaGB <= 0 || off.RepoSizeBytes <= 0 { return bandOK } pct := float64(off.RepoSizeBytes) * 100 / float64(int64(off.QuotaGB)<<30) return bandForPercent(pct, 90, 95) } // isStale: enabled + ESCROWED (the only state where runs are expected) with no SUCCESSFUL run in // >staleAfter (or never ran, ANCHORED — see below). Pending/disabled = normal onboarding, never stale. // // R-100 — PRESENCE IS NOT SUCCESS. This counted from `LastRun`, which the controller writes // unconditionally at the end of every run INCLUDING failures. So it asked "how long since we last // TRIED", and a tier failing on every single run refreshed the clock nightly and read as perfectly // fresh forever. It now counts from `LastSuccess`, which only a successful run advances. // // The old comment here said "a recent-but-failing run is NOT stale (backup_failed owns that signal)", // and that WAS true — `backup_failed` does fire, nightly, and reaches the operator. The defect is not // silence, it is DEFEATED DEFENCE IN DEPTH: this checker is the hub-side, pull-based net that exists to // be independent of controller-PUSHED events, and anchoring it on a field the failing controller keeps // refreshing made it depend on the very thing it backs up. F-HUB (the hub dropping an event under // SQLITE_BUSY, no retry) is exactly that loss. INVARIANT pinned by TestIsStale_* in offsite_r100_test.go. // // Part-7 (v0.73.0) — the never-ran branch no longer fires on sight. The 2026-07-23 cry-wolf: // demo-hp's tier was repaired and escrowed at 10:01Z and offsite_stale fired MINUTES later // (`last_run:"" … threshold 48h`), because "enabled + escrowed + never ran" had no time anchor. // Boundary: reaching this code at all means the latest report CARRIES the offsite object — i.e. // the v0.72.0 delivery state is `applied` (Check nil-skips everything else); pre-applied // never-ran shapes are offsite_delivery_stuck's alone — ONE STATE, ONE OWNER, never both. // Anchor: the newest of one_time_secrets.consumed_at (delivery completed) and the customer's // escrow-blob timestamp (host_escrow.updated_at/created_at — runs become POSSIBLE only at the // ceremony). The EXISTING staleAfter threshold, anchored there, IS the grace — no new knob. func (oc *OffsiteChecker) isStale(customerID string, off *offsiteReport) bool { if !off.Enabled || off.EscrowState != "escrowed" { return false } // NEVER RAN AT ALL — unchanged v0.73.0 behaviour, and it must stay unchanged. A newborn box has no // LastRun and no LastSuccess; alarming it on sight is the 2026-07-23 cry-wolf. Checked on LastRun // (not LastSuccess) deliberately: LastRun is the "has anything ever happened here" signal, and a box // whose first run FAILED has a LastRun but no LastSuccess — that is a run, not a newborn, and it // belongs on the anchored path below. if off.LastRun == "" { anchor := oc.neverRanAnchor(customerID) if anchor.IsZero() { return true // legacy shape (no secret timestamps, no escrow row) — fail toward visibility, as before } return oc.now().Sub(anchor) > oc.staleAfter } // LEGACY CONTROLLER — it has run, but sends no last_success (pre-v0.181.0). Handle it EXPLICITLY, // in the direction that preserves known behaviour: count from LastRun exactly as before. // - treating absence as FAILURE would alarm every un-upgraded box in the fleet at once; // - treating it as SUCCESS is the bug. // Preserving today's behaviour is correct here; the controller version floor drives the upgrade. // Same degrade direction as R-88 Part 2's age_state, and for the same reason. if off.LastSuccess == "" { oc.warnLegacyOnce(customerID) t, err := time.Parse(time.RFC3339, off.LastRun) if err != nil { return true // unparseable = unknown-old — fail toward visibility } return oc.now().Sub(t) > oc.staleAfter } // THE ANCHORED VERDICT. Note what is NOT consulted: LastStatus. A single failed night does not make // a tier stale — the threshold simply keeps running from the last good run, so one blip is tolerated // and a persistent failure is caught. Reading LastStatus here ("error ⇒ stale") would alarm on every // transient blip, which is the F-A1 noise path that trains an operator to ignore the alarm. // `running` is a real wire value (a report captured mid-run) and is likewise none of our business. t, err := time.Parse(time.RFC3339, off.LastSuccess) if err != nil { return true // unparseable = unknown-old — fail toward visibility } return oc.now().Sub(t) > oc.staleAfter } // warnLegacyOnce logs the legacy degrade a single time per customer. Once, because this is a // steady-state condition until the box upgrades — a per-cycle line would be pure noise — but it must be // logged at all, so a fleet silently running on the old anchor is visible rather than assumed. func (oc *OffsiteChecker) warnLegacyOnce(customerID string) { oc.legacyMu.Lock() defer oc.legacyMu.Unlock() if oc.legacyWarned == nil { oc.legacyWarned = map[string]bool{} } if oc.legacyWarned[customerID] { return } oc.legacyWarned[customerID] = true if oc.logger != nil { oc.logger.Printf("[WARN] [offsite] %s: controller sends no last_success — staleness degraded to the last-ATTEMPT anchor (pre-v0.181.0 controller; a persistently failing tier will not go stale here until it upgrades)", customerID) } } // neverRanAnchor returns the newest hub-held timestamp from which a never-ran-but-applied tier's // staleness may be counted (zero when the hub holds neither — the pre-v0.5x legacy shape). func (oc *OffsiteChecker) neverRanAnchor(customerID string) time.Time { var anchor time.Time if info, err := oc.store.GetOneTimeSecretInfo(customerID); err == nil && info != nil && info.ConsumedAt.After(anchor) { anchor = info.ConsumedAt } if t, err := oc.store.LatestEscrowTimeForCustomer(customerID); err == nil && t.After(anchor) { anchor = t } return anchor } // Check evaluates every customer's latest report. Escalation-only emits; recovery re-arms silently. func (oc *OffsiteChecker) Check() { customers, err := oc.store.GetCustomers() if err != nil { oc.logger.Printf("[WARN] Offsite check failed: %v", err) return } oc.mu.Lock() defer oc.mu.Unlock() seen := make(map[string]bool, len(customers)) for _, c := range customers { // GetCustomers can return the same customer twice when two reports tie on received_at // (second-resolution timestamps) — process each customer once per sweep. if seen[c.CustomerID] { continue } seen[c.CustomerID] = true off := parseOffsite(c.ReportJSON) if off == nil { delete(oc.fillStates, c.CustomerID) // vanished object (disabled / downgraded) → re-arm delete(oc.staleStates, c.CustomerID) continue } if oc.store.IsCustomerBlocked(c.CustomerID) { delete(oc.fillStates, c.CustomerID) delete(oc.staleStates, c.CustomerID) continue } // FILL (quota>0 only) newBand := oc.fillBand(off) if bandRank(newBand) > bandRank(oc.fillStates[c.CustomerID]) { oc.emitFill(c.CustomerID, off, newBand) } oc.fillStates[c.CustomerID] = newBand // STALENESS (binary, warn-severity) newStale := "ok" if oc.isStale(c.CustomerID, off) { newStale = "stale" } if newStale == "stale" && oc.staleStates[c.CustomerID] != "stale" { oc.emitStale(c.CustomerID, off) } // Part-7: make the anchored never-ran evaluation VISIBLE once (first observation of the // shape), so a live newborn tier's deferral is provable from the log without spamming // every sweep. if off.LastRun == "" && newStale == "ok" && off.Enabled && off.EscrowState == "escrowed" { if _, known := oc.staleStates[c.CustomerID]; !known { oc.logger.Printf("[INFO] Offsite staleness: %s never-ran within the anchored threshold (anchor %s) — newborn tier, not stale", c.CustomerID, oc.neverRanAnchor(c.CustomerID).UTC().Format(time.RFC3339)) } } oc.staleStates[c.CustomerID] = newStale } for k := range oc.fillStates { if !seen[k] { delete(oc.fillStates, k) } } for k := range oc.staleStates { if !seen[k] { delete(oc.staleStates, k) } } } // GetFillState / GetStaleState expose current states for tests. func (oc *OffsiteChecker) GetFillState(customerID string) string { oc.mu.Lock() defer oc.mu.Unlock() if s := oc.fillStates[customerID]; s != "" { return s } return "unknown" } func (oc *OffsiteChecker) GetStaleState(customerID string) string { oc.mu.Lock() defer oc.mu.Unlock() if s := oc.staleStates[customerID]; s != "" { return s } return "unknown" } func (oc *OffsiteChecker) emitFill(customerID string, off *offsiteReport, band string) { usedGB := off.RepoSizeBytes >> 30 pct := float64(off.RepoSizeBytes) * 100 / float64(int64(off.QuotaGB)<<30) var eventType, severity, message string switch band { case bandCritical: eventType, severity = "offsite_fill_critical", "critical" message = fmt.Sprintf("Customer %s: offsite backup at %.0f%% of its %d GB quota (%d GB used) — at 100%% new offsite runs are refused; consider the freeze lever or a bigger quota", customerID, pct, off.QuotaGB, usedGB) case bandWarning: eventType, severity = "offsite_fill_warning", "warning" message = fmt.Sprintf("Customer %s: offsite backup at %.0f%% of its %d GB quota (%d GB used)", customerID, pct, off.QuotaGB, usedGB) default: return } details, _ := json.Marshal(map[string]any{ "customer_id": customerID, "quota_gb": off.QuotaGB, "repo_size_bytes": off.RepoSizeBytes, "percent": pct, }) oc.logger.Printf("[INFO] Offsite fill: %s %.0f%% (%s)", customerID, pct, eventType) if _, err := oc.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil { oc.logger.Printf("[WARN] Failed to save offsite fill event for %s: %v", customerID, err) return } if oc.onEvent != nil { oc.onEvent(customerID, eventType, severity, message, string(details), "hub") } } // staleAge describes WHY the tier is stale, in the terms the verdict actually used. // // R-100: this must not say "last run 8h ago" while alarming, which is what it did when the verdict // moved to the success anchor — a tier that RUNS nightly and FAILS nightly would have alarmed with a // fresh-looking timestamp, and the operator would have read a true alarm as a false one. The two cases // are genuinely different diagnoses and the message now separates them: // - runs are not happening at all → check the schedule/controller // - runs happen and FAIL → check the error; this is the case backup_failed also reports func (oc *OffsiteChecker) staleAge(off *offsiteReport) (age, hint string) { parse := func(v string) (time.Duration, bool) { t, err := time.Parse(time.RFC3339, v) if err != nil { return 0, false } return oc.now().Sub(t).Round(time.Hour), true } if off.LastSuccess == "" { if d, ok := parse(off.LastRun); ok { return fmt.Sprintf("no run has EVER succeeded (last attempt %s ago, status %q)", d, off.LastStatus), " Runs are happening and failing — check the error, not the schedule" } return "never ran", " The offsite leg is silently not running; check the controller/schedule" } d, ok := parse(off.LastSuccess) if !ok { return "last successful run at an unparseable time", " Check the controller/schedule" } if la, ok := parse(off.LastRun); ok && off.LastRun != off.LastSuccess { return fmt.Sprintf("last SUCCESSFUL run %s ago (it last attempted %s ago, status %q)", d, la, off.LastStatus), " Runs are happening and failing — check the error, not the schedule" } return fmt.Sprintf("last successful run %s ago", d), " The offsite leg is silently not running; check the controller/schedule" } func (oc *OffsiteChecker) emitStale(customerID string, off *offsiteReport) { age, hint := oc.staleAge(off) message := fmt.Sprintf("Customer %s: offsite backup is STALE — enabled + escrowed but %s (threshold %s).%s", customerID, age, oc.staleAfter, hint) details, _ := json.Marshal(map[string]any{ "customer_id": customerID, "last_run": off.LastRun, "last_success": off.LastSuccess, "last_status": off.LastStatus, "stale_after": oc.staleAfter.String(), }) oc.logger.Printf("[INFO] Offsite staleness: %s (%s)", customerID, age) if _, err := oc.store.SaveEvent(customerID, "offsite_stale", "warning", message, string(details), "hub"); err != nil { oc.logger.Printf("[WARN] Failed to save offsite staleness event for %s: %v", customerID, err) return } if oc.onEvent != nil { oc.onEvent(customerID, "offsite_stale", "warning", message, string(details), "hub") } }