package monitor import ( "encoding/json" "fmt" "log" "sync" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // OffsiteChecker (SLICE 4) watches each customer's offsite backup health from the controller report's // `offsite` status object. Two independent signals, one checker (sibling of StorageFillChecker — same // born/persistent, escalation-only emit, recovery re-arm shape; NOT bolted onto the disk checkers — // different data source, different remedy text): // // - FILL: repo_size_bytes vs the shared-model soft quota (quota_gb>0) at warn 90% / crit 95% — the // operator's early warning before the controller's own 100% run-refusal bites the customer. // - STALENESS: enabled + escrowed but no run in >48h (or never) — the silently-STUCK detector. A // RECENTLY-failing offsite is NOT stale (backup_failed already alerts it); staleness is the // complement: nothing is even trying. Pending/disabled targets are normal onboarding, never stale. // // Reports without an `offsite` object (pre-v0.109 controllers, offbox not enabled) are skipped nil-safe. type OffsiteChecker struct { store *store.Store logger *log.Logger onEvent EventNotifyFunc staleAfter time.Duration now func() time.Time // injectable clock (tests) mu sync.Mutex fillStates map[string]string // customerID → fill band staleStates map[string]string // customerID → "ok" | "stale" } const defaultOffsiteStaleAfter = 48 * time.Hour // offsiteReport mirrors the controller report's `offsite` object (v0.109.0). type offsiteReport struct { Enabled bool `json:"enabled"` EscrowState string `json:"escrow_state"` LastRun string `json:"last_run"` LastStatus string `json:"last_status"` SnapshotCount int `json:"snapshot_count"` RepoSizeBytes int64 `json:"repo_size_bytes"` QuotaGB int `json:"quota_gb"` } // NewOffsiteChecker builds the checker. Same seeding philosophy as StorageFillChecker: already-breached // customers are left UNSEEDED so their first Check emits (born/persistent); the dispatcher's cooldown // dedups a hub restart. func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNotifyFunc, logger *log.Logger) *OffsiteChecker { if staleAfter <= 0 { staleAfter = defaultOffsiteStaleAfter } oc := &OffsiteChecker{ store: s, logger: logger, onEvent: onEvent, staleAfter: staleAfter, now: time.Now, fillStates: make(map[string]string), staleStates: make(map[string]string), } customers, err := s.GetCustomers() if err != nil { logger.Printf("[WARN] Offsite checker: failed to seed states: %v", err) return oc } var seeded int for _, c := range customers { off := parseOffsite(c.ReportJSON) if off == nil || s.IsCustomerBlocked(c.CustomerID) { continue } if band := oc.fillBand(off); band == bandOK { oc.fillStates[c.CustomerID] = bandOK seeded++ } if !oc.isStale(c.CustomerID, off) { oc.staleStates[c.CustomerID] = "ok" } } logger.Printf("[INFO] Offsite checker initialized: fill warn=90%% crit=95%%, stale after %s, %d ok-seeded", staleAfter, seeded) return oc } func parseOffsite(reportJSON string) *offsiteReport { var r struct { Offsite *offsiteReport `json:"offsite"` } if json.Unmarshal([]byte(reportJSON), &r) != nil { return nil } return r.Offsite // nil when absent (old controller / offbox not enabled) — the caller skips } // fillBand maps the quota usage to a band. quota<=0 (dedicated/unset) never alerts. func (oc *OffsiteChecker) fillBand(off *offsiteReport) string { if off.QuotaGB <= 0 || off.RepoSizeBytes <= 0 { return bandOK } pct := float64(off.RepoSizeBytes) * 100 / float64(int64(off.QuotaGB)<<30) return bandForPercent(pct, 90, 95) } // isStale: enabled + ESCROWED (the only state where runs are expected) with no run in >staleAfter (or // never ran, ANCHORED — see below). Pending/disabled = normal onboarding, never stale. A // recent-but-failing run is NOT stale (backup_failed owns that signal). // // Part-7 (v0.73.0) — the never-ran branch no longer fires on sight. The 2026-07-23 cry-wolf: // demo-hp's tier was repaired and escrowed at 10:01Z and offsite_stale fired MINUTES later // (`last_run:"" … threshold 48h`), because "enabled + escrowed + never ran" had no time anchor. // Boundary: reaching this code at all means the latest report CARRIES the offsite object — i.e. // the v0.72.0 delivery state is `applied` (Check nil-skips everything else); pre-applied // never-ran shapes are offsite_delivery_stuck's alone — ONE STATE, ONE OWNER, never both. // Anchor: the newest of one_time_secrets.consumed_at (delivery completed) and the customer's // escrow-blob timestamp (host_escrow.updated_at/created_at — runs become POSSIBLE only at the // ceremony). The EXISTING staleAfter threshold, anchored there, IS the grace — no new knob. func (oc *OffsiteChecker) isStale(customerID string, off *offsiteReport) bool { if !off.Enabled || off.EscrowState != "escrowed" { return false } if off.LastRun == "" { anchor := oc.neverRanAnchor(customerID) if anchor.IsZero() { return true // legacy shape (no secret timestamps, no escrow row) — fail toward visibility, as before } return oc.now().Sub(anchor) > oc.staleAfter } t, err := time.Parse(time.RFC3339, off.LastRun) if err != nil { return true // unparseable = unknown-old — fail toward visibility } return oc.now().Sub(t) > oc.staleAfter } // neverRanAnchor returns the newest hub-held timestamp from which a never-ran-but-applied tier's // staleness may be counted (zero when the hub holds neither — the pre-v0.5x legacy shape). func (oc *OffsiteChecker) neverRanAnchor(customerID string) time.Time { var anchor time.Time if info, err := oc.store.GetOneTimeSecretInfo(customerID); err == nil && info != nil && info.ConsumedAt.After(anchor) { anchor = info.ConsumedAt } if t, err := oc.store.LatestEscrowTimeForCustomer(customerID); err == nil && t.After(anchor) { anchor = t } return anchor } // Check evaluates every customer's latest report. Escalation-only emits; recovery re-arms silently. func (oc *OffsiteChecker) Check() { customers, err := oc.store.GetCustomers() if err != nil { oc.logger.Printf("[WARN] Offsite check failed: %v", err) return } oc.mu.Lock() defer oc.mu.Unlock() seen := make(map[string]bool, len(customers)) for _, c := range customers { // GetCustomers can return the same customer twice when two reports tie on received_at // (second-resolution timestamps) — process each customer once per sweep. if seen[c.CustomerID] { continue } seen[c.CustomerID] = true off := parseOffsite(c.ReportJSON) if off == nil { delete(oc.fillStates, c.CustomerID) // vanished object (disabled / downgraded) → re-arm delete(oc.staleStates, c.CustomerID) continue } if oc.store.IsCustomerBlocked(c.CustomerID) { delete(oc.fillStates, c.CustomerID) delete(oc.staleStates, c.CustomerID) continue } // FILL (quota>0 only) newBand := oc.fillBand(off) if bandRank(newBand) > bandRank(oc.fillStates[c.CustomerID]) { oc.emitFill(c.CustomerID, off, newBand) } oc.fillStates[c.CustomerID] = newBand // STALENESS (binary, warn-severity) newStale := "ok" if oc.isStale(c.CustomerID, off) { newStale = "stale" } if newStale == "stale" && oc.staleStates[c.CustomerID] != "stale" { oc.emitStale(c.CustomerID, off) } // Part-7: make the anchored never-ran evaluation VISIBLE once (first observation of the // shape), so a live newborn tier's deferral is provable from the log without spamming // every sweep. if off.LastRun == "" && newStale == "ok" && off.Enabled && off.EscrowState == "escrowed" { if _, known := oc.staleStates[c.CustomerID]; !known { oc.logger.Printf("[INFO] Offsite staleness: %s never-ran within the anchored threshold (anchor %s) — newborn tier, not stale", c.CustomerID, oc.neverRanAnchor(c.CustomerID).UTC().Format(time.RFC3339)) } } oc.staleStates[c.CustomerID] = newStale } for k := range oc.fillStates { if !seen[k] { delete(oc.fillStates, k) } } for k := range oc.staleStates { if !seen[k] { delete(oc.staleStates, k) } } } // GetFillState / GetStaleState expose current states for tests. func (oc *OffsiteChecker) GetFillState(customerID string) string { oc.mu.Lock() defer oc.mu.Unlock() if s := oc.fillStates[customerID]; s != "" { return s } return "unknown" } func (oc *OffsiteChecker) GetStaleState(customerID string) string { oc.mu.Lock() defer oc.mu.Unlock() if s := oc.staleStates[customerID]; s != "" { return s } return "unknown" } func (oc *OffsiteChecker) emitFill(customerID string, off *offsiteReport, band string) { usedGB := off.RepoSizeBytes >> 30 pct := float64(off.RepoSizeBytes) * 100 / float64(int64(off.QuotaGB)<<30) var eventType, severity, message string switch band { case bandCritical: eventType, severity = "offsite_fill_critical", "critical" message = fmt.Sprintf("Customer %s: offsite backup at %.0f%% of its %d GB quota (%d GB used) — at 100%% new offsite runs are refused; consider the freeze lever or a bigger quota", customerID, pct, off.QuotaGB, usedGB) case bandWarning: eventType, severity = "offsite_fill_warning", "warning" message = fmt.Sprintf("Customer %s: offsite backup at %.0f%% of its %d GB quota (%d GB used)", customerID, pct, off.QuotaGB, usedGB) default: return } details, _ := json.Marshal(map[string]any{ "customer_id": customerID, "quota_gb": off.QuotaGB, "repo_size_bytes": off.RepoSizeBytes, "percent": pct, }) oc.logger.Printf("[INFO] Offsite fill: %s %.0f%% (%s)", customerID, pct, eventType) if _, err := oc.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil { oc.logger.Printf("[WARN] Failed to save offsite fill event for %s: %v", customerID, err) return } if oc.onEvent != nil { oc.onEvent(customerID, eventType, severity, message, string(details), "hub") } } func (oc *OffsiteChecker) emitStale(customerID string, off *offsiteReport) { age := "never ran" if off.LastRun != "" { if t, err := time.Parse(time.RFC3339, off.LastRun); err == nil { age = fmt.Sprintf("last run %s ago", oc.now().Sub(t).Round(time.Hour)) } } message := fmt.Sprintf("Customer %s: offsite backup is STALE — enabled + escrowed but %s (threshold %s). The offsite leg is silently not running; check the controller/schedule", customerID, age, oc.staleAfter) details, _ := json.Marshal(map[string]any{ "customer_id": customerID, "last_run": off.LastRun, "last_status": off.LastStatus, "stale_after": oc.staleAfter.String(), }) oc.logger.Printf("[INFO] Offsite staleness: %s (%s)", customerID, age) if _, err := oc.store.SaveEvent(customerID, "offsite_stale", "warning", message, string(details), "hub"); err != nil { oc.logger.Printf("[WARN] Failed to save offsite staleness event for %s: %v", customerID, err) return } if oc.onEvent != nil { oc.onEvent(customerID, "offsite_stale", "warning", message, string(details), "hub") } }