package monitor import ( "context" "encoding/json" "fmt" "log" "sync" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi" "gitea.dooplex.hu/admin/felhom-hub/internal/offsite" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // OffsiteBoxChecker (v0.64.0, R-5) watches the SHARED POOL BOX as a whole — the aggregate the // per-customer OffsiteChecker cannot see. Two independent box-level signals, one checker, mirroring the // OffsiteChecker/StorageFillChecker shape (escalation-only emit, recovery re-arms silently, born/persistent // via an unseeded band). It is DELIBERATELY separate from OffsiteChecker — a different data source (the // Hetzner unified API, not the controller report) and a customer-less scope. // // - FILL: box used vs box capacity (from the box TYPE's size, never stats.size) at warn 80% / crit 90%. // - OVERSUBSCRIPTION: Σ(shared+enabled customers' soft quotas) vs capacity, at a warn ratio (2.0×). The // number that says how far the sold promises exceed the disk. Independent of fill (both can fire). // // Fetch is throttled (boxFetchInterval); between refreshes Check() serves the cached snapshot. A failed // fetch keeps the last snapshot (marked Degraded) and never drives a band transition — absence of data is // NEVER treated as 0%. Events carry the customer-less scope "pool-box": the dispatcher uses it as a // cooldown key + display string only, and processCustomer no-ops on it (no prefs → no customer email; // verified against dispatcher.go), so a box event reaches ONLY the operator channel. NO SaveEvent — the // scope has no customer row to key it to. type OffsiteBoxChecker struct { api hetznerapi.CloudAPI boxID int64 store *store.Store fillWarn float64 fillCrit float64 oversubWarn float64 onEvent EventNotifyFunc logger *log.Logger now func() time.Time // injectable clock (tests) mu sync.Mutex snap BoxSnapshot haveSnap bool lastFetch time.Time fillBand string // "" until first eval → born/persistent (a born-breach emits on the first sweep) oversubBand string } // offsiteBoxScope is the customer-less cooldown/display key for pool-box events. const offsiteBoxScope = "pool-box" const ( defaultOffsiteBoxFillWarnPercent = 80.0 defaultOffsiteBoxFillCritPercent = 90.0 defaultOffsiteOversubWarnRatio = 2.0 // boxFetchInterval throttles the Hetzner read: Check() runs on the 60 s sweep but fetches at most // once per this window (≈4 API reads/hour, not ≈60). A manual page load never forces a fetch. boxFetchInterval = 15 * time.Minute ) // BoxSnapshot is the cached pool-box aggregate the web layer renders. All sizes are BYTES. FillBand / // OversubBand carry the checker's own band verdict so the UI colors bars EXACTLY as the alerts fire (no // threshold re-derivation in the web layer). type BoxSnapshot struct { CapacityBytes int64 UsedBytes int64 DataBytes int64 SnapshotBytes int64 FillPercent float64 SumQuotaGB int64 Ratio float64 // Σ(shared quota bytes) / capacity FillBand string // "" | ok | warning | critical OversubBand string // "" | ok | warning FetchedAt time.Time BoxType string Degraded bool // last fetch failed OR capacity==0 → render stale; no alert transitions } // NewOffsiteBoxChecker builds the checker. Invalid thresholds fall back to the documented defaults // (warn 80% / crit 90% / oversub 2.0×). Nothing is seeded → the first Check that finds a breach emits // (born/persistent; the dispatcher's 1 h cooldown dedups a hub restart). func NewOffsiteBoxChecker(api hetznerapi.CloudAPI, boxID int64, s *store.Store, fillWarn, fillCrit, oversubWarn float64, onEvent EventNotifyFunc, logger *log.Logger) *OffsiteBoxChecker { if fillWarn <= 0 || fillWarn >= 100 { fillWarn = defaultOffsiteBoxFillWarnPercent } if fillCrit <= 0 || fillCrit > 100 || fillCrit <= fillWarn { fillCrit = defaultOffsiteBoxFillCritPercent } if fillCrit <= fillWarn { fillWarn, fillCrit = defaultOffsiteBoxFillWarnPercent, defaultOffsiteBoxFillCritPercent } if oversubWarn <= 0 { oversubWarn = defaultOffsiteOversubWarnRatio } c := &OffsiteBoxChecker{ api: api, boxID: boxID, store: s, fillWarn: fillWarn, fillCrit: fillCrit, oversubWarn: oversubWarn, onEvent: onEvent, logger: logger, now: time.Now, } logger.Printf("[INFO] Offsite pool-box checker initialized: box=%d fill warn=%.0f%% crit=%.0f%%, oversub warn=%.2fx, refresh %s", boxID, fillWarn, fillCrit, oversubWarn, boxFetchInterval) return c } // Check runs on the 60 s sweep. It fetches (throttled), recomputes the snapshot, and emits on each // band escalation. Recovery de-escalation re-arms silently. Degraded data drives no transition. func (c *OffsiteBoxChecker) Check() { c.mu.Lock() defer c.mu.Unlock() // Fetch throttle: between refreshes serve the cache — bands only move on a fresh fetch. if c.haveSnap && c.now().Sub(c.lastFetch) < boxFetchInterval { return } c.lastFetch = c.now() // updated even on failure → a failing API is retried once per window, not per sweep box, err := c.api.GetStorageBox(context.Background(), c.boxID) if err != nil { c.logger.Printf("[WARN] Offsite pool-box: refresh failed (keeping last snapshot): %v", err) if c.haveSnap { c.snap.Degraded = true // serve the last-known, visibly stale; NO band transition (D) } return } sumQuotaGB, qerr := c.sumSharedQuotaGB() if qerr != nil { c.logger.Printf("[WARN] Offsite pool-box: quota sum failed (ratio omitted this cycle): %v", qerr) sumQuotaGB = 0 // a wrong sum is worse than a nominal 0.00× — never emit oversub off bad data } capacity := box.StorageBoxType.Size used := box.Stats.Size snap := BoxSnapshot{ CapacityBytes: capacity, UsedBytes: used, DataBytes: box.Stats.SizeData, SnapshotBytes: box.Stats.SizeSnapshots, SumQuotaGB: sumQuotaGB, FetchedAt: c.now(), BoxType: box.StorageBoxType.Name, } if capacity > 0 { snap.FillPercent = float64(used) * 100 / float64(capacity) snap.Ratio = float64(sumQuotaGB<<30) / float64(capacity) snap.FillBand = bandForPercent(snap.FillPercent, c.fillWarn, c.fillCrit) snap.OversubBand = bandOK if snap.Ratio >= c.oversubWarn { snap.OversubBand = bandWarning // ratio is single-threshold — no "critical" } } else { snap.Degraded = true // zero capacity (initializing box / probe surprise) — guard the division } c.snap = snap c.haveSnap = true if snap.Degraded { c.logger.Printf("[INFO] Offsite pool-box refreshed: DEGRADED (capacity unavailable) — box %d", c.boxID) return // no band transitions on degraded data } // Periodic operator visibility of the pool trend (one line per 15-min refresh; keys, no secrets). c.logger.Printf("[INFO] Offsite pool-box refreshed: %.1f%% full (%s of %s), Σ shared quota %d GB, oversub %.2fx", snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes), snap.SumQuotaGB, snap.Ratio) // FILL band (escalation-only; recovery re-arms because bandOK has rank 0). if bandRank(snap.FillBand) > bandRank(c.fillBand) { c.emitFill(snap, snap.FillBand) } c.fillBand = snap.FillBand // OVERSUBSCRIPTION band — independent signal (both can fire in one sweep; neither masks the other). if bandRank(snap.OversubBand) > bandRank(c.oversubBand) { c.emitOversub(snap) } c.oversubBand = snap.OversubBand } // Snapshot returns the current cached aggregate + whether one exists (mutex copy-out). The web layer's // only source — it NEVER fetches from Hetzner in the request path. func (c *OffsiteBoxChecker) Snapshot() (BoxSnapshot, bool) { c.mu.Lock() defer c.mu.Unlock() return c.snap, c.haveSnap } // FillState / OversubState expose the current bands (tests). func (c *OffsiteBoxChecker) FillState() string { c.mu.Lock(); defer c.mu.Unlock(); return orUnknown(c.fillBand) } func (c *OffsiteBoxChecker) OversubState() string { c.mu.Lock(); defer c.mu.Unlock(); return orUnknown(c.oversubBand) } func orUnknown(b string) string { if b == "" { return "unknown" } return b } // sumSharedQuotaGB sums the soft quotas of every ENABLED, SHARED customer from the authoritative // ConfigJSON descriptor (never the report echo). Dedicated + disabled customers are excluded. func (c *OffsiteBoxChecker) sumSharedQuotaGB() (int64, error) { cfgs, err := c.store.ListCustomerConfigs() if err != nil { return 0, err } var sum int64 for _, cfg := range cfgs { d, derr := offsite.ReadDescriptor(cfg.ConfigJSON) if derr != nil || d == nil { continue } if d.Enabled && d.Type == "shared" && d.QuotaGB > 0 { sum += int64(d.QuotaGB) } } return sum, nil } func (c *OffsiteBoxChecker) emitFill(snap BoxSnapshot, band string) { var severity, message string switch band { case bandCritical: severity = "critical" message = fmt.Sprintf("Offsite pool box %.0f%% full (%s of %s) — approaching capacity; add capacity or reduce retention before customer offsite runs start failing", snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes)) case bandWarning: severity = "warning" message = fmt.Sprintf("Offsite pool box %.0f%% full (%s of %s) — the shared pool is filling", snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes)) default: return } details, _ := json.Marshal(map[string]any{ "scope": offsiteBoxScope, "capacity_bytes": snap.CapacityBytes, "used_bytes": snap.UsedBytes, "fill_percent": snap.FillPercent, "warn_percent": c.fillWarn, "crit_percent": c.fillCrit, }) c.logger.Printf("[INFO] Offsite pool-box fill: %.0f%% (%s)", snap.FillPercent, band) // NO SaveEvent — the scope is customer-less; emit straight to the operator dispatcher. if c.onEvent != nil { c.onEvent(offsiteBoxScope, "offsite_box_fill", severity, message, string(details), "hub") } } func (c *OffsiteBoxChecker) emitOversub(snap BoxSnapshot) { message := fmt.Sprintf("Offsite pool oversubscription %.2fx — Σ(shared soft quotas) %d GB against %s capacity exceeds the %.2fx threshold; a full-usage scenario would overrun the pool", snap.Ratio, snap.SumQuotaGB, fmtSize(snap.CapacityBytes), c.oversubWarn) details, _ := json.Marshal(map[string]any{ "scope": offsiteBoxScope, "sum_quota_gb": snap.SumQuotaGB, "capacity_bytes": snap.CapacityBytes, "ratio": snap.Ratio, "oversub_warn_ratio": c.oversubWarn, }) c.logger.Printf("[INFO] Offsite pool-box oversub: %.2fx (Σquota %d GB)", snap.Ratio, snap.SumQuotaGB) if c.onEvent != nil { c.onEvent(offsiteBoxScope, "offsite_box_oversub", "warning", message, string(details), "hub") } } // fmtSize renders bytes as GB (one decimal) below 1 TB, TB above — for the operator email/log only. func fmtSize(b int64) string { const gb = 1 << 30 if b >= 1<<40 { return fmt.Sprintf("%.2f TB", float64(b)/float64(int64(1)<<40)) } return fmt.Sprintf("%.1f GB", float64(b)/float64(gb)) }