261 lines
11 KiB
Go
261 lines
11 KiB
Go
package monitor
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"log"
|
||
"sync"
|
||
"time"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/offsite"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||
)
|
||
|
||
// OffsiteBoxChecker (v0.64.0, R-5) watches the SHARED POOL BOX as a whole — the aggregate the
|
||
// per-customer OffsiteChecker cannot see. Two independent box-level signals, one checker, mirroring the
|
||
// OffsiteChecker/StorageFillChecker shape (escalation-only emit, recovery re-arms silently, born/persistent
|
||
// via an unseeded band). It is DELIBERATELY separate from OffsiteChecker — a different data source (the
|
||
// Hetzner unified API, not the controller report) and a customer-less scope.
|
||
//
|
||
// - FILL: box used vs box capacity (from the box TYPE's size, never stats.size) at warn 80% / crit 90%.
|
||
// - OVERSUBSCRIPTION: Σ(shared+enabled customers' soft quotas) vs capacity, at a warn ratio (2.0×). The
|
||
// number that says how far the sold promises exceed the disk. Independent of fill (both can fire).
|
||
//
|
||
// Fetch is throttled (boxFetchInterval); between refreshes Check() serves the cached snapshot. A failed
|
||
// fetch keeps the last snapshot (marked Degraded) and never drives a band transition — absence of data is
|
||
// NEVER treated as 0%. Events carry the customer-less scope "pool-box": the dispatcher uses it as a
|
||
// cooldown key + display string only, and processCustomer no-ops on it (no prefs → no customer email;
|
||
// verified against dispatcher.go), so a box event reaches ONLY the operator channel. NO SaveEvent — the
|
||
// scope has no customer row to key it to.
|
||
type OffsiteBoxChecker struct {
|
||
api hetznerapi.CloudAPI
|
||
boxID int64
|
||
store *store.Store
|
||
fillWarn float64
|
||
fillCrit float64
|
||
oversubWarn float64
|
||
onEvent EventNotifyFunc
|
||
logger *log.Logger
|
||
now func() time.Time // injectable clock (tests)
|
||
|
||
mu sync.Mutex
|
||
snap BoxSnapshot
|
||
haveSnap bool
|
||
lastFetch time.Time
|
||
fillBand string // "" until first eval → born/persistent (a born-breach emits on the first sweep)
|
||
oversubBand string
|
||
}
|
||
|
||
// offsiteBoxScope is the customer-less cooldown/display key for pool-box events.
|
||
const offsiteBoxScope = "pool-box"
|
||
|
||
const (
|
||
defaultOffsiteBoxFillWarnPercent = 80.0
|
||
defaultOffsiteBoxFillCritPercent = 90.0
|
||
defaultOffsiteOversubWarnRatio = 2.0
|
||
// boxFetchInterval throttles the Hetzner read: Check() runs on the 60 s sweep but fetches at most
|
||
// once per this window (≈4 API reads/hour, not ≈60). A manual page load never forces a fetch.
|
||
boxFetchInterval = 15 * time.Minute
|
||
)
|
||
|
||
// BoxSnapshot is the cached pool-box aggregate the web layer renders. All sizes are BYTES. FillBand /
|
||
// OversubBand carry the checker's own band verdict so the UI colors bars EXACTLY as the alerts fire (no
|
||
// threshold re-derivation in the web layer).
|
||
type BoxSnapshot struct {
|
||
CapacityBytes int64
|
||
UsedBytes int64
|
||
DataBytes int64
|
||
SnapshotBytes int64
|
||
FillPercent float64
|
||
SumQuotaGB int64
|
||
Ratio float64 // Σ(shared quota bytes) / capacity
|
||
FillBand string // "" | ok | warning | critical
|
||
OversubBand string // "" | ok | warning
|
||
FetchedAt time.Time
|
||
BoxType string
|
||
Degraded bool // last fetch failed OR capacity==0 → render stale; no alert transitions
|
||
}
|
||
|
||
// NewOffsiteBoxChecker builds the checker. Invalid thresholds fall back to the documented defaults
|
||
// (warn 80% / crit 90% / oversub 2.0×). Nothing is seeded → the first Check that finds a breach emits
|
||
// (born/persistent; the dispatcher's 1 h cooldown dedups a hub restart).
|
||
func NewOffsiteBoxChecker(api hetznerapi.CloudAPI, boxID int64, s *store.Store, fillWarn, fillCrit, oversubWarn float64, onEvent EventNotifyFunc, logger *log.Logger) *OffsiteBoxChecker {
|
||
if fillWarn <= 0 || fillWarn >= 100 {
|
||
fillWarn = defaultOffsiteBoxFillWarnPercent
|
||
}
|
||
if fillCrit <= 0 || fillCrit > 100 || fillCrit <= fillWarn {
|
||
fillCrit = defaultOffsiteBoxFillCritPercent
|
||
}
|
||
if fillCrit <= fillWarn {
|
||
fillWarn, fillCrit = defaultOffsiteBoxFillWarnPercent, defaultOffsiteBoxFillCritPercent
|
||
}
|
||
if oversubWarn <= 0 {
|
||
oversubWarn = defaultOffsiteOversubWarnRatio
|
||
}
|
||
c := &OffsiteBoxChecker{
|
||
api: api, boxID: boxID, store: s,
|
||
fillWarn: fillWarn, fillCrit: fillCrit, oversubWarn: oversubWarn,
|
||
onEvent: onEvent, logger: logger, now: time.Now,
|
||
}
|
||
logger.Printf("[INFO] Offsite pool-box checker initialized: box=%d fill warn=%.0f%% crit=%.0f%%, oversub warn=%.2fx, refresh %s",
|
||
boxID, fillWarn, fillCrit, oversubWarn, boxFetchInterval)
|
||
return c
|
||
}
|
||
|
||
// Check runs on the 60 s sweep. It fetches (throttled), recomputes the snapshot, and emits on each
|
||
// band escalation. Recovery de-escalation re-arms silently. Degraded data drives no transition.
|
||
func (c *OffsiteBoxChecker) Check() {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
|
||
// Fetch throttle: between refreshes serve the cache — bands only move on a fresh fetch.
|
||
if c.haveSnap && c.now().Sub(c.lastFetch) < boxFetchInterval {
|
||
return
|
||
}
|
||
c.lastFetch = c.now() // updated even on failure → a failing API is retried once per window, not per sweep
|
||
|
||
box, err := c.api.GetStorageBox(context.Background(), c.boxID)
|
||
if err != nil {
|
||
c.logger.Printf("[WARN] Offsite pool-box: refresh failed (keeping last snapshot): %v", err)
|
||
if c.haveSnap {
|
||
c.snap.Degraded = true // serve the last-known, visibly stale; NO band transition (D)
|
||
}
|
||
return
|
||
}
|
||
|
||
sumQuotaGB, qerr := c.sumSharedQuotaGB()
|
||
if qerr != nil {
|
||
c.logger.Printf("[WARN] Offsite pool-box: quota sum failed (ratio omitted this cycle): %v", qerr)
|
||
sumQuotaGB = 0 // a wrong sum is worse than a nominal 0.00× — never emit oversub off bad data
|
||
}
|
||
|
||
capacity := box.StorageBoxType.Size
|
||
used := box.Stats.Size
|
||
snap := BoxSnapshot{
|
||
CapacityBytes: capacity, UsedBytes: used,
|
||
DataBytes: box.Stats.SizeData, SnapshotBytes: box.Stats.SizeSnapshots,
|
||
SumQuotaGB: sumQuotaGB, FetchedAt: c.now(), BoxType: box.StorageBoxType.Name,
|
||
}
|
||
if capacity > 0 {
|
||
snap.FillPercent = float64(used) * 100 / float64(capacity)
|
||
snap.Ratio = float64(sumQuotaGB<<30) / float64(capacity)
|
||
snap.FillBand = bandForPercent(snap.FillPercent, c.fillWarn, c.fillCrit)
|
||
snap.OversubBand = bandOK
|
||
if snap.Ratio >= c.oversubWarn {
|
||
snap.OversubBand = bandWarning // ratio is single-threshold — no "critical"
|
||
}
|
||
} else {
|
||
snap.Degraded = true // zero capacity (initializing box / probe surprise) — guard the division
|
||
}
|
||
c.snap = snap
|
||
c.haveSnap = true
|
||
|
||
if snap.Degraded {
|
||
c.logger.Printf("[INFO] Offsite pool-box refreshed: DEGRADED (capacity unavailable) — box %d", c.boxID)
|
||
return // no band transitions on degraded data
|
||
}
|
||
// Periodic operator visibility of the pool trend (one line per 15-min refresh; keys, no secrets).
|
||
c.logger.Printf("[INFO] Offsite pool-box refreshed: %.1f%% full (%s of %s), Σ shared quota %d GB, oversub %.2fx",
|
||
snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes), snap.SumQuotaGB, snap.Ratio)
|
||
|
||
// FILL band (escalation-only; recovery re-arms because bandOK has rank 0).
|
||
if bandRank(snap.FillBand) > bandRank(c.fillBand) {
|
||
c.emitFill(snap, snap.FillBand)
|
||
}
|
||
c.fillBand = snap.FillBand
|
||
|
||
// OVERSUBSCRIPTION band — independent signal (both can fire in one sweep; neither masks the other).
|
||
if bandRank(snap.OversubBand) > bandRank(c.oversubBand) {
|
||
c.emitOversub(snap)
|
||
}
|
||
c.oversubBand = snap.OversubBand
|
||
}
|
||
|
||
// Snapshot returns the current cached aggregate + whether one exists (mutex copy-out). The web layer's
|
||
// only source — it NEVER fetches from Hetzner in the request path.
|
||
func (c *OffsiteBoxChecker) Snapshot() (BoxSnapshot, bool) {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
return c.snap, c.haveSnap
|
||
}
|
||
|
||
// FillState / OversubState expose the current bands (tests).
|
||
func (c *OffsiteBoxChecker) FillState() string { c.mu.Lock(); defer c.mu.Unlock(); return orUnknown(c.fillBand) }
|
||
func (c *OffsiteBoxChecker) OversubState() string { c.mu.Lock(); defer c.mu.Unlock(); return orUnknown(c.oversubBand) }
|
||
|
||
func orUnknown(b string) string {
|
||
if b == "" {
|
||
return "unknown"
|
||
}
|
||
return b
|
||
}
|
||
|
||
// sumSharedQuotaGB sums the soft quotas of every ENABLED, SHARED customer from the authoritative
|
||
// ConfigJSON descriptor (never the report echo). Dedicated + disabled customers are excluded.
|
||
func (c *OffsiteBoxChecker) sumSharedQuotaGB() (int64, error) {
|
||
cfgs, err := c.store.ListCustomerConfigs()
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
var sum int64
|
||
for _, cfg := range cfgs {
|
||
d, derr := offsite.ReadDescriptor(cfg.ConfigJSON)
|
||
if derr != nil || d == nil {
|
||
continue
|
||
}
|
||
if d.Enabled && d.Type == "shared" && d.QuotaGB > 0 {
|
||
sum += int64(d.QuotaGB)
|
||
}
|
||
}
|
||
return sum, nil
|
||
}
|
||
|
||
func (c *OffsiteBoxChecker) emitFill(snap BoxSnapshot, band string) {
|
||
var severity, message string
|
||
switch band {
|
||
case bandCritical:
|
||
severity = "critical"
|
||
message = fmt.Sprintf("Offsite pool box %.0f%% full (%s of %s) — approaching capacity; add capacity or reduce retention before customer offsite runs start failing",
|
||
snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes))
|
||
case bandWarning:
|
||
severity = "warning"
|
||
message = fmt.Sprintf("Offsite pool box %.0f%% full (%s of %s) — the shared pool is filling",
|
||
snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes))
|
||
default:
|
||
return
|
||
}
|
||
details, _ := json.Marshal(map[string]any{
|
||
"scope": offsiteBoxScope, "capacity_bytes": snap.CapacityBytes, "used_bytes": snap.UsedBytes,
|
||
"fill_percent": snap.FillPercent, "warn_percent": c.fillWarn, "crit_percent": c.fillCrit,
|
||
})
|
||
c.logger.Printf("[INFO] Offsite pool-box fill: %.0f%% (%s)", snap.FillPercent, band)
|
||
// NO SaveEvent — the scope is customer-less; emit straight to the operator dispatcher.
|
||
if c.onEvent != nil {
|
||
c.onEvent(offsiteBoxScope, "offsite_box_fill", severity, message, string(details), "hub")
|
||
}
|
||
}
|
||
|
||
func (c *OffsiteBoxChecker) emitOversub(snap BoxSnapshot) {
|
||
message := fmt.Sprintf("Offsite pool oversubscription %.2fx — Σ(shared soft quotas) %d GB against %s capacity exceeds the %.2fx threshold; a full-usage scenario would overrun the pool",
|
||
snap.Ratio, snap.SumQuotaGB, fmtSize(snap.CapacityBytes), c.oversubWarn)
|
||
details, _ := json.Marshal(map[string]any{
|
||
"scope": offsiteBoxScope, "sum_quota_gb": snap.SumQuotaGB, "capacity_bytes": snap.CapacityBytes,
|
||
"ratio": snap.Ratio, "oversub_warn_ratio": c.oversubWarn,
|
||
})
|
||
c.logger.Printf("[INFO] Offsite pool-box oversub: %.2fx (Σquota %d GB)", snap.Ratio, snap.SumQuotaGB)
|
||
if c.onEvent != nil {
|
||
c.onEvent(offsiteBoxScope, "offsite_box_oversub", "warning", message, string(details), "hub")
|
||
}
|
||
}
|
||
|
||
// fmtSize renders bytes as GB (one decimal) below 1 TB, TB above — for the operator email/log only.
|
||
func fmtSize(b int64) string {
|
||
const gb = 1 << 30
|
||
if b >= 1<<40 {
|
||
return fmt.Sprintf("%.2f TB", float64(b)/float64(int64(1)<<40))
|
||
}
|
||
return fmt.Sprintf("%.1f GB", float64(b)/float64(gb))
|
||
}
|