package monitor import ( "context" "encoding/json" "errors" "fmt" "log" "sync" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync" ) // usageReader is the ep0 datastore-usage seam (satisfied by *tenantsync.Client). Narrow so tests inject // a fake and the checker never depends on the full tenantsync surface. type usageReader interface { Usage(ctx context.Context) (tenantsync.BoxUsage, error) } // PBSDRBoxChecker (v0.65.0) watches the PBS DR datastore (felhom-offsite on ep0) fill — the sibling of // OffsiteBoxChecker over a DIFFERENT source: the ep0 read-only `usage` op (df on the datastore path), NOT // the Hetzner API. FILL ONLY (PBS DR uses namespaces, not soft quotas — no oversubscription concept). Same // shape: 15-min throttle, cached snapshot, escalation-only emit + silent recovery re-arm, degraded-on- // failure honesty. THREE snapshot states: // - "ok" → bands drive alerts. // - "unavailable" → the endpoint script predates the usage op (≤ v1.1.0) → ErrUsageUnsupported. An // EXPECTED pre-update condition: neutral, NO alert, logged once. The gauge shows n/a. // - "degraded" → exec failed/timed out → keep the last snapshot, NO band transition (missing ≠ 0%). // // Events carry the customer-less scope "pbsdr-box" → operator channel ONLY (processCustomer no-ops on it); // NO SaveEvent (no customer row to key it to). Deliberately NOT bolted onto OffsiteBoxChecker. type PBSDRBoxChecker struct { reader usageReader fillWarn float64 fillCrit float64 onEvent EventNotifyFunc logger *log.Logger now func() time.Time // injectable clock (tests) mu sync.Mutex snap PBSBoxSnapshot haveSnap bool lastFetch time.Time fillBand string // "" until first ok eval → born/persistent; only an "ok" fetch updates it unavailLogged bool // log the "unavailable" state ONCE, not per sweep } // pbsdrBoxScope is the customer-less cooldown/display key for PBS-DR box events. const pbsdrBoxScope = "pbsdr-box" // PBS-DR snapshot states. const ( PBSStateOK = "ok" PBSStateUnavailable = "unavailable" PBSStateDegraded = "degraded" ) const ( defaultPBSDRBoxFillWarnPercent = 80.0 defaultPBSDRBoxFillCritPercent = 90.0 ) // PBSBoxSnapshot is the cached PBS-DR datastore aggregate the web layer renders. Sizes in BYTES. type PBSBoxSnapshot struct { CapacityBytes int64 UsedBytes int64 FillPercent float64 FillBand string // "" | ok | warning | critical State string // ok | unavailable | degraded FetchedAt time.Time } // NewPBSDRBoxChecker builds the checker (defaults 80/90 on invalid thresholds). Nothing seeded → the // first ok Check that finds a breach emits (born/persistent; the dispatcher's 1 h cooldown dedups a restart). func NewPBSDRBoxChecker(reader usageReader, fillWarn, fillCrit float64, onEvent EventNotifyFunc, logger *log.Logger) *PBSDRBoxChecker { if fillWarn <= 0 || fillWarn >= 100 { fillWarn = defaultPBSDRBoxFillWarnPercent } if fillCrit <= 0 || fillCrit > 100 || fillCrit <= fillWarn { fillCrit = defaultPBSDRBoxFillCritPercent } if fillCrit <= fillWarn { fillWarn, fillCrit = defaultPBSDRBoxFillWarnPercent, defaultPBSDRBoxFillCritPercent } c := &PBSDRBoxChecker{ reader: reader, fillWarn: fillWarn, fillCrit: fillCrit, onEvent: onEvent, logger: logger, now: time.Now, } logger.Printf("[INFO] PBS-DR box checker initialized: fill warn=%.0f%% crit=%.0f%%, refresh %s", fillWarn, fillCrit, boxFetchInterval) return c } // Check runs on the 60 s sweep; fetches (throttled) and emits on each fill escalation. func (c *PBSDRBoxChecker) Check() { c.mu.Lock() defer c.mu.Unlock() if c.haveSnap && c.now().Sub(c.lastFetch) < boxFetchInterval { return // throttle — serve the cache } c.lastFetch = c.now() // updated even on failure → retried once per window, not per sweep usage, err := c.reader.Usage(context.Background()) if err != nil { if errors.Is(err, tenantsync.ErrUsageUnsupported) { // EXPECTED pre-update condition — a distinct "unavailable" state, NOT degraded, NO alert. c.snap = PBSBoxSnapshot{State: PBSStateUnavailable, FetchedAt: c.now()} c.haveSnap = true if !c.unavailLogged { c.logger.Printf("[INFO] PBS-DR box: usage op unavailable — endpoint tenantsync update (v1.2.0) pending; gauge shows n/a until then") c.unavailLogged = true } return // fillBand untouched — an expected data gap never re-arms/transitions } c.logger.Printf("[WARN] PBS-DR box: usage read failed (keeping last snapshot): %v", err) if c.haveSnap { c.snap.State = PBSStateDegraded // serve last-known, visibly stale; NO band transition } return } c.unavailLogged = false // recovered from unavailable snap := PBSBoxSnapshot{CapacityBytes: usage.Total, UsedBytes: usage.Used, State: PBSStateOK, FetchedAt: c.now()} if usage.Total > 0 { snap.FillPercent = float64(usage.Used) * 100 / float64(usage.Total) snap.FillBand = bandForPercent(snap.FillPercent, c.fillWarn, c.fillCrit) } else { snap.State = PBSStateDegraded // zero total — guard the division } c.snap = snap c.haveSnap = true if snap.State != PBSStateOK { return } c.logger.Printf("[INFO] PBS-DR box refreshed: %.1f%% full (%s of %s)", snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes)) // FILL band (escalation-only; recovery re-arms because bandOK has rank 0). if bandRank(snap.FillBand) > bandRank(c.fillBand) { c.emitFill(snap, snap.FillBand) } c.fillBand = snap.FillBand } // Snapshot returns the cached aggregate + whether one exists (mutex copy-out). The web layer's only // source — it NEVER polls ep0 in the request path. func (c *PBSDRBoxChecker) Snapshot() (PBSBoxSnapshot, bool) { c.mu.Lock() defer c.mu.Unlock() return c.snap, c.haveSnap } // FillState exposes the current band (tests). func (c *PBSDRBoxChecker) FillState() string { c.mu.Lock() defer c.mu.Unlock() return orUnknown(c.fillBand) } func (c *PBSDRBoxChecker) emitFill(snap PBSBoxSnapshot, band string) { var severity, message string switch band { case bandCritical: severity = "critical" message = fmt.Sprintf("PBS DR datastore %.0f%% full (%s of %s) — approaching capacity; the offsite DR tier's retention is at risk; free space or grow the datastore", snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes)) case bandWarning: severity = "warning" message = fmt.Sprintf("PBS DR datastore %.0f%% full (%s of %s) — the offsite DR datastore is filling", snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes)) default: return } details, _ := json.Marshal(map[string]any{ "scope": pbsdrBoxScope, "capacity_bytes": snap.CapacityBytes, "used_bytes": snap.UsedBytes, "fill_percent": snap.FillPercent, "warn_percent": c.fillWarn, "crit_percent": c.fillCrit, }) c.logger.Printf("[INFO] PBS-DR box fill: %.0f%% (%s)", snap.FillPercent, band) // NO SaveEvent — customer-less scope; straight to the operator dispatcher. if c.onEvent != nil { c.onEvent(pbsdrBoxScope, "pbsdr_box_fill", severity, message, string(details), "hub") } }