felhom.eu/hub/internal/monitor/pbsdr_box.go

package monitor

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync"
)

// usageReader is the ep0 datastore-usage seam (satisfied by *tenantsync.Client). Narrow so tests inject
// a fake and the checker never depends on the full tenantsync surface.
type usageReader interface {
	Usage(ctx context.Context) (tenantsync.BoxUsage, error)
}

// PBSDRBoxChecker (v0.65.0) watches the PBS DR datastore (felhom-offsite on ep0) fill — the sibling of
// OffsiteBoxChecker over a DIFFERENT source: the ep0 read-only `usage` op (df on the datastore path), NOT
// the Hetzner API. FILL ONLY (PBS DR uses namespaces, not soft quotas — no oversubscription concept). Same
// shape: 15-min throttle, cached snapshot, escalation-only emit + silent recovery re-arm, degraded-on-
// failure honesty. THREE snapshot states:
//   - "ok"          → bands drive alerts.
//   - "unavailable" → the endpoint script predates the usage op (≤ v1.1.0) → ErrUsageUnsupported. An
//                     EXPECTED pre-update condition: neutral, NO alert, logged once. The gauge shows n/a.
//   - "degraded"    → exec failed/timed out → keep the last snapshot, NO band transition (missing ≠ 0%).
//
// Events carry the customer-less scope "pbsdr-box" → operator channel ONLY (processCustomer no-ops on it);
// NO SaveEvent (no customer row to key it to). Deliberately NOT bolted onto OffsiteBoxChecker.
type PBSDRBoxChecker struct {
	reader   usageReader
	fillWarn float64
	fillCrit float64
	onEvent  EventNotifyFunc
	logger   *log.Logger
	now      func() time.Time // injectable clock (tests)

	mu            sync.Mutex
	snap          PBSBoxSnapshot
	haveSnap      bool
	lastFetch     time.Time
	fillBand      string // "" until first ok eval → born/persistent; only an "ok" fetch updates it
	unavailLogged bool   // log the "unavailable" state ONCE, not per sweep
}

// pbsdrBoxScope is the customer-less cooldown/display key for PBS-DR box events.
const pbsdrBoxScope = "pbsdr-box"

// PBS-DR snapshot states.
const (
	PBSStateOK          = "ok"
	PBSStateUnavailable = "unavailable"
	PBSStateDegraded    = "degraded"
)

const (
	defaultPBSDRBoxFillWarnPercent = 80.0
	defaultPBSDRBoxFillCritPercent = 90.0
)

// PBSBoxSnapshot is the cached PBS-DR datastore aggregate the web layer renders. Sizes in BYTES.
type PBSBoxSnapshot struct {
	CapacityBytes int64
	UsedBytes     int64
	FillPercent   float64
	FillBand      string // "" | ok | warning | critical
	State         string // ok | unavailable | degraded
	FetchedAt     time.Time
}

// NewPBSDRBoxChecker builds the checker (defaults 80/90 on invalid thresholds). Nothing seeded → the
// first ok Check that finds a breach emits (born/persistent; the dispatcher's 1 h cooldown dedups a restart).
func NewPBSDRBoxChecker(reader usageReader, fillWarn, fillCrit float64, onEvent EventNotifyFunc, logger *log.Logger) *PBSDRBoxChecker {
	if fillWarn <= 0 || fillWarn >= 100 {
		fillWarn = defaultPBSDRBoxFillWarnPercent
	}
	if fillCrit <= 0 || fillCrit > 100 || fillCrit <= fillWarn {
		fillCrit = defaultPBSDRBoxFillCritPercent
	}
	if fillCrit <= fillWarn {
		fillWarn, fillCrit = defaultPBSDRBoxFillWarnPercent, defaultPBSDRBoxFillCritPercent
	}
	c := &PBSDRBoxChecker{
		reader: reader, fillWarn: fillWarn, fillCrit: fillCrit,
		onEvent: onEvent, logger: logger, now: time.Now,
	}
	logger.Printf("[INFO] PBS-DR box checker initialized: fill warn=%.0f%% crit=%.0f%%, refresh %s", fillWarn, fillCrit, boxFetchInterval)
	return c
}

// Check runs on the 60 s sweep; fetches (throttled) and emits on each fill escalation.
func (c *PBSDRBoxChecker) Check() {
	c.mu.Lock()
	defer c.mu.Unlock()

	if c.haveSnap && c.now().Sub(c.lastFetch) < boxFetchInterval {
		return // throttle — serve the cache
	}
	c.lastFetch = c.now() // updated even on failure → retried once per window, not per sweep

	usage, err := c.reader.Usage(context.Background())
	if err != nil {
		if errors.Is(err, tenantsync.ErrUsageUnsupported) {
			// EXPECTED pre-update condition — a distinct "unavailable" state, NOT degraded, NO alert.
			c.snap = PBSBoxSnapshot{State: PBSStateUnavailable, FetchedAt: c.now()}
			c.haveSnap = true
			if !c.unavailLogged {
				c.logger.Printf("[INFO] PBS-DR box: usage op unavailable — endpoint tenantsync update (v1.2.0) pending; gauge shows n/a until then")
				c.unavailLogged = true
			}
			return // fillBand untouched — an expected data gap never re-arms/transitions
		}
		c.logger.Printf("[WARN] PBS-DR box: usage read failed (keeping last snapshot): %v", err)
		if c.haveSnap {
			c.snap.State = PBSStateDegraded // serve last-known, visibly stale; NO band transition
		}
		return
	}
	c.unavailLogged = false // recovered from unavailable

	snap := PBSBoxSnapshot{CapacityBytes: usage.Total, UsedBytes: usage.Used, State: PBSStateOK, FetchedAt: c.now()}
	if usage.Total > 0 {
		snap.FillPercent = float64(usage.Used) * 100 / float64(usage.Total)
		snap.FillBand = bandForPercent(snap.FillPercent, c.fillWarn, c.fillCrit)
	} else {
		snap.State = PBSStateDegraded // zero total — guard the division
	}
	c.snap = snap
	c.haveSnap = true

	if snap.State != PBSStateOK {
		return
	}
	c.logger.Printf("[INFO] PBS-DR box refreshed: %.1f%% full (%s of %s)", snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes))

	// FILL band (escalation-only; recovery re-arms because bandOK has rank 0).
	if bandRank(snap.FillBand) > bandRank(c.fillBand) {
		c.emitFill(snap, snap.FillBand)
	}
	c.fillBand = snap.FillBand
}

// Snapshot returns the cached aggregate + whether one exists (mutex copy-out). The web layer's only
// source — it NEVER polls ep0 in the request path.
func (c *PBSDRBoxChecker) Snapshot() (PBSBoxSnapshot, bool) {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.snap, c.haveSnap
}

// FillState exposes the current band (tests).
func (c *PBSDRBoxChecker) FillState() string {
	c.mu.Lock()
	defer c.mu.Unlock()
	return orUnknown(c.fillBand)
}

func (c *PBSDRBoxChecker) emitFill(snap PBSBoxSnapshot, band string) {
	var severity, message string
	switch band {
	case bandCritical:
		severity = "critical"
		message = fmt.Sprintf("PBS DR datastore %.0f%% full (%s of %s) — approaching capacity; the offsite DR tier's retention is at risk; free space or grow the datastore",
			snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes))
	case bandWarning:
		severity = "warning"
		message = fmt.Sprintf("PBS DR datastore %.0f%% full (%s of %s) — the offsite DR datastore is filling",
			snap.FillPercent, fmtSize(snap.UsedBytes), fmtSize(snap.CapacityBytes))
	default:
		return
	}
	details, _ := json.Marshal(map[string]any{
		"scope": pbsdrBoxScope, "capacity_bytes": snap.CapacityBytes, "used_bytes": snap.UsedBytes,
		"fill_percent": snap.FillPercent, "warn_percent": c.fillWarn, "crit_percent": c.fillCrit,
	})
	c.logger.Printf("[INFO] PBS-DR box fill: %.0f%% (%s)", snap.FillPercent, band)
	// NO SaveEvent — customer-less scope; straight to the operator dispatcher.
	if c.onEvent != nil {
		c.onEvent(pbsdrBoxScope, "pbsdr_box_fill", severity, message, string(details), "hub")
	}
}