felhom.eu/hub/internal/monitor/offsite.go

package monitor

import (
	"encoding/json"
	"fmt"
	"log"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// OffsiteChecker (SLICE 4) watches each customer's offsite backup health from the controller report's
// `offsite` status object. Two independent signals, one checker (sibling of StorageFillChecker — same
// born/persistent, escalation-only emit, recovery re-arm shape; NOT bolted onto the disk checkers —
// different data source, different remedy text):
//
//   - FILL: repo_size_bytes vs the shared-model soft quota (quota_gb>0) at warn 90% / crit 95% — the
//     operator's early warning before the controller's own 100% run-refusal bites the customer.
//   - STALENESS: enabled + escrowed but no run in >48h (or never) — the silently-STUCK detector. A
//     RECENTLY-failing offsite is NOT stale (backup_failed already alerts it); staleness is the
//     complement: nothing is even trying. Pending/disabled targets are normal onboarding, never stale.
//
// Reports without an `offsite` object (pre-v0.109 controllers, offbox not enabled) are skipped nil-safe.
type OffsiteChecker struct {
	store      *store.Store
	logger     *log.Logger
	onEvent    EventNotifyFunc
	staleAfter time.Duration
	now        func() time.Time // injectable clock (tests)

	mu          sync.Mutex
	fillStates  map[string]string // customerID → fill band
	staleStates map[string]string // customerID → "ok" | "stale"
}

const defaultOffsiteStaleAfter = 48 * time.Hour

// offsiteReport mirrors the controller report's `offsite` object (v0.109.0).
type offsiteReport struct {
	Enabled       bool   `json:"enabled"`
	EscrowState   string `json:"escrow_state"`
	LastRun       string `json:"last_run"`
	LastStatus    string `json:"last_status"`
	SnapshotCount int    `json:"snapshot_count"`
	RepoSizeBytes int64  `json:"repo_size_bytes"`
	QuotaGB       int    `json:"quota_gb"`
}

// NewOffsiteChecker builds the checker. Same seeding philosophy as StorageFillChecker: already-breached
// customers are left UNSEEDED so their first Check emits (born/persistent); the dispatcher's cooldown
// dedups a hub restart.
func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNotifyFunc, logger *log.Logger) *OffsiteChecker {
	if staleAfter <= 0 {
		staleAfter = defaultOffsiteStaleAfter
	}
	oc := &OffsiteChecker{
		store: s, logger: logger, onEvent: onEvent, staleAfter: staleAfter, now: time.Now,
		fillStates: make(map[string]string), staleStates: make(map[string]string),
	}
	customers, err := s.GetCustomers()
	if err != nil {
		logger.Printf("[WARN] Offsite checker: failed to seed states: %v", err)
		return oc
	}
	var seeded int
	for _, c := range customers {
		off := parseOffsite(c.ReportJSON)
		if off == nil || s.IsCustomerBlocked(c.CustomerID) {
			continue
		}
		if band := oc.fillBand(off); band == bandOK {
			oc.fillStates[c.CustomerID] = bandOK
			seeded++
		}
		if !oc.isStale(c.CustomerID, off) {
			oc.staleStates[c.CustomerID] = "ok"
		}
	}
	logger.Printf("[INFO] Offsite checker initialized: fill warn=90%% crit=95%%, stale after %s, %d ok-seeded", staleAfter, seeded)
	return oc
}

func parseOffsite(reportJSON string) *offsiteReport {
	var r struct {
		Offsite *offsiteReport `json:"offsite"`
	}
	if json.Unmarshal([]byte(reportJSON), &r) != nil {
		return nil
	}
	return r.Offsite // nil when absent (old controller / offbox not enabled) — the caller skips
}

// fillBand maps the quota usage to a band. quota<=0 (dedicated/unset) never alerts.
func (oc *OffsiteChecker) fillBand(off *offsiteReport) string {
	if off.QuotaGB <= 0 || off.RepoSizeBytes <= 0 {
		return bandOK
	}
	pct := float64(off.RepoSizeBytes) * 100 / float64(int64(off.QuotaGB)<<30)
	return bandForPercent(pct, 90, 95)
}

// isStale: enabled + ESCROWED (the only state where runs are expected) with no run in >staleAfter (or
// never ran, ANCHORED — see below). Pending/disabled = normal onboarding, never stale. A
// recent-but-failing run is NOT stale (backup_failed owns that signal).
//
// Part-7 (v0.73.0) — the never-ran branch no longer fires on sight. The 2026-07-23 cry-wolf:
// demo-hp's tier was repaired and escrowed at 10:01Z and offsite_stale fired MINUTES later
// (`last_run:"" … threshold 48h`), because "enabled + escrowed + never ran" had no time anchor.
// Boundary: reaching this code at all means the latest report CARRIES the offsite object — i.e.
// the v0.72.0 delivery state is `applied` (Check nil-skips everything else); pre-applied
// never-ran shapes are offsite_delivery_stuck's alone — ONE STATE, ONE OWNER, never both.
// Anchor: the newest of one_time_secrets.consumed_at (delivery completed) and the customer's
// escrow-blob timestamp (host_escrow.updated_at/created_at — runs become POSSIBLE only at the
// ceremony). The EXISTING staleAfter threshold, anchored there, IS the grace — no new knob.
func (oc *OffsiteChecker) isStale(customerID string, off *offsiteReport) bool {
	if !off.Enabled || off.EscrowState != "escrowed" {
		return false
	}
	if off.LastRun == "" {
		anchor := oc.neverRanAnchor(customerID)
		if anchor.IsZero() {
			return true // legacy shape (no secret timestamps, no escrow row) — fail toward visibility, as before
		}
		return oc.now().Sub(anchor) > oc.staleAfter
	}
	t, err := time.Parse(time.RFC3339, off.LastRun)
	if err != nil {
		return true // unparseable = unknown-old — fail toward visibility
	}
	return oc.now().Sub(t) > oc.staleAfter
}

// neverRanAnchor returns the newest hub-held timestamp from which a never-ran-but-applied tier's
// staleness may be counted (zero when the hub holds neither — the pre-v0.5x legacy shape).
func (oc *OffsiteChecker) neverRanAnchor(customerID string) time.Time {
	var anchor time.Time
	if info, err := oc.store.GetOneTimeSecretInfo(customerID); err == nil && info != nil && info.ConsumedAt.After(anchor) {
		anchor = info.ConsumedAt
	}
	if t, err := oc.store.LatestEscrowTimeForCustomer(customerID); err == nil && t.After(anchor) {
		anchor = t
	}
	return anchor
}

// Check evaluates every customer's latest report. Escalation-only emits; recovery re-arms silently.
func (oc *OffsiteChecker) Check() {
	customers, err := oc.store.GetCustomers()
	if err != nil {
		oc.logger.Printf("[WARN] Offsite check failed: %v", err)
		return
	}
	oc.mu.Lock()
	defer oc.mu.Unlock()

	seen := make(map[string]bool, len(customers))
	for _, c := range customers {
		// GetCustomers can return the same customer twice when two reports tie on received_at
		// (second-resolution timestamps) — process each customer once per sweep.
		if seen[c.CustomerID] {
			continue
		}
		seen[c.CustomerID] = true
		off := parseOffsite(c.ReportJSON)
		if off == nil {
			delete(oc.fillStates, c.CustomerID) // vanished object (disabled / downgraded) → re-arm
			delete(oc.staleStates, c.CustomerID)
			continue
		}
		if oc.store.IsCustomerBlocked(c.CustomerID) {
			delete(oc.fillStates, c.CustomerID)
			delete(oc.staleStates, c.CustomerID)
			continue
		}

		// FILL (quota>0 only)
		newBand := oc.fillBand(off)
		if bandRank(newBand) > bandRank(oc.fillStates[c.CustomerID]) {
			oc.emitFill(c.CustomerID, off, newBand)
		}
		oc.fillStates[c.CustomerID] = newBand

		// STALENESS (binary, warn-severity)
		newStale := "ok"
		if oc.isStale(c.CustomerID, off) {
			newStale = "stale"
		}
		if newStale == "stale" && oc.staleStates[c.CustomerID] != "stale" {
			oc.emitStale(c.CustomerID, off)
		}
		// Part-7: make the anchored never-ran evaluation VISIBLE once (first observation of the
		// shape), so a live newborn tier's deferral is provable from the log without spamming
		// every sweep.
		if off.LastRun == "" && newStale == "ok" && off.Enabled && off.EscrowState == "escrowed" {
			if _, known := oc.staleStates[c.CustomerID]; !known {
				oc.logger.Printf("[INFO] Offsite staleness: %s never-ran within the anchored threshold (anchor %s) — newborn tier, not stale",
					c.CustomerID, oc.neverRanAnchor(c.CustomerID).UTC().Format(time.RFC3339))
			}
		}
		oc.staleStates[c.CustomerID] = newStale
	}
	for k := range oc.fillStates {
		if !seen[k] {
			delete(oc.fillStates, k)
		}
	}
	for k := range oc.staleStates {
		if !seen[k] {
			delete(oc.staleStates, k)
		}
	}
}

// GetFillState / GetStaleState expose current states for tests.
func (oc *OffsiteChecker) GetFillState(customerID string) string {
	oc.mu.Lock()
	defer oc.mu.Unlock()
	if s := oc.fillStates[customerID]; s != "" {
		return s
	}
	return "unknown"
}

func (oc *OffsiteChecker) GetStaleState(customerID string) string {
	oc.mu.Lock()
	defer oc.mu.Unlock()
	if s := oc.staleStates[customerID]; s != "" {
		return s
	}
	return "unknown"
}

func (oc *OffsiteChecker) emitFill(customerID string, off *offsiteReport, band string) {
	usedGB := off.RepoSizeBytes >> 30
	pct := float64(off.RepoSizeBytes) * 100 / float64(int64(off.QuotaGB)<<30)
	var eventType, severity, message string
	switch band {
	case bandCritical:
		eventType, severity = "offsite_fill_critical", "critical"
		message = fmt.Sprintf("Customer %s: offsite backup at %.0f%% of its %d GB quota (%d GB used) — at 100%% new offsite runs are refused; consider the freeze lever or a bigger quota", customerID, pct, off.QuotaGB, usedGB)
	case bandWarning:
		eventType, severity = "offsite_fill_warning", "warning"
		message = fmt.Sprintf("Customer %s: offsite backup at %.0f%% of its %d GB quota (%d GB used)", customerID, pct, off.QuotaGB, usedGB)
	default:
		return
	}
	details, _ := json.Marshal(map[string]any{
		"customer_id": customerID, "quota_gb": off.QuotaGB, "repo_size_bytes": off.RepoSizeBytes, "percent": pct,
	})
	oc.logger.Printf("[INFO] Offsite fill: %s %.0f%% (%s)", customerID, pct, eventType)
	if _, err := oc.store.SaveEvent(customerID, eventType, severity, message, string(details), "hub"); err != nil {
		oc.logger.Printf("[WARN] Failed to save offsite fill event for %s: %v", customerID, err)
		return
	}
	if oc.onEvent != nil {
		oc.onEvent(customerID, eventType, severity, message, string(details), "hub")
	}
}

func (oc *OffsiteChecker) emitStale(customerID string, off *offsiteReport) {
	age := "never ran"
	if off.LastRun != "" {
		if t, err := time.Parse(time.RFC3339, off.LastRun); err == nil {
			age = fmt.Sprintf("last run %s ago", oc.now().Sub(t).Round(time.Hour))
		}
	}
	message := fmt.Sprintf("Customer %s: offsite backup is STALE — enabled + escrowed but %s (threshold %s). The offsite leg is silently not running; check the controller/schedule", customerID, age, oc.staleAfter)
	details, _ := json.Marshal(map[string]any{
		"customer_id": customerID, "last_run": off.LastRun, "last_status": off.LastStatus, "stale_after": oc.staleAfter.String(),
	})
	oc.logger.Printf("[INFO] Offsite staleness: %s (%s)", customerID, age)
	if _, err := oc.store.SaveEvent(customerID, "offsite_stale", "warning", message, string(details), "hub"); err != nil {
		oc.logger.Printf("[WARN] Failed to save offsite staleness event for %s: %v", customerID, err)
		return
	}
	if oc.onEvent != nil {
		oc.onEvent(customerID, "offsite_stale", "warning", message, string(details), "hub")
	}
}