felhom.eu/hub/internal/monitor/restoretest.go

package monitor

import (
	"encoding/json"
	"fmt"
	"log"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// R-85 Part 2 — the restore-test result becomes a SIGNAL.
//
// Until now a failed restore-test was a `[WARN]` log line in the ingest handler and nothing else:
// no event, no notification, no staleness gauge. That was true for the LOCAL tier that was already
// being tested — so the loudest DR signal this system produces was, in practice, inaudible.
//
// TWO SIGNALS, DELIBERATELY NOT MERGED. They mean different things and warrant different urgency:
//
//	restore_test_failed  — a run completed and did NOT pass. Something is broken NOW.
//	restore_test_stale   — a tier has not been PROVEN within its expected interval. Nothing has
//	                       necessarily broken; we simply no longer know whether it works.
//
// Merging them would collapse "your DR is broken" into "your DR is unverified", and the second is
// the one that quietly becomes the first.
//
// ── THE INVARIANT, inherited from R-81 ────────────────────────────────────────────────────────
//
// Absence of a signal is UNKNOWN, and becomes a fault only once it has outlived an ANCHORED window.
// This monitor family has made the opposite mistake three times (hub v0.12.0, v0.73.0, R-81); this
// is a NEW monitor written straight after the third, so it copies R-81's verdict structure rather
// than re-deriving it. A tier never proven on a newborn box is UNKNOWN, never FAILED.

// restoreProvenStaleAfter is how long a tier may go unproven before it is called stale.
//
// Derivation, not a guess: the restore-test cadence is 24h and rotation is oldest-first across two
// tiers, so each tier is proven roughly every 2 days. 7 days therefore tolerates ~3 consecutive
// missed opportunities before alarming — loud enough to matter, quiet enough not to fire on one
// skipped cycle (a deferral behind a long backup is normal, not a fault). It is also comfortably
// inside the 2-week offsite retention (operator ruling 2026-07-26), so a tier is never reported
// stale against an archive that is about to be pruned anyway.
const restoreProvenStaleAfter = 7 * 24 * time.Hour

// Event types. Operator-tier only — see the dispatcher note in RestoreTestChecker.
const (
	EventRestoreTestFailed = "restore_test_failed"
	EventRestoreTestStale  = "restore_test_stale"
)

// hostReportRestoreTests is the slice of a host-report this checker reads.
type hostReportRestoreTests struct {
	RestoreTests []struct {
		SourceArchive string `json:"source_archive"`
		SourceTier    string `json:"source_tier"`
		Pass          bool   `json:"pass"`
		Error         string `json:"error"`
		TestedAt      string `json:"tested_at"`
	} `json:"restore_tests"`
	StorageTargets []struct {
		Name    string `json:"name"`
		Type    string `json:"type"`
		Content string `json:"content"`
	} `json:"storage_targets"`
}

// RestoreTestChecker watches restore-test outcomes per customer.
type RestoreTestChecker struct {
	store   *store.Store
	logger  *log.Logger
	onEvent EventNotifyFunc
	now     func() time.Time

	mu sync.Mutex
	// failStates is edge-trigger state: customerID|archive → already reported. A failing tier is
	// re-reported only when the FAILING RUN changes, so a permanently broken tier does not emit
	// every 60s sweep — the flapping-spam rule every checker here follows.
	failStates map[string]bool
	// staleStates is customerID|tier → "ok"|"stale", so the stale signal is edge-triggered too.
	staleStates map[string]string
}

// NewRestoreTestChecker builds the checker. onEvent may be nil.
func NewRestoreTestChecker(s *store.Store, onEvent EventNotifyFunc, logger *log.Logger) *RestoreTestChecker {
	return &RestoreTestChecker{
		store: s, logger: logger, onEvent: onEvent,
		now:         func() time.Time { return time.Now().UTC() },
		failStates:  map[string]bool{},
		staleStates: map[string]string{},
	}
}

// Check sweeps every active customer. Safe to call on the shared monitor ticker.
func (c *RestoreTestChecker) Check() {
	ids, err := c.store.GetActiveCustomerIDs()
	if err != nil {
		c.logger.Printf("[WARN] restore-test check: failed to list customers: %v", err)
		return
	}
	now := c.now()
	for _, id := range ids {
		latest, rerr := c.store.GetLatestHostReportJSON(id)
		if rerr != nil || latest == "" {
			continue // no agent host-report → nothing to judge (the R-81 legacy-shape branch)
		}
		c.checkFailure(id, latest)
		c.checkStaleness(id, latest, now)
	}
}

// checkFailure raises restore_test_failed when the latest report carries a FAILED run.
func (c *RestoreTestChecker) checkFailure(customerID, reportJSON string) {
	var hr hostReportRestoreTests
	if json.Unmarshal([]byte(reportJSON), &hr) != nil {
		return
	}
	for _, rt := range hr.RestoreTests {
		if rt.Pass {
			// A pass CLEARS the edge state for that archive so a later failure re-reports.
			c.mu.Lock()
			delete(c.failStates, customerID+"|"+rt.SourceArchive)
			c.mu.Unlock()
			continue
		}
		key := customerID + "|" + rt.SourceArchive
		c.mu.Lock()
		already := c.failStates[key]
		c.failStates[key] = true
		c.mu.Unlock()
		if already {
			continue
		}
		tier := rt.SourceTier
		if tier == "" {
			tier = "unknown"
		}
		msg := fmt.Sprintf("Restore-test FAILED on the %s tier: archive %s could not be restored+booted (%s)",
			tier, rt.SourceArchive, rt.Error)
		c.emit(customerID, EventRestoreTestFailed, "error", msg)
	}
}

// checkStaleness raises restore_test_stale for a tier not PROVEN within restoreProvenStaleAfter.
//
// "Proven" means a PASSING run for that tier, found across the hub's retained host-report window —
// the R-81 mechanism. The window is load-bearing here: the agent reports only its LATEST
// restore-test and its store is in-memory, so the latest report alone cannot answer "when was the
// OTHER tier last proven?". The hub's own history can.
func (c *RestoreTestChecker) checkStaleness(customerID, latestJSON string, now time.Time) {
	var latest hostReportRestoreTests
	if json.Unmarshal([]byte(latestJSON), &latest) != nil {
		return
	}
	tiers := expectedRestoreTiers(latest)
	if len(tiers) == 0 {
		return
	}

	rows, err := c.store.GetHostReportsSince(customerID, now.Add(-2*restoreProvenStaleAfter))
	if err != nil {
		c.logger.Printf("[WARN] restore-test check: window read failed for %s: %v", customerID, err)
		return
	}
	proven := lastProvenPerTier(rows)

	first, ferr := c.store.GetFirstHostReportAt(customerID)
	if ferr != nil {
		c.logger.Printf("[WARN] restore-test check: first-contact read failed for %s: %v", customerID, ferr)
		return
	}

	for _, tier := range tiers {
		v := assessRestoreProven(tier, proven[tier], first, now)
		key := customerID + "|" + tier
		c.mu.Lock()
		prev := c.staleStates[key]
		state := "ok"
		if v.verdict == verdictMissed {
			state = "stale"
		}
		c.staleStates[key] = state
		c.mu.Unlock()

		switch {
		case state == "stale" && prev != "stale":
			c.emit(customerID, EventRestoreTestStale, "warning", v.reason)
		case v.verdict == verdictUnknown && prev == "":
			// Make the deferral VISIBLE exactly once, the v0.73.0 / R-81 precedent: a quiet check
			// must never be indistinguishable from a check that did not run.
			c.logger.Printf("[INFO] restore-test staleness: %s %s", customerID, v.reason)
		}
	}
}

// assessRestoreProven is the per-tier verdict. PURE (now injected) so the policy is unit-tested —
// the property that made R-81 provable, kept deliberately.
//
//	no proof, anchor NOT elapsed → UNKNOWN (newborn box; never an alarm)
//	no proof, anchor elapsed     → MISSED
//	proof older than the window  → MISSED
//	otherwise                    → OK
func assessRestoreProven(tier string, provenAt, firstReportAt, now time.Time) backupAssessment {
	if provenAt.IsZero() {
		if firstReportAt.IsZero() {
			return backupAssessment{verdict: verdictMissed,
				reason: fmt.Sprintf("%s tier: never restore-proven, and no first-contact anchor to defer against", tier)}
		}
		watched := now.Sub(firstReportAt)
		if watched <= restoreProvenStaleAfter {
			return backupAssessment{verdict: verdictUnknown,
				reason: fmt.Sprintf("%s tier: not restore-proven yet, but only watching for %s (grace %s since first contact %s) — newborn, not a fault",
					tier, watched.Round(time.Hour), restoreProvenStaleAfter, firstReportAt.Format(time.RFC3339))}
		}
		return backupAssessment{verdict: verdictMissed,
			reason: fmt.Sprintf("%s tier: NEVER successfully restore-proven in %s of watching (limit %s) — the tier is unverified, not known-broken",
				tier, watched.Round(time.Hour), restoreProvenStaleAfter)}
	}
	if age := now.Sub(provenAt); age > restoreProvenStaleAfter {
		return backupAssessment{verdict: verdictMissed,
			reason: fmt.Sprintf("%s tier: last successful restore-test was %s ago (limit %s) — the tier is unverified, not known-broken",
				tier, age.Round(time.Hour), restoreProvenStaleAfter)}
	}
	return backupAssessment{verdict: verdictOK}
}

// expectedRestoreTiers names the tiers this box actually HAS, so a box without an offsite tier is
// never reported stale for one. Same gate as Slice C's `expected`, and for the same reason: without
// it every box lacking a tier would alarm once the anchor elapsed — absence-is-not-failure,
// re-introduced one level down.
func expectedRestoreTiers(hr hostReportRestoreTests) []string {
	var out []string
	seenHost, seenOffsite := false, false
	for _, st := range hr.StorageTargets {
		isPBS := st.Type == "pbs"
		if isPBS && !seenOffsite {
			out = append(out, "pbs")
			seenOffsite = true
			continue
		}
		if !isPBS && !seenHost && containsBackupContent(st.Content) {
			out = append(out, "local")
			seenHost = true
		}
	}
	return out
}

func containsBackupContent(content string) bool {
	for i := 0; i+6 <= len(content); i++ {
		if content[i:i+6] == "backup" {
			return true
		}
	}
	return false
}

// lastProvenPerTier walks the retained window for the newest PASSING run per tier.
func lastProvenPerTier(rows []store.HostReportRow) map[string]time.Time {
	out := map[string]time.Time{}
	for _, r := range rows {
		var hr hostReportRestoreTests
		if json.Unmarshal([]byte(r.ReportJSON), &hr) != nil {
			continue // one malformed retained report must not blind the scan
		}
		for _, rt := range hr.RestoreTests {
			if !rt.Pass || rt.SourceTier == "" {
				continue
			}
			t, err := time.Parse(time.RFC3339, rt.TestedAt)
			if err != nil {
				continue
			}
			if cur, ok := out[rt.SourceTier]; !ok || t.After(cur) {
				out[rt.SourceTier] = t.UTC()
			}
		}
	}
	return out
}

// emit saves the event and notifies. OPERATOR-TIER ONLY: neither type has a `customerMessages`
// entry, so the dispatcher cannot route it to a customer. That is deliberate — a customer can take
// no action on a failed restore-test, and „a visszaállítási teszt nem sikerült" would frighten
// without informing. A PERSISTENTLY unproven DR tier may eventually warrant a customer-visible
// statement, but that needs copy review, not a side effect of this task.
func (c *RestoreTestChecker) emit(customerID, eventType, severity, msg string) {
	if _, err := c.store.SaveEvent(customerID, eventType, severity, msg, "{}", "hub"); err != nil {
		c.logger.Printf("[WARN] restore-test check: failed to save %s for %s: %v", eventType, customerID, err)
		return
	}
	c.logger.Printf("[%s] restore-test: %s %s", severityLabel(severity), customerID, msg)
	if c.onEvent != nil {
		c.onEvent(customerID, eventType, severity, msg, "{}", "hub")
	}
}

func severityLabel(sev string) string {
	if sev == "error" {
		return "ERROR"
	}
	return "WARN"
}