felhom.eu/hub/internal/monitor/deadline_tiers.go

package monitor

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// R-82 Slice C — TIER-AWARE backup freshness.
//
// R-81 merged every backup signal into one "newest" and judged it against a single 26h threshold.
// That was correct while a box had exactly one whole-guest tier. With the R-82 split it is not:
// `backupStaleAfter`'s own comment records why —
//
//	"The moment PBS moves to a WEEKLY cadence, a perfectly healthy weekly snapshot is >26h old six
//	days in seven and this constant alarms on it."
//
// So each tier is now judged against ITS OWN threshold, and R-81's structure is preserved intact:
// three-valued verdicts, absence anchored at first contact, and one distinct reason string per
// failure mode.

// offsiteBackupStaleAfter is the maximum age of the newest OFFSITE (PBS) snapshot. 8 days = the 7-day
// weekly cadence plus a day of headroom — the same ratio the 26h host threshold gives a 24h cadence,
// so a healthy weekly tier never trips it and a genuinely missed week always does.
const offsiteBackupStaleAfter = 8 * 24 * time.Hour

// backupTier names one whole-guest backup tier, for thresholds and for reason strings.
type backupTier struct {
	name      string // "host" | "offsite" — used verbatim in reason strings
	staleWhen time.Duration
}

var (
	tierHost    = backupTier{name: "host", staleWhen: backupStaleAfter}
	tierOffsite = backupTier{name: "offsite", staleWhen: offsiteBackupStaleAfter}
)

// tierView is what one tier's evidence looks like in a single report.
type tierView struct {
	expected bool      // this box HAS this tier — absence is meaningful
	newest   time.Time // newest evidence in THIS report
	have     bool
	verifyOK bool // false only when the newest OFFSITE snapshot's verify_state is literally "failed"
	failed   bool
}

// pbsTargetSet returns the storage ids that are PBS-type backup storages on this host.
//
// THIS IS THE SLICE-A.4 RULE, and getting it wrong is the trap that slice recorded: a PBS-targeted
// vzdump appears in BOTH `backups[]` (as a Backup with target_id "felhom-pbs") AND in
// `pbs_snapshots[]` (enumerated independently from PBS by the agent's verify loop). Classifying by
// ARRAY MEMBERSHIP would therefore attribute a PBS backup to the host tier and make a stale host
// tier look fresh. Classify by TARGET TYPE — join target_id → storage_targets[].name → .type.
//
// storage_targets is preferred over pbs_dr.storage_id because pbs_dr is null on a box that has a PBS
// storage but no DR descriptor yet (drill-r50 was exactly that shape).
func pbsTargetSet(hr hostReportBackups) map[string]bool {
	out := map[string]bool{}
	for _, st := range hr.StorageTargets {
		if strings.EqualFold(strings.TrimSpace(st.Type), "pbs") && st.Name != "" {
			out[st.Name] = true
		}
	}
	return out
}

// splitTiers classifies one report's backup evidence into the host and offsite tiers.
//
// A tier is "expected" when the box demonstrably HAS it — either a storage of that kind is
// configured, or evidence for it exists. Without that gate every box without an offsite tier would
// alarm as soon as the anchor elapsed, which is precisely the absence-is-not-failure mistake R-81
// exists to prevent, re-introduced one level down.
func splitTiers(hr hostReportBackups) (host, offsite tierView) {
	pbs := pbsTargetSet(hr)

	for name := range pbs {
		_ = name
		offsite.expected = true
	}
	for _, st := range hr.StorageTargets {
		if pbs[st.Name] {
			continue
		}
		if strings.Contains(st.Content, "backup") {
			host.expected = true
		}
	}

	// PBS snapshots are offsite evidence by construction.
	for _, ps := range hr.PBSSnapshots {
		offsite.expected = true
		t, ok := parseBackupTime(ps.BackupTime)
		if !ok {
			continue
		}
		if !offsite.have || t.After(offsite.newest) {
			offsite.have, offsite.newest = true, t
			offsite.failed = strings.EqualFold(strings.TrimSpace(ps.VerifyState), "failed")
		}
	}

	// vzdump records land in whichever tier their TARGET belongs to.
	for _, b := range hr.Backups {
		tv := &host
		if pbs[b.TargetID] {
			tv = &offsite
		}
		tv.expected = true
		if !b.Success {
			continue
		}
		t, ok := parseBackupTime(b.StartedAt)
		if !ok {
			continue
		}
		if !tv.have || t.After(tv.newest) {
			tv.have, tv.newest = true, t
			if tv == &offsite {
				// A vzdump record carries no verify_state; it never clears a failed snapshot verdict
				// and never sets one.
				_ = tv
			}
		}
	}
	return host, offsite
}

// assessTier judges ONE tier, preserving R-81's structure exactly: absence is UNKNOWN until it
// outlives the tier's OWN threshold measured from first contact.
func assessTier(t backupTier, v tierView, windowNewest time.Time, windowHave bool, firstReportAt, now time.Time) backupAssessment {
	newest, have := v.newest, v.have
	if windowHave && (!have || windowNewest.After(newest)) {
		newest, have = windowNewest, true
	}

	if !have {
		if firstReportAt.IsZero() {
			return backupAssessment{verdict: verdictMissed,
				reason: fmt.Sprintf("%s tier: no backup evidence in any retained host-report, and no first-contact anchor to defer against", t.name)}
		}
		watched := now.Sub(firstReportAt)
		if watched <= t.staleWhen {
			return backupAssessment{verdict: verdictUnknown,
				reason: fmt.Sprintf("%s tier: no backup evidence yet, but only watching for %s (grace %s since first contact %s) — newborn tier, not a fault",
					t.name, watched.Round(time.Hour), t.staleWhen, firstReportAt.Format(time.RFC3339))}
		}
		return backupAssessment{verdict: verdictMissed,
			reason: fmt.Sprintf("%s tier: no backup evidence in any host-report for %s (limit %s, first contact %s)",
				t.name, watched.Round(time.Hour), t.staleWhen, firstReportAt.Format(time.RFC3339))}
	}

	if age := now.Sub(newest); age > t.staleWhen {
		return backupAssessment{verdict: verdictMissed,
			reason: fmt.Sprintf("%s tier: newest backup is %s old (limit %s)", t.name, age.Round(time.Hour), t.staleWhen)}
	}
	if v.failed {
		return backupAssessment{verdict: verdictMissed,
			reason: fmt.Sprintf("%s tier: newest PBS snapshot failed verification", t.name)}
	}
	return backupAssessment{verdict: verdictOK}
}

// worst folds per-tier verdicts into one. MISSED dominates UNKNOWN dominates OK, and the reason
// travels with the winner so the event still names exactly one cause. When several tiers are
// missed, both reasons are joined — a box with two broken tiers must not report only one.
func worst(as ...backupAssessment) backupAssessment {
	out := backupAssessment{verdict: verdictOK}
	var reasons []string
	for _, a := range as {
		if a.verdict > out.verdict {
			out.verdict = a.verdict
			reasons = nil
		}
		if a.verdict == out.verdict && a.reason != "" {
			reasons = append(reasons, a.reason)
		}
	}
	out.reason = strings.Join(reasons, "; ")
	return out
}

// newestBackupEvidenceByTier is newestBackupEvidence's tier-aware twin: it walks the retained
// host-report window and returns the newest evidence PER TIER.
//
// Per-tier is load-bearing, not tidiness: R-81's single "newest across everything" would let a fresh
// daily host backup satisfy the offsite tier's window lookup, which is the same class of error as
// the agent-side one Slice A fixed (a fresh local backup satisfying the weekly PBS cadence).
//
// Early exit: stop as soon as every EXPECTED tier has evidence inside its own threshold — nothing
// older can change any verdict then. The healthy path reads one row.
func newestBackupEvidenceByTier(rows []store.HostReportRow, wantHost, wantOffsite bool, now time.Time) (hostNewest time.Time, hostHave bool, offNewest time.Time, offHave bool) {
	for _, r := range rows {
		var hr hostReportBackups
		if err := json.Unmarshal([]byte(r.ReportJSON), &hr); err != nil {
			continue // one malformed retained report must not blind the scan
		}
		h, o := splitTiers(hr)
		if h.have && (!hostHave || h.newest.After(hostNewest)) {
			hostNewest, hostHave = h.newest, true
		}
		if o.have && (!offHave || o.newest.After(offNewest)) {
			offNewest, offHave = o.newest, true
		}
		hostSettled := !wantHost || (hostHave && now.Sub(hostNewest) <= tierHost.staleWhen)
		offSettled := !wantOffsite || (offHave && now.Sub(offNewest) <= tierOffsite.staleWhen)
		if hostSettled && offSettled {
			break
		}
	}
	return hostNewest, hostHave, offNewest, offHave
}