felhom.eu/hub/internal/monitor/deadline.go

package monitor

import (
	"encoding/json"
	"fmt"
	"log"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// backupStaleAfter is the maximum age of the newest offsite backup before the daily
// deadline check raises expected_backup_missed. 26h covers an evening backup schedule
// (e.g. ~18:00–22:00) plus headroom, so a healthy once-daily cadence never trips the
// early-morning check.
const backupStaleAfter = 26 * time.Hour

// hostReportBackups is the minimal slice of an agent host-report the deadline check
// reads to judge backup freshness (pbs_snapshots is the offsite-DR signal; backups is
// the local vzdump fallback). Mirrors the agent's hub.PBSSnapshot / hub.Backup wire
// contract for just the fields we need.
type hostReportBackups struct {
	PBSSnapshots []struct {
		BackupTime  string `json:"backup_time"`
		VerifyState string `json:"verify_state"`
	} `json:"pbs_snapshots"`
	Backups []struct {
		StartedAt string `json:"started_at"`
		Success   bool   `json:"success"`
	} `json:"backups"`
}

// backupAssessment is the verdict for one customer's offsite backup health.
type backupAssessment struct {
	missed bool   // raise expected_backup_missed
	reason string // human-readable cause (event message + logs)
}

// assessBackupFreshness decides whether a customer's latest host-report shows a healthy,
// recent backup. Pure (now is injected) so the policy is unit-tested. Only POSITIVE
// evidence of a problem fires an alarm:
//   - no PBS snapshot AND no successful vzdump in the report  → missed ("no backup recorded")
//   - newest backup older than backupStaleAfter               → missed ("stale")
//   - the newest PBS snapshot's verify_state is "failed"      → missed ("verify failed")
//
// A fresh-but-not-yet-verified snapshot (verify_state "none"/"") is NOT treated as a
// failure: PBS verification runs on its own cadence, so a snapshot taken hours before the
// 03:00 check may legitimately be unverified. Alarming on that would re-introduce exactly
// the daily false alarm this repoint removes (hence "failed" only, not "≠ ok").
func assessBackupFreshness(reportJSON string, now time.Time) backupAssessment {
	var hr hostReportBackups
	if err := json.Unmarshal([]byte(reportJSON), &hr); err != nil {
		// Unparseable report → can't confirm a backup. Surface it rather than swallow it.
		return backupAssessment{missed: true, reason: "latest host-report could not be parsed"}
	}

	var newestPBS time.Time
	var newestPBSVerify string
	havePBS := false
	for _, ps := range hr.PBSSnapshots {
		t, ok := parseBackupTime(ps.BackupTime)
		if !ok {
			continue
		}
		if !havePBS || t.After(newestPBS) {
			havePBS = true
			newestPBS = t
			newestPBSVerify = strings.ToLower(strings.TrimSpace(ps.VerifyState))
		}
	}

	var newestVzdump time.Time
	haveVzdump := false
	for _, b := range hr.Backups {
		if !b.Success {
			continue
		}
		t, ok := parseBackupTime(b.StartedAt)
		if !ok {
			continue
		}
		if !haveVzdump || t.After(newestVzdump) {
			haveVzdump = true
			newestVzdump = t
		}
	}

	if !havePBS && !haveVzdump {
		return backupAssessment{missed: true, reason: "no PBS snapshot or successful backup in the latest host-report"}
	}

	newest := newestPBS
	if haveVzdump && (!havePBS || newestVzdump.After(newest)) {
		newest = newestVzdump
	}
	if age := now.Sub(newest); age > backupStaleAfter {
		return backupAssessment{missed: true, reason: fmt.Sprintf("newest backup is %s old (limit %s)", age.Round(time.Hour), backupStaleAfter)}
	}
	if havePBS && newestPBSVerify == "failed" {
		return backupAssessment{missed: true, reason: "newest PBS snapshot failed verification"}
	}
	return backupAssessment{missed: false}
}

// parseBackupTime parses an RFC3339 timestamp from a host-report and normalizes to UTC.
func parseBackupTime(s string) (time.Time, bool) {
	s = strings.TrimSpace(s)
	if s == "" {
		return time.Time{}, false
	}
	if t, err := time.Parse(time.RFC3339, s); err == nil {
		return t.UTC(), true
	}
	return time.Time{}, false
}

// budapest returns the Europe/Budapest timezone (cached).
var budapest *time.Location

func init() {
	var err error
	budapest, err = time.LoadLocation("Europe/Budapest")
	if err != nil {
		// Fallback: UTC+1 (CET base; DST handled by OS if available)
		budapest = time.FixedZone("CET", 3600)
	}
}

// CheckBackupDeadlines checks whether active customers had their expected
// daily backups and DB dumps. Runs once daily (early morning, Budapest time).
//
// Backup half: read the agent's latest host-report and raise expected_backup_missed
// only when its PBS snapshots / vzdump backups show no fresh, verified backup (see
// assessBackupFreshness). This replaced the old backup_completed-event check, which
// fired daily for every healthy customer because no component emits that event anymore
// (the controller's disk-tier backup moved to the agent in slice 8C).
//
// DB-dump half: unchanged — the in-guest controller still emits db_dump_completed, so
// the event-based check there is correct.
//
// Customers whose nodes are "down" (no report in >1h) are skipped — they
// already have staleness events.
func CheckBackupDeadlines(s *store.Store, staleness *StalenessChecker, onEvent EventNotifyFunc, logger *log.Logger) {
	customerIDs, err := s.GetActiveCustomerIDs()
	if err != nil {
		logger.Printf("[WARN] Deadline check: failed to get active customers: %v", err)
		return
	}

	// Budapest midnight today
	now := time.Now().In(budapest)
	midnightBudapest := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, budapest)
	sinceUTC := midnightBudapest.UTC()

	var backupMissed, dbdumpMissed, skipped int

	for _, id := range customerIDs {
		// Skip nodes that are down — they already have staleness events
		if staleness != nil && staleness.GetState(id) == "down" {
			skipped++
			continue
		}

		// Check blocked
		if s.IsCustomerBlocked(id) {
			continue
		}

		// Backup freshness from the agent's host-report (PBS snapshots + vzdump),
		// the authoritative offsite-backup signal post-slice-8C.
		reportJSON, rerr := s.GetLatestHostReportJSON(id)
		switch {
		case rerr != nil:
			logger.Printf("[WARN] Deadline check: failed to read host-report for %s: %v", id, rerr)
		case reportJSON == "":
			// No agent host-report at all (legacy/defunct controller-only customer).
			// Liveness is owned by the host-staleness checker; the backup deadline check
			// has no PBS data to judge here and must not emit a daily backup alarm of its
			// own. (The DB-dump half below still applies.)
		default:
			if a := assessBackupFreshness(reportJSON, time.Now().UTC()); a.missed {
				msg := "No fresh verified backup: " + a.reason
				if _, err := s.SaveEvent(id, "expected_backup_missed", "error", msg, "{}", "hub"); err != nil {
					logger.Printf("[WARN] Failed to save expected_backup_missed for %s: %v", id, err)
				} else if onEvent != nil {
					onEvent(id, "expected_backup_missed", "error", msg, "{}", "hub")
				}
				backupMissed++
			}
		}

		// Check db_dump_completed / db_dump_failed since midnight
		dumpOK, _ := s.GetEventsByType(id, "db_dump_completed", sinceUTC)
		dumpFailed, _ := s.GetEventsByType(id, "db_dump_failed", sinceUTC)
		if len(dumpOK) == 0 && len(dumpFailed) == 0 {
			msg := "No DB dump completed or failed since midnight"
			if _, err := s.SaveEvent(id, "expected_dbdump_missed", "error", msg, "{}", "hub"); err != nil {
				logger.Printf("[WARN] Failed to save expected_dbdump_missed for %s: %v", id, err)
			} else if onEvent != nil {
				onEvent(id, "expected_dbdump_missed", "error", msg, "{}", "hub")
			}
			dbdumpMissed++
		}
	}

	logger.Printf("[INFO] Deadline check: %d customers, %d backup missed, %d dbdump missed, %d skipped (down)",
		len(customerIDs), backupMissed, dbdumpMissed, skipped)
}