package monitor import ( "encoding/json" "fmt" "log" "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // backupStaleAfter is the maximum age of the newest offsite backup before the daily // deadline check raises expected_backup_missed. 26h covers an evening backup schedule // (e.g. ~18:00–22:00) plus headroom, so a healthy once-daily cadence never trips the // early-morning check. const backupStaleAfter = 26 * time.Hour // hostReportBackups is the minimal slice of an agent host-report the deadline check // reads to judge backup freshness (pbs_snapshots is the offsite-DR signal; backups is // the local vzdump fallback). Mirrors the agent's hub.PBSSnapshot / hub.Backup wire // contract for just the fields we need. type hostReportBackups struct { PBSSnapshots []struct { BackupTime string `json:"backup_time"` VerifyState string `json:"verify_state"` } `json:"pbs_snapshots"` Backups []struct { StartedAt string `json:"started_at"` Success bool `json:"success"` } `json:"backups"` } // backupAssessment is the verdict for one customer's offsite backup health. type backupAssessment struct { missed bool // raise expected_backup_missed reason string // human-readable cause (event message + logs) } // assessBackupFreshness decides whether a customer's latest host-report shows a healthy, // recent backup. Pure (now is injected) so the policy is unit-tested. Only POSITIVE // evidence of a problem fires an alarm: // - no PBS snapshot AND no successful vzdump in the report → missed ("no backup recorded") // - newest backup older than backupStaleAfter → missed ("stale") // - the newest PBS snapshot's verify_state is "failed" → missed ("verify failed") // // A fresh-but-not-yet-verified snapshot (verify_state "none"/"") is NOT treated as a // failure: PBS verification runs on its own cadence, so a snapshot taken hours before the // 03:00 check may legitimately be unverified. Alarming on that would re-introduce exactly // the daily false alarm this repoint removes (hence "failed" only, not "≠ ok"). func assessBackupFreshness(reportJSON string, now time.Time) backupAssessment { var hr hostReportBackups if err := json.Unmarshal([]byte(reportJSON), &hr); err != nil { // Unparseable report → can't confirm a backup. Surface it rather than swallow it. return backupAssessment{missed: true, reason: "latest host-report could not be parsed"} } var newestPBS time.Time var newestPBSVerify string havePBS := false for _, ps := range hr.PBSSnapshots { t, ok := parseBackupTime(ps.BackupTime) if !ok { continue } if !havePBS || t.After(newestPBS) { havePBS = true newestPBS = t newestPBSVerify = strings.ToLower(strings.TrimSpace(ps.VerifyState)) } } var newestVzdump time.Time haveVzdump := false for _, b := range hr.Backups { if !b.Success { continue } t, ok := parseBackupTime(b.StartedAt) if !ok { continue } if !haveVzdump || t.After(newestVzdump) { haveVzdump = true newestVzdump = t } } if !havePBS && !haveVzdump { return backupAssessment{missed: true, reason: "no PBS snapshot or successful backup in the latest host-report"} } newest := newestPBS if haveVzdump && (!havePBS || newestVzdump.After(newest)) { newest = newestVzdump } if age := now.Sub(newest); age > backupStaleAfter { return backupAssessment{missed: true, reason: fmt.Sprintf("newest backup is %s old (limit %s)", age.Round(time.Hour), backupStaleAfter)} } if havePBS && newestPBSVerify == "failed" { return backupAssessment{missed: true, reason: "newest PBS snapshot failed verification"} } return backupAssessment{missed: false} } // parseBackupTime parses an RFC3339 timestamp from a host-report and normalizes to UTC. func parseBackupTime(s string) (time.Time, bool) { s = strings.TrimSpace(s) if s == "" { return time.Time{}, false } if t, err := time.Parse(time.RFC3339, s); err == nil { return t.UTC(), true } return time.Time{}, false } // budapest returns the Europe/Budapest timezone (cached). var budapest *time.Location func init() { var err error budapest, err = time.LoadLocation("Europe/Budapest") if err != nil { // Fallback: UTC+1 (CET base; DST handled by OS if available) budapest = time.FixedZone("CET", 3600) } } // CheckBackupDeadlines checks whether active customers had their expected // daily backups and DB dumps. Runs once daily (early morning, Budapest time). // // Backup half: read the agent's latest host-report and raise expected_backup_missed // only when its PBS snapshots / vzdump backups show no fresh, verified backup (see // assessBackupFreshness). This replaced the old backup_completed-event check, which // fired daily for every healthy customer because no component emits that event anymore // (the controller's disk-tier backup moved to the agent in slice 8C). // // DB-dump half: unchanged — the in-guest controller still emits db_dump_completed, so // the event-based check there is correct. // // Customers whose nodes are "down" (no report in >1h) are skipped — they // already have staleness events. func CheckBackupDeadlines(s *store.Store, staleness *StalenessChecker, onEvent EventNotifyFunc, logger *log.Logger) { customerIDs, err := s.GetActiveCustomerIDs() if err != nil { logger.Printf("[WARN] Deadline check: failed to get active customers: %v", err) return } // Budapest midnight today now := time.Now().In(budapest) midnightBudapest := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, budapest) sinceUTC := midnightBudapest.UTC() var backupMissed, dbdumpMissed, skipped int for _, id := range customerIDs { // Skip nodes that are down — they already have staleness events if staleness != nil && staleness.GetState(id) == "down" { skipped++ continue } // Check blocked if s.IsCustomerBlocked(id) { continue } // Backup freshness from the agent's host-report (PBS snapshots + vzdump), // the authoritative offsite-backup signal post-slice-8C. reportJSON, rerr := s.GetLatestHostReportJSON(id) switch { case rerr != nil: logger.Printf("[WARN] Deadline check: failed to read host-report for %s: %v", id, rerr) case reportJSON == "": // No agent host-report at all (legacy/defunct controller-only customer). // Liveness is owned by the host-staleness checker; the backup deadline check // has no PBS data to judge here and must not emit a daily backup alarm of its // own. (The DB-dump half below still applies.) default: if a := assessBackupFreshness(reportJSON, time.Now().UTC()); a.missed { msg := "No fresh verified backup: " + a.reason if _, err := s.SaveEvent(id, "expected_backup_missed", "error", msg, "{}", "hub"); err != nil { logger.Printf("[WARN] Failed to save expected_backup_missed for %s: %v", id, err) } else if onEvent != nil { onEvent(id, "expected_backup_missed", "error", msg, "{}", "hub") } backupMissed++ } } // Check db_dump_completed / db_dump_failed since midnight dumpOK, _ := s.GetEventsByType(id, "db_dump_completed", sinceUTC) dumpFailed, _ := s.GetEventsByType(id, "db_dump_failed", sinceUTC) if len(dumpOK) == 0 && len(dumpFailed) == 0 { msg := "No DB dump completed or failed since midnight" if _, err := s.SaveEvent(id, "expected_dbdump_missed", "error", msg, "{}", "hub"); err != nil { logger.Printf("[WARN] Failed to save expected_dbdump_missed for %s: %v", id, err) } else if onEvent != nil { onEvent(id, "expected_dbdump_missed", "error", msg, "{}", "hub") } dbdumpMissed++ } } logger.Printf("[INFO] Deadline check: %d customers, %d backup missed, %d dbdump missed, %d skipped (down)", len(customerIDs), backupMissed, dbdumpMissed, skipped) }