hub v0.134.0: a down box is judged on longer missed-backup lines (R-872); household outage mail at most weekly (R-873); backup_catchup_done allowed (R-871)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-10-05 09:24:14 +02:00
parent 8e873ee4a0
commit 6da3a7d866
7 changed files with 278 additions and 1 deletions
@@ -334,6 +334,68 @@ func stalenessState(staleness *StalenessChecker, customerID string) string {
 	return staleness.GetState(customerID)
 }

+// downDumpMissedAfter / downBackupMissedAfter (R-872): the lines a DOWN box is judged on. 48 h = two nights without
+// a database dump (a box that died last night is the staleness alarm's alone). 72 h for the whole-guest backup = the
+// agent's own catch-up valve (cadence 24 h + 24 h) plus a day. Decided by CC — operator may reverse.
+const (
+	downDumpMissedAfter   = 48 * time.Hour
+	downBackupMissedAfter = 72 * time.Hour
+)
+
+// judgeDownCustomer is R-872's judgement of a box that is down at the deadline. It raises at most one
+// expected_dbdump_missed and one expected_backup_missed, each only past its longer line, and never for a box that
+// was bound or first reported less than downDumpMissedAfter ago (nothing has been expected yet).
+func judgeDownCustomer(s *store.Store, id string, now time.Time, onEvent EventNotifyFunc, logger *log.Logger) (backupMissed, dbdumpMissed int) {
+	if bound, err := s.HasEverBoundHost(id); err == nil && !bound {
+		return 0, 0
+	}
+	first, ferr := s.GetFirstHostReportAt(id)
+	if ferr != nil || first.IsZero() || now.Sub(first) < downDumpMissedAfter {
+		return 0, 0
+	}
+	raise := func(typ, msg string) {
+		if _, err := s.SaveEvent(id, typ, "error", msg, "{}", "hub"); err != nil {
+			logger.Printf("[WARN] Failed to save %s for %s: %v", typ, id, err)
+		} else if onEvent != nil {
+			onEvent(id, typ, "error", msg, "{}", "hub")
+		}
+	}
+	// Database dumps: the newest db_dump_completed in the last 7 days.
+	if dumps, err := s.GetEventsByType(id, "db_dump_completed", now.Add(-backupEvidenceLookback)); err == nil {
+		var newest time.Time
+		for _, e := range dumps {
+			if e.CreatedAt.After(newest) {
+				newest = e.CreatedAt
+			}
+		}
+		if newest.IsZero() || now.Sub(newest) > downDumpMissedAfter {
+			last := "none in the last 7 days"
+			if !newest.IsZero() {
+				last = newest.UTC().Format(time.RFC3339)
+			}
+			raise("expected_dbdump_missed", fmt.Sprintf("No DB dump for over %s while the box is down at its deadline (last: %s) — it may be switched off at its backup time (R-872)",
+				downDumpMissedAfter, last))
+			dbdumpMissed = 1
+		}
+	}
+	// Whole-guest backup: the newest backup any retained host report shows.
+	if rows, err := s.GetHostReportsSince(id, now.Add(-backupEvidenceLookback)); err == nil {
+		newest, have := newestBackupEvidence(rows, now)
+		if !have || now.Sub(newest) > downBackupMissedAfter {
+			last := "none in the last 7 days"
+			if have {
+				last = newest.UTC().Format(time.RFC3339)
+			}
+			raise("expected_backup_missed", fmt.Sprintf("No fresh verified backup for over %s while the box is down at its deadline (last: %s) (R-872)",
+				downBackupMissedAfter, last))
+			backupMissed = 1
+		}
+	}
+	logger.Printf("[INFO] Deadline check: %s is DOWN — judged on the longer lines (dump %s, whole-guest %s): dump missed=%d backup missed=%d",
+		id, downDumpMissedAfter, downBackupMissedAfter, dbdumpMissed, backupMissed)
+	return backupMissed, dbdumpMissed
+}
+
 func CheckBackupDeadlines(s *store.Store, staleness *StalenessChecker, onEvent EventNotifyFunc, logger *log.Logger) {
 	customerIDs, err := s.GetActiveCustomerIDs()
 	if err != nil {
@@ -357,7 +419,24 @@ func CheckBackupDeadlines(s *store.Store, staleness *StalenessChecker, onEvent E
 		// `expected_backup_missed` / `expected_dbdump_missed` every morning about a machine we asked
 		// to be quiet. That is R-195's shape exactly — a skip keyed off the wrong fact missing the
 		// customer it would most obviously cover — which is why both doors are closed together.
-		if st := stalenessState(staleness, id); st == "down" || st == StateDisabled {
+		st := stalenessState(staleness, id)
+		if st == StateDisabled {
+			skipped++
+			continue
+		}
+		// R-872 (v0.134.0): a box that is DOWN at 05:00 is no longer skipped outright. It was, so that a box that
+		// just died raises its staleness alarm and not a backup alarm too — but a box that is down at EVERY
+		// deadline (a laptop switched off at night, Tester 2, measured 2026-10-05: "1 skipped (down)") was then
+		// never judged at all, and its missing backups stayed silent for ever. A down box is now judged on a
+		// LONGER line (downDumpMissedAfter / downBackupMissedAfter): a box that died last night still raises only
+		// its staleness alarm; a box that has gone two nights without a dump raises the backup alarm, down or not.
+		// Pinned by TestR872_*.
+		if st == "down" {
+			if !s.IsCustomerBlocked(id) {
+				b, d := judgeDownCustomer(s, id, time.Now().UTC(), onEvent, logger)
+				backupMissed += b
+				dbdumpMissed += d
+			}
 			skipped++
 			continue
 		}
@@ -0,0 +1,86 @@
+package monitor
+
+import (
+	"database/sql"
+	"io"
+	"log"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// R-872 (v0.134.0) — a box DOWN at the 05:00 deadline is judged on longer lines instead of being skipped.
+// THE MEASURED SHAPE (Tester 2, 2026-10-05 05:00 Budapest): "Deadline check: … 0 backup missed … 1 skipped (down)" —
+// a laptop off at every deadline, no dump ever, no alarm ever.
+// COMPANION RED-PROOF: restore `if st == "down" || st == StateDisabled { skipped++; continue }` → the Tester 2
+// shape raises nothing.
+
+func downBox(t *testing.T, firstReportAge time.Duration) (*store.Store, string, *StalenessChecker) {
+	t.Helper()
+	st, path := seedStalenessCustomer(t, "ok", 3*time.Hour) // the controller report is 3 h old → down
+	if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.SaveHostReport("h1", "c1", []byte(`{"pbs_snapshots":[],"backups":[]}`), store.HostReportDenorm{}); err != nil {
+		t.Fatal(err)
+	}
+	r872Backdate(t, path, "UPDATE host_reports SET received_at = ?", time.Now().UTC().Add(-firstReportAge))
+	sc, _ := newChecker(t, st)
+	sc.Check()
+	if sc.GetState("c1") != "down" {
+		t.Fatalf("setup: state %q, want down", sc.GetState("c1"))
+	}
+	return st, path, sc
+}
+
+func r872Backdate(t *testing.T, path, q string, at time.Time) {
+	t.Helper()
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	if _, err := db.Exec(q, at.Format("2006-01-02 15:04:05")); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func deadlineEvents(st *store.Store, sc *StalenessChecker) map[string]int {
+	got := map[string]int{}
+	CheckBackupDeadlines(st, sc, func(cid, et, sev, msg, det, src string) { got[et]++ }, log.New(io.Discard, "", 0))
+	return got
+}
+
+// Tester 2's shape: bound 5 days ago, down at every deadline, no dump, no backup → both alarms.
+func TestR872_DownEveryNightRaisesTheMissedAlarms(t *testing.T) {
+	st, _, sc := downBox(t, 5*24*time.Hour)
+	got := deadlineEvents(st, sc)
+	if got["expected_dbdump_missed"] != 1 || got["expected_backup_missed"] != 1 {
+		t.Fatalf("events %v — a box off at every deadline must raise the missed-backup alarms, down or not", got)
+	}
+}
+
+// A box that went down last night but made its dump yesterday evening (the catch-up) → no alarm (staleness owns it).
+func TestR872_DownWithARecentDumpIsQuiet(t *testing.T) {
+	st, path, sc := downBox(t, 5*24*time.Hour)
+	if _, err := st.SaveEvent("c1", "db_dump_completed", "info", "ok", "{}", "controller"); err != nil {
+		t.Fatal(err)
+	}
+	r872Backdate(t, path, "UPDATE events SET created_at = ? WHERE event_type = 'db_dump_completed'", time.Now().UTC().Add(-20*time.Hour))
+	ts := time.Now().UTC().Add(-30 * time.Hour).Format(time.RFC3339)
+	if err := st.SaveHostReport("h1", "c1", []byte(`{"pbs_snapshots":[{"backup_time":"`+ts+`","verify_state":"ok"}],"backups":[]}`), store.HostReportDenorm{}); err != nil {
+		t.Fatal(err)
+	}
+	if got := deadlineEvents(st, sc); got["expected_dbdump_missed"] != 0 || got["expected_backup_missed"] != 0 {
+		t.Fatalf("events %v — a dump 20 h ago and a backup 30 h ago are inside the down box's lines", got)
+	}
+}
+
+// A new box (first report a day ago) that is down → nothing expected yet.
+func TestR872_NewDownBoxIsQuiet(t *testing.T) {
+	st, _, sc := downBox(t, 24*time.Hour)
+	if got := deadlineEvents(st, sc); len(got) != 0 {
+		t.Fatalf("events %v for a box bound a day ago", got)
+	}
+}